benchmark_test.go

     1  // Copyright 2013 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package testing_test
     6  
     7  import (
     8  	"bytes"
     9  	"cmp"
    10  	"context"
    11  	"errors"
    12  	"internal/asan"
    13  	"internal/msan"
    14  	"internal/race"
    15  	"internal/testenv"
    16  	"runtime"
    17  	"slices"
    18  	"strings"
    19  	"sync/atomic"
    20  	"testing"
    21  	"text/template"
    22  	"time"
    23  )
    24  
    25  var prettyPrintTests = []struct {
    26  	v        float64
    27  	expected string
    28  }{
    29  	{0, "         0 x"},
    30  	{1234.1, "      1234 x"},
    31  	{-1234.1, "     -1234 x"},
    32  	{999.950001, "      1000 x"},
    33  	{999.949999, "       999.9 x"},
    34  	{99.9950001, "       100.0 x"},
    35  	{99.9949999, "        99.99 x"},
    36  	{-99.9949999, "       -99.99 x"},
    37  	{0.000999950001, "         0.001000 x"},
    38  	{0.000999949999, "         0.0009999 x"}, // smallest case
    39  	{0.0000999949999, "         0.0001000 x"},
    40  }
    41  
    42  func TestPrettyPrint(t *testing.T) {
    43  	for _, tt := range prettyPrintTests {
    44  		buf := new(strings.Builder)
    45  		testing.PrettyPrint(buf, tt.v, "x")
    46  		if tt.expected != buf.String() {
    47  			t.Errorf("prettyPrint(%v): expected %q, actual %q", tt.v, tt.expected, buf.String())
    48  		}
    49  	}
    50  }
    51  
    52  func TestResultString(t *testing.T) {
    53  	// Test fractional ns/op handling
    54  	r := testing.BenchmarkResult{
    55  		N: 100,
    56  		T: 240 * time.Nanosecond,
    57  	}
    58  	if r.NsPerOp() != 2 {
    59  		t.Errorf("NsPerOp: expected 2, actual %v", r.NsPerOp())
    60  	}
    61  	if want, got := "     100\t         2.400 ns/op", r.String(); want != got {
    62  		t.Errorf("String: expected %q, actual %q", want, got)
    63  	}
    64  
    65  	// Test sub-1 ns/op (issue #31005)
    66  	r.T = 40 * time.Nanosecond
    67  	if want, got := "     100\t         0.4000 ns/op", r.String(); want != got {
    68  		t.Errorf("String: expected %q, actual %q", want, got)
    69  	}
    70  
    71  	// Test 0 ns/op
    72  	r.T = 0
    73  	if want, got := "     100", r.String(); want != got {
    74  		t.Errorf("String: expected %q, actual %q", want, got)
    75  	}
    76  }
    77  
    78  func TestRunParallel(t *testing.T) {
    79  	if testing.Short() {
    80  		t.Skip("skipping in short mode")
    81  	}
    82  	testing.Benchmark(func(b *testing.B) {
    83  		procs := uint32(0)
    84  		iters := uint64(0)
    85  		b.SetParallelism(3)
    86  		b.RunParallel(func(pb *testing.PB) {
    87  			atomic.AddUint32(&procs, 1)
    88  			for pb.Next() {
    89  				atomic.AddUint64(&iters, 1)
    90  			}
    91  		})
    92  		if want := uint32(3 * runtime.GOMAXPROCS(0)); procs != want {
    93  			t.Errorf("got %v procs, want %v", procs, want)
    94  		}
    95  		if iters != uint64(b.N) {
    96  			t.Errorf("got %v iters, want %v", iters, b.N)
    97  		}
    98  	})
    99  }
   100  
   101  func TestRunParallelFail(t *testing.T) {
   102  	testing.Benchmark(func(b *testing.B) {
   103  		b.RunParallel(func(pb *testing.PB) {
   104  			// The function must be able to log/abort
   105  			// w/o crashing/deadlocking the whole benchmark.
   106  			b.Log("log")
   107  			b.Error("error")
   108  		})
   109  	})
   110  }
   111  
   112  func TestRunParallelFatal(t *testing.T) {
   113  	testing.Benchmark(func(b *testing.B) {
   114  		b.RunParallel(func(pb *testing.PB) {
   115  			for pb.Next() {
   116  				if b.N > 1 {
   117  					b.Fatal("error")
   118  				}
   119  			}
   120  		})
   121  	})
   122  }
   123  
   124  func TestRunParallelSkipNow(t *testing.T) {
   125  	testing.Benchmark(func(b *testing.B) {
   126  		b.RunParallel(func(pb *testing.PB) {
   127  			for pb.Next() {
   128  				if b.N > 1 {
   129  					b.SkipNow()
   130  				}
   131  			}
   132  		})
   133  	})
   134  }
   135  
   136  func TestBenchmarkContext(t *testing.T) {
   137  	testing.Benchmark(func(b *testing.B) {
   138  		ctx := b.Context()
   139  		if err := ctx.Err(); err != nil {
   140  			b.Fatalf("expected non-canceled context, got %v", err)
   141  		}
   142  
   143  		var innerCtx context.Context
   144  		b.Run("inner", func(b *testing.B) {
   145  			innerCtx = b.Context()
   146  			if err := innerCtx.Err(); err != nil {
   147  				b.Fatalf("expected inner benchmark to not inherit canceled context, got %v", err)
   148  			}
   149  		})
   150  		b.Run("inner2", func(b *testing.B) {
   151  			if !errors.Is(innerCtx.Err(), context.Canceled) {
   152  				t.Fatal("expected context of sibling benchmark to be canceled after its test function finished")
   153  			}
   154  		})
   155  
   156  		t.Cleanup(func() {
   157  			if !errors.Is(ctx.Err(), context.Canceled) {
   158  				t.Fatal("expected context canceled before cleanup")
   159  			}
   160  		})
   161  	})
   162  }
   163  
   164  // Some auxiliary functions for measuring allocations in a b.Loop benchmark below,
   165  // where in this case mid-stack inlining allows stack allocation of a slice.
   166  // This is based on the example in go.dev/issue/73137.
   167  
   168  func newX() []byte {
   169  	out := make([]byte, 8)
   170  	return use1(out)
   171  }
   172  
   173  //go:noinline
   174  func use1(out []byte) []byte {
   175  	return out
   176  }
   177  
   178  // An auxiliary function for measuring allocations with a simple function argument
   179  // in the b.Loop body.
   180  
   181  //go:noinline
   182  func use2(x any) {}
   183  
   184  func TestBenchmarkBLoopAllocs(t *testing.T) {
   185  	testenv.SkipIfOptimizationOff(t)
   186  	if race.Enabled || asan.Enabled || msan.Enabled {
   187  		t.Skip("skipping in case sanitizers alter allocation behavior")
   188  	}
   189  
   190  	t.Run("call-result", func(t *testing.T) {
   191  		bRet := testing.Benchmark(func(b *testing.B) {
   192  			b.ReportAllocs()
   193  			for b.Loop() {
   194  				newX()
   195  			}
   196  		})
   197  		if bRet.N == 0 {
   198  			t.Fatalf("benchmark reported 0 iterations")
   199  		}
   200  		if bRet.AllocsPerOp() != 0 {
   201  			t.Errorf("want 0 allocs, got %d", bRet.AllocsPerOp())
   202  		}
   203  	})
   204  
   205  	t.Run("call-arg", func(t *testing.T) {
   206  		bRet := testing.Benchmark(func(b *testing.B) {
   207  			b.ReportAllocs()
   208  			for b.Loop() {
   209  				use2(make([]byte, 1000))
   210  			}
   211  		})
   212  		if bRet.N == 0 {
   213  			t.Fatalf("benchmark reported 0 iterations")
   214  		}
   215  		if bRet.AllocsPerOp() != 0 {
   216  			t.Errorf("want 0 allocs, got %d", bRet.AllocsPerOp())
   217  		}
   218  	})
   219  }
   220  
   221  func ExampleB_RunParallel() {
   222  	// Parallel benchmark for text/template.Template.Execute on a single object.
   223  	testing.Benchmark(func(b *testing.B) {
   224  		templ := template.Must(template.New("test").Parse("Hello, {{.}}!"))
   225  		// RunParallel will create GOMAXPROCS goroutines
   226  		// and distribute work among them.
   227  		b.RunParallel(func(pb *testing.PB) {
   228  			// Each goroutine has its own bytes.Buffer.
   229  			var buf bytes.Buffer
   230  			for pb.Next() {
   231  				// The loop body is executed b.N times total across all goroutines.
   232  				buf.Reset()
   233  				templ.Execute(&buf, "World")
   234  			}
   235  		})
   236  	})
   237  }
   238  
   239  func TestReportMetric(t *testing.T) {
   240  	res := testing.Benchmark(func(b *testing.B) {
   241  		b.ReportMetric(12345, "ns/op")
   242  		b.ReportMetric(0.2, "frobs/op")
   243  	})
   244  	// Test built-in overriding.
   245  	if res.NsPerOp() != 12345 {
   246  		t.Errorf("NsPerOp: expected %v, actual %v", 12345, res.NsPerOp())
   247  	}
   248  	// Test stringing.
   249  	res.N = 1 // Make the output stable
   250  	want := "       1\t     12345 ns/op\t         0.2000 frobs/op"
   251  	if want != res.String() {
   252  		t.Errorf("expected %q, actual %q", want, res.String())
   253  	}
   254  }
   255  
   256  func ExampleB_ReportMetric() {
   257  	// This reports a custom benchmark metric relevant to a
   258  	// specific algorithm (in this case, sorting).
   259  	testing.Benchmark(func(b *testing.B) {
   260  		var compares int64
   261  		for b.Loop() {
   262  			s := []int{5, 4, 3, 2, 1}
   263  			slices.SortFunc(s, func(a, b int) int {
   264  				compares++
   265  				return cmp.Compare(a, b)
   266  			})
   267  		}
   268  		// This metric is per-operation, so divide by b.N and
   269  		// report it as a "/op" unit.
   270  		b.ReportMetric(float64(compares)/float64(b.N), "compares/op")
   271  		// This metric is per-time, so divide by b.Elapsed and
   272  		// report it as a "/ns" unit.
   273  		b.ReportMetric(float64(compares)/float64(b.Elapsed().Nanoseconds()), "compares/ns")
   274  	})
   275  }
   276  
   277  func ExampleB_ReportMetric_parallel() {
   278  	// This reports a custom benchmark metric relevant to a
   279  	// specific algorithm (in this case, sorting) in parallel.
   280  	testing.Benchmark(func(b *testing.B) {
   281  		var compares atomic.Int64
   282  		b.RunParallel(func(pb *testing.PB) {
   283  			for pb.Next() {
   284  				s := []int{5, 4, 3, 2, 1}
   285  				slices.SortFunc(s, func(a, b int) int {
   286  					// Because RunParallel runs the function many
   287  					// times in parallel, we must increment the
   288  					// counter atomically to avoid racing writes.
   289  					compares.Add(1)
   290  					return cmp.Compare(a, b)
   291  				})
   292  			}
   293  		})
   294  
   295  		// NOTE: Report each metric once, after all of the parallel
   296  		// calls have completed.
   297  
   298  		// This metric is per-operation, so divide by b.N and
   299  		// report it as a "/op" unit.
   300  		b.ReportMetric(float64(compares.Load())/float64(b.N), "compares/op")
   301  		// This metric is per-time, so divide by b.Elapsed and
   302  		// report it as a "/ns" unit.
   303  		b.ReportMetric(float64(compares.Load())/float64(b.Elapsed().Nanoseconds()), "compares/ns")
   304  	})
   305  }
   306
View as plain text