loop_test.go

     1  // Copyright 2024 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package testing
     6  
     7  import (
     8  	"bytes"
     9  	"strings"
    10  	"time"
    11  )
    12  
    13  // See also TestBenchmarkBLoop* in other files.
    14  
    15  func TestBenchmarkBLoop(t *T) {
    16  	var initialStart highPrecisionTime
    17  	var firstStart highPrecisionTime
    18  	var scaledStart highPrecisionTime
    19  	var runningEnd bool
    20  	runs := 0
    21  	iters := 0
    22  	firstBN := 0
    23  	restBN := 0
    24  	finalBN := 0
    25  	bRet := Benchmark(func(b *B) {
    26  		initialStart = b.start
    27  		runs++
    28  		for b.Loop() {
    29  			if iters == 0 {
    30  				firstStart = b.start
    31  				firstBN = b.N
    32  			} else {
    33  				restBN = max(restBN, b.N)
    34  			}
    35  			if iters == 1 {
    36  				scaledStart = b.start
    37  			}
    38  			iters++
    39  		}
    40  		finalBN = b.N
    41  		runningEnd = b.timerOn
    42  	})
    43  	// Verify that a b.Loop benchmark is invoked just once.
    44  	if runs != 1 {
    45  		t.Errorf("want runs == 1, got %d", runs)
    46  	}
    47  	// Verify that at least one iteration ran.
    48  	if iters == 0 {
    49  		t.Fatalf("no iterations ran")
    50  	}
    51  	// Verify that b.N, bRet.N, and the b.Loop() iteration count match.
    52  	if finalBN != iters || bRet.N != iters {
    53  		t.Errorf("benchmark iterations mismatch: %d loop iterations, final b.N=%d, bRet.N=%d", iters, finalBN, bRet.N)
    54  	}
    55  	// Verify that b.N was 0 inside the loop
    56  	if firstBN != 0 {
    57  		t.Errorf("want b.N == 0 on first iteration, got %d", firstBN)
    58  	}
    59  	if restBN != 0 {
    60  		t.Errorf("want b.N == 0 on subsequent iterations, got %d", restBN)
    61  	}
    62  	// Make sure the benchmark ran for an appropriate amount of time.
    63  	if bRet.T < benchTime.d {
    64  		t.Fatalf("benchmark ran for %s, want >= %s", bRet.T, benchTime.d)
    65  	}
    66  	// Verify that the timer is reset on the first loop, and then left alone.
    67  	if firstStart == initialStart {
    68  		t.Errorf("b.Loop did not reset the timer")
    69  	}
    70  	if scaledStart != firstStart {
    71  		t.Errorf("b.Loop stops and restarts the timer during iteration")
    72  	}
    73  	// Verify that it stopped the timer after the last loop.
    74  	if runningEnd {
    75  		t.Errorf("timer was still running after last iteration")
    76  	}
    77  }
    78  
    79  func TestBenchmarkBLoopCheapEarlyTerminate(t *T) {
    80  	if Short() {
    81  		t.Skip("B.Loop test needs to run for > 1s to saturate 1e9 iterations")
    82  	}
    83  	runCnt := 0
    84  	// Set the benchmark time high enough that we're likely to hit the 1B
    85  	// iteration limit even on very slow hardware.
    86  	// (on an AMD Ryzen 5900X, this benchmark runs in just over a second)
    87  	//
    88  	// Notably, the assertions below shouldn't fail if a test-run is slow
    89  	// enough that it doesn't saturate the limit.
    90  	const maxBenchTime = time.Second * 30
    91  	res := Benchmark(func(b *B) {
    92  		// Set the benchmark time _much_ higher than required to hit 1e9 iterations.
    93  		b.benchTime.d = maxBenchTime
    94  		for b.Loop() {
    95  			runCnt++
    96  		}
    97  	})
    98  	if runCnt > maxBenchPredictIters {
    99  		t.Errorf("loop body ran more than max (%d) times: %d", maxBenchPredictIters, runCnt)
   100  		if res.T >= maxBenchTime {
   101  			t.Logf("cheap benchmark exhausted time budget: %s; ran for %s", maxBenchTime, res.T)
   102  		}
   103  	}
   104  
   105  	if res.N != runCnt {
   106  		t.Errorf("disagreeing loop counts: res.N reported %d, while b.Loop() iterated %d times", res.N, runCnt)
   107  	}
   108  
   109  	if res.N > maxBenchPredictIters {
   110  		t.Errorf("benchmark result claims more runs than max (%d) times: %d", maxBenchPredictIters, res.N)
   111  	}
   112  
   113  }
   114  
   115  func TestBenchmarkBLoopBreak(t *T) {
   116  	var bState *B
   117  	var bLog bytes.Buffer
   118  	bRet := Benchmark(func(b *B) {
   119  		// The Benchmark function provides no access to the failure state and
   120  		// discards the log, so capture the B and save its log.
   121  		bState = b
   122  		b.common.w = &bLog
   123  
   124  		for i := 0; b.Loop(); i++ {
   125  			if i == 2 {
   126  				break
   127  			}
   128  		}
   129  	})
   130  	if !bState.failed {
   131  		t.Errorf("benchmark should have failed")
   132  	}
   133  	const wantLog = "benchmark function returned without B.Loop"
   134  	if log := bLog.String(); !strings.Contains(log, wantLog) {
   135  		t.Errorf("missing error %q in output:\n%s", wantLog, log)
   136  	}
   137  	// A benchmark that exits early should not report its target iteration count
   138  	// because it's not meaningful.
   139  	if bRet.N != 0 {
   140  		t.Errorf("want N == 0, got %d", bRet.N)
   141  	}
   142  }
   143  
   144  func TestBenchmarkBLoopError(t *T) {
   145  	// Test that a benchmark that exits early because of an error doesn't *also*
   146  	// complain that the benchmark exited early.
   147  	var bState *B
   148  	var bLog bytes.Buffer
   149  	bRet := Benchmark(func(b *B) {
   150  		bState = b
   151  		b.common.w = &bLog
   152  
   153  		for i := 0; b.Loop(); i++ {
   154  			b.Error("error")
   155  			return
   156  		}
   157  	})
   158  	if !bState.failed {
   159  		t.Errorf("benchmark should have failed")
   160  	}
   161  	const noWantLog = "benchmark function returned without B.Loop"
   162  	if log := bLog.String(); strings.Contains(log, noWantLog) {
   163  		t.Errorf("unexpected error %q in output:\n%s", noWantLog, log)
   164  	}
   165  	if bRet.N != 0 {
   166  		t.Errorf("want N == 0, got %d", bRet.N)
   167  	}
   168  }
   169  
   170  func TestBenchmarkBLoopStop(t *T) {
   171  	var bState *B
   172  	var bLog bytes.Buffer
   173  	bRet := Benchmark(func(b *B) {
   174  		bState = b
   175  		b.common.w = &bLog
   176  
   177  		for i := 0; b.Loop(); i++ {
   178  			b.StopTimer()
   179  		}
   180  	})
   181  	if !bState.failed {
   182  		t.Errorf("benchmark should have failed")
   183  	}
   184  	const wantLog = "B.Loop called with timer stopped"
   185  	if log := bLog.String(); !strings.Contains(log, wantLog) {
   186  		t.Errorf("missing error %q in output:\n%s", wantLog, log)
   187  	}
   188  	if bRet.N != 0 {
   189  		t.Errorf("want N == 0, got %d", bRet.N)
   190  	}
   191  }
   192
View as plain text