Extracted from ongoing work on changing the way the parser accepts the input (i.e., the termination condition).
commit 9c21da3363054cccf7618d0b3a2ec91f68e67662 Author: Akim Demaille <[email protected]> Date: Sun May 10 11:46:40 2020 +0200 bench: add support to randomize the order of execution It's amazing how much the order matters. To a point that many of these benches are meaningless. For instance (some of the benches where run with `make -C benches/latest rand BENCHFLAGS=--benchmark_min_time=3`): compiler: g++ -std=c++11 -O2 0. %define nofinal 1. -------------------------------------------------- Benchmark Time CPU Iterations -------------------------------------------------- BM_y0 1543 ns 1541 ns 441660 BM_y1 1521 ns 1520 ns 456535 -------------------------------------------------- BM_y0 1531 ns 1530 ns 440584 BM_y1 1512 ns 1511 ns 457591 -------------------------------------------------- BM_y0 1539 ns 1538 ns 2749330 BM_y1 1516 ns 1515 ns 2771500 -------------------------------------------------- BM_y0 1571 ns 1570 ns 2600782 BM_y1 1542 ns 1541 ns 2708349 -------------------------------------------------- BM_y0 1530 ns 1529 ns 2670363 BM_y1 1519 ns 1518 ns 2764096 -------------------------------------------------- Benchmark Time CPU Iterations -------------------------------------------------- BM_y1 1529 ns 1528 ns 451937 BM_y0 1508 ns 1507 ns 453944 -------------------------------------------------- BM_y1 1525 ns 1524 ns 2750684 BM_y0 1516 ns 1515 ns 2794034 -------------------------------------------------- BM_y1 1526 ns 1525 ns 2749620 BM_y0 1515 ns 1514 ns 2808112 -------------------------------------------------- BM_y1 1524 ns 1523 ns 4475844 BM_y0 1502 ns 1501 ns 4611665 * etc/bench.pl.in: here. diff --git a/etc/bench.pl.in b/etc/bench.pl.in index 669e013c..b7d44f41 100755 --- a/etc/bench.pl.in +++ b/etc/bench.pl.in @@ -949,6 +949,10 @@ EOF bench: main \t./main \$(BENCHFLAGS) | tee -a README.md +.PHONY: rand +rand: +\t-rm main \$(OBJECTS) +\tmake bench OBJECTS="\$\$(shuf -e \$(OBJECTS) | xargs)" EOF my @obj = (); @@ -971,8 +975,9 @@ BENCHMARK_MAIN(); EOF print $makefile <<EOF; -main: @{obj} -\t$compiler -o main $cflags main.cc @obj -lbenchmark +OBJECTS = @{obj} +main: \$(OBJECTS) +\t$compiler -o main $cflags main.cc \$(OBJECTS) -lbenchmark EOF run "make";
