Randomize the order of benchmark suites by default

CHANGES.md

@@ -1,3 +1,7 @@
+## Next version
+
+- Randomize suites to expose variation from effects on runtime (@polytypic)
+
 ## 0.1.3
 
 - Add `Metric.make` to allow ad hoc metrics (@polytypic)

lib/cmd.ml

@@ -70,12 +70,26 @@ let build_filter = function
            | exception Not_found -> false
     end
 
+let shuffle xs =
+  let xs = Array.of_list xs in
+  let state = Random.State.make_self_init () in
+  let n = Array.length xs in
+  for i = 0 to n - 2 do
+    let j = Random.State.int state (n - i) + i in
+    let t = xs.(i) in
+    xs.(i) <- xs.(j);
+    xs.(j) <- t
+  done;
+  Array.to_list xs
+
 let run ~benchmarks ?(budgetf = 0.025) ?(filters = []) ?(debug = false)
-    ?(output = `JSON) ?(argv = Sys.argv) ?(flush = true) () =
+    ?(output = `JSON) ?(argv = Sys.argv) ?(flush = true) ?(randomize = true) ()
+    =
   let budgetf = ref budgetf in
   let filters = ref filters in
   let debug = ref debug in
   let output = ref output in
+  let randomize = ref randomize in
 
   let rec specs =
     [
@@ -112,17 +126,21 @@ let run ~benchmarks ?(budgetf = 0.025) ?(filters = []) ?(debug = false)
   if !budgetf < 0.0 || 60.0 *. 60.0 < !budgetf then
     invalid_arg "budgetf out of range";
 
-  let results =
-    `Assoc
-      [
-        ( "results",
-          `List
-            (benchmarks
-            |> List.filter (build_filter !filters)
-            |> List.map (run_benchmark ~debug:!debug ~budgetf:!budgetf)) );
-      ]
+  let benchmark_results =
+    benchmarks
+    |> List.filter (build_filter !filters)
+    |> (if !randomize then shuffle else Fun.id)
+    |> List.map (run_benchmark ~debug:!debug ~budgetf:!budgetf)
+    |> List.sort @@ fun l r ->
+       let name_of = function
+         | `Assoc (("name", `String name) :: _) -> name
+         | _ -> failwith "bug"
+       in
+       String.compare (name_of l) (name_of r)
   in
 
+  let results = `Assoc [ ("results", `List benchmark_results) ] in
+
   begin
     match !output with
     | `JSON -> Yojson.Safe.pretty_print ~std:true Format.std_formatter results

lib/multicore_bench.mli

@@ -180,6 +180,7 @@ module Cmd : sig
     ?output:output ->
     ?argv:string array ->
     ?flush:bool ->
+    ?randomize:bool ->
     unit ->
     unit
   (** [run ~benchmarks ()] interprets command line arguments and runs the
@@ -205,6 +206,9 @@ module Cmd : sig
       - [~flush]: Whether to flush the standard output after writing it.
         Defaults to [true].
 
+      - [~randomize]: Whether to randomize the order of suites or not.  Defaults
+        to [true].
+
       Command line arguments take precedence over the optional arguments.  In
       other words, you can specify the optional arguments to give defaults for
       the benchmark executable. *)