bench: elixir_bench runs Benchee and erlang_bench runs erlperf (closes #9588)

MODULE.bazel

@@ -1149,10 +1149,14 @@ cuda.toolkit(
 use_repo(cuda, "cuda")
 
 erlang = use_extension("//erlang/toolchain:repos.bzl", "erlang")
-use_repo(erlang, "dx_elvis", "dx_erlfmt", "dx_otp_windows_arm64", "dx_otp_windows_x86_64", "rules_erlang")
+use_repo(erlang, "dx_elvis", "dx_erlang_argparse", "dx_erlfmt", "dx_erlperf", "dx_otp_windows_arm64", "dx_otp_windows_x86_64", "rules_erlang")
 
 elixir = use_extension("//elixir/toolchain:extensions.bzl", "elixir")
-use_repo(elixir, "dx_credo", "dx_elixir", "rules_elixir")
+elixir.lock(
+    name = "dx_benchee",
+    lock = "//elixir/bench:mix.lock",
+)
+use_repo(elixir, "dx_benchee", "dx_credo", "dx_elixir", "rules_elixir")
 
 # needed: @rules_powershell (powershell extension) loads @rules_batch
 bazel_dep(name = "rules_batch", version = "0.1.1")

docs/cli/commands/bench.md

@@ -24,7 +24,7 @@ sh_test(
 
 `dx bench` selects the `dx-bench` tests inside the scope with `attr(tags, ...)` over `tests(<scope>)`, then runs them with `--runs_per_test=1 --nocache_test_results --local_test_jobs=1 --test_output=errors --test_env=DX_BENCH=1`. Bazel flags after `--` are forwarded, except the ones dx owns.
 
-Every `<lang>_bench` macro (`go_bench`, `rust_bench`, `java_bench`, `kotlin_bench`, `python_bench`, `csharp_bench`, `fsharp_bench`, `d_bench`, `javascript_bench`, `typescript_bench`, `cc_bench`, `zig_bench`, `perl_bench`, `ruby_bench`, `dart_bench`, `php_bench`) declares `<name>` as the manual `dx-bench` test dx runs. Test attributes such as `size` (default `small`), `timeout`, `env`, `args`, `data`, `flaky` and `shard_count` go to `<name>`, the other attributes go to the private `<name>_bin`, and `tags` are merged with `dx-bench` and `manual`.
+Every `<lang>_bench` macro (`go_bench`, `rust_bench`, `java_bench`, `kotlin_bench`, `python_bench`, `csharp_bench`, `fsharp_bench`, `d_bench`, `javascript_bench`, `typescript_bench`, `cc_bench`, `zig_bench`, `perl_bench`, `ruby_bench`, `dart_bench`, `php_bench`, `elixir_bench`, `erlang_bench`) declares `<name>` as the manual `dx-bench` test dx runs. Test attributes such as `size` (default `small`), `timeout`, `env`, `args`, `data`, `flaky` and `shard_count` go to `<name>`, the other attributes go to the private `<name>_bin`, and `tags` are merged with `dx-bench` and `manual`.
 
 Go: `go_bench(name, srcs, embed = ...)` from `//go/rules:defs.bzl` runs the `Benchmark*` functions with `-test.bench=. -test.benchmem` and turns each `ns/op`, `B/op` and `allocs/op` column into one result named `<Benchmark> <unit>`. The Go gazelle extension adds `<name>_bench` next to the `go_test` when a test file declares a `Benchmark*` function.
 

elixir/bench/BUILD.bazel

@@ -0,0 +1,23 @@
+load("//elixir/rules:defs.bzl", "ELIXIR_COMPATIBLE_WITH", "elixir_app")
+load("//quality:fixtures.bzl", "real_source_target")
+
+package(default_visibility = ["//visibility:public"])
+
+exports_files(["mix.lock"])
+
+elixir_app(
+    name = "dx_bench",
+    testonly = True,
+    srcs = ["lib/dx_bench.ex"],
+    app_description = "Writes Benchee results as dx-bench.json.",
+    app_name = "dx_bench",
+    app_version = "0.0.0",
+    target_compatible_with = ELIXIR_COMPATIBLE_WITH,
+    deps = ["@dx_benchee//:benchee"],
+)
+
+real_source_target(
+    name = "corpus_starlark",
+    starlark_srcs = ["BUILD.bazel"],
+    tags = ["corpus"],
+)

elixir/bench/lib/dx_bench.ex

@@ -0,0 +1,25 @@
+defmodule DxBench do
+  @moduledoc false
+
+  @doc "Runs Benchee on the jobs and writes dx-bench.json to the test outputs directory."
+  def run(jobs, opts \\ []) when is_map(jobs) do
+    defaults = [time: 1, warmup: 0.2, formatters: [], print: [benchmarking: false, configuration: false, fast_warning: false]]
+    suite = Benchee.run(jobs, Keyword.merge(defaults, opts))
+    write(suite.scenarios)
+    suite
+  end
+
+  def results(scenarios) do
+    for scenario <- scenarios do
+      stats = scenario.run_time_data.statistics
+      %{name: scenario.name, unit: "ns", mean: stats.average, stddev: stats.std_dev, iterations: stats.sample_size}
+    end
+  end
+
+  defp write(scenarios) do
+    case System.get_env("TEST_UNDECLARED_OUTPUTS_DIR") do
+      nil -> :ok
+      dir -> File.write!(Path.join(dir, "dx-bench.json"), JSON.encode!(%{schema: "dx-bench/v1", results: results(scenarios)}))
+    end
+  end
+end

elixir/bench/mix.lock

@@ -0,0 +1,5 @@
+%{
+  "benchee": {:hex, :benchee, "1.5.1", "b95cbc36c4b98969a5c592a246e171041eb683c56bad1cb4f49a3b081ba66087", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.1", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a539301f8dfd4efc5c5123bfb9d47ebde20092a863a5b5b16c2a60d2243dfce7"},
+  "deep_merge": {:hex, :deep_merge, "1.0.2", "476aa7ea61c54de96220051b998d893869069094da65b96101aebf79416f8a1e", [:mix], [], "hexpm", "737a53cdc9758fedbb608bdc213969e65729466c4ef3cd8e8726d0335dff116c"},
+  "statistex": {:hex, :statistex, "1.1.1", "73612aa7f79e53c30569be065fd121e380f1cf57bc4c2da5b41be9246da18df9", [:mix], [], "hexpm", "310c4b49b34adf683de3103639006bed233ab54c08a4add65a531448e653857c"},
+}

elixir/rules/defs.bzl

@@ -8,6 +8,7 @@ load("//erlang/rules:defs.bzl", _escript_archive = "erlang_escript_archive", "ru
 load("//erlang/rules:providers.bzl", "BeamInfo", "DialyzerPltInfo", "ErlangNifInfo")
 load("//erlang/toolchain:cells.bzl", "otp_compatible_with")
 load("//erlang/toolchain:pins.bzl", "OTP_DIALYZER_UNPORTED_CELLS")
+load("//libs/starlark:fuzz_bench.bzl", "merge_tags")
 load("//libs/starlark:wrapper.bzl", "dx_executable_forward_rule", "dx_lcov_merger_attr", "dx_library_forward_rule", "dx_upstream_tags", "dx_wrap_test")
 load("//quality:sources.bzl", "QualitySourcesInfo")
 load(":kwargs.bzl", _COVER_LOAD = "COVER_LOAD", _ELIXIR_CELLS = "ELIXIR_CELLS", _ELIXIR_COMPATIBLE_WITH = "ELIXIR_COMPATIBLE_WITH", "elixir_app_kwargs", "elixir_cover_opts", "elixir_kwargs")
@@ -175,3 +176,13 @@ def elixir_test(name, *, srcs = None, visibility = None, **kwargs):
     upstream = dict(kwargs, elixir_opts = elixir_cover_opts(kwargs.get("elixir_opts")))
     apps = {"apps": kwargs["deps"]} if kwargs.get("deps") else None
     dx_wrap_test(name, _ex_unit_test, _ex_unit_forward_test, helper_first(srcs), visibility = visibility, upstream_kwargs = upstream, extra_forward_kwargs = apps, **kwargs)
+
+def elixir_bench(name, srcs, tags = None, deps = None, **kwargs):
+    """elixir_test run by dx bench: DxBench.run in srcs runs Benchee and writes dx-bench.json."""
+    elixir_test(
+        name = name,
+        srcs = srcs,
+        tags = merge_tags(tags, "dx-bench", "manual"),
+        deps = (deps or []) + [Label("//elixir/bench:dx_bench")],
+        **kwargs
+    )

erlang/bench/BUILD.bazel

@@ -0,0 +1,19 @@
+load("//erlang/rules:defs.bzl", "ERLANG_COMPATIBLE_WITH", "erlang_app")
+load("//quality:fixtures.bzl", "real_source_target")
+
+package(default_visibility = ["//visibility:public"])
+
+erlang_app(
+    testonly = True,
+    app_description = "Writes erlperf results as dx-bench.json.",
+    app_name = "dx_bench",
+    app_version = "0.0.0",
+    target_compatible_with = ERLANG_COMPATIBLE_WITH,
+    deps = ["@dx_erlperf//:erlang_app"],
+)
+
+real_source_target(
+    name = "corpus_starlark",
+    starlark_srcs = ["BUILD.bazel"],
+    tags = ["corpus"],
+)

erlang/bench/src/dx_bench.erl

@@ -0,0 +1,34 @@
+-module(dx_bench).
+
+-export([run/1, run/2]).
+
+%% Runs each fun with erlperf and writes dx-bench.json to the test outputs directory.
+-spec run(#{term() => fun(() -> term())}) -> [map()].
+run(Jobs) ->
+    run(Jobs, #{}).
+
+-spec run(#{term() => fun(() -> term())}, erlperf:run_options()) -> [map()].
+run(Jobs, Options) ->
+    {ok, _} = application:ensure_all_started(erlperf),
+    Defaults = #{report => full, samples => 3, sample_duration => 200, warmup => 1},
+    Results = [result(Name, Fun, maps:merge(Defaults, Options)) || {Name, Fun} <- lists:sort(maps:to_list(Jobs))],
+    write(Results),
+    Results.
+
+result(Name, Fun, #{sample_duration := Duration} = Options) ->
+    [#{result := #{average := Average, stddev := Stddev}}] = erlperf:benchmark([#{runner => Fun}], Options, undefined),
+    Scale = 1000 / Duration,
+    #{name => unicode:characters_to_binary(io_lib:format("~tp", [Name])),
+      unit => <<"ops/s">>,
+      mean => Average * Scale,
+      stddev => Stddev * Scale,
+      iterations => maps:get(samples, Options)}.
+
+write(Results) ->
+    case os:getenv("TEST_UNDECLARED_OUTPUTS_DIR") of
+        false ->
+            ok;
+        Dir ->
+            Document = #{schema => <<"dx-bench/v1">>, results => Results},
+            ok = file:write_file(filename:join(Dir, "dx-bench.json"), json:encode(Document))
+    end.

erlang/rules/defs.bzl

@@ -15,6 +15,7 @@ load("//erlang/rules:providers.bzl", "BeamInfo", "ErlangNifInfo")
 load("//erlang/toolchain:cells.bzl", "otp_compatible_with")
 load("//erlang/toolchain:pins.bzl", "OTP_DIALYZER_UNPORTED_CELLS", "OTP_NIF_UNPORTED_CELLS")
 load("//libs/launcher:defs.bzl", "NATIVE_LAUNCHER_ATTRS", "target_is_windows", "write_native_launcher")
+load("//libs/starlark:fuzz_bench.bzl", "merge_tags")
 load("//libs/starlark:wrapper.bzl", "DX_EXEC_PROVIDES", "dx_binary_forward_kwargs", "dx_binary_upstream_kwargs", "dx_executable_forward_rule", "dx_forward_data_attr", "dx_language_kwargs", "dx_lcov_merger_attr", "dx_library_forward_rule", "dx_native_coverage_attrs", "dx_native_coverage_toolchains", "dx_native_test_coverage", "dx_quality_sources", "dx_setdefaults", "dx_test_forward_kwargs", "dx_test_upstream_kwargs", "dx_upstream_tags", "dx_wrap", "dx_wrap_test")
 load("//quality:sources.bzl", "QualitySourcesInfo")
 load(":kwargs.bzl", _ERLANG_CELLS = "ERLANG_CELLS", _ERLANG_COMPATIBLE_WITH = "ERLANG_COMPATIBLE_WITH", "erlang_app_forward_kwargs", "erlang_kwargs")
@@ -539,3 +540,13 @@ def erlang_ct_suite_test(name, suite_name = "", visibility = None, **kwargs):
         visibility = visibility,
         **dx_test_forward_kwargs(kwargs)
     )
+
+def erlang_bench(name, suite_name = "", tags = None, runtime_deps = None, **kwargs):
+    """ct_suite run by dx bench: dx_bench:run in the suite runs erlperf and writes dx-bench.json."""
+    erlang_ct_suite_test(
+        name = name,
+        suite_name = suite_name,
+        tags = merge_tags(tags, "dx-bench", "manual"),
+        runtime_deps = (runtime_deps or []) + [Label("//erlang/bench:erlang_app")],
+        **kwargs
+    )

erlang/toolchain/pins.bzl

@@ -63,6 +63,20 @@ TOOL_SOURCES = {
     ),
 }
 
+# Hex releases fetched for erlang_bench: erlperf and its one dependency.
+HEX_SOURCES = {
+    "dx_erlang_argparse": struct(
+        package = "argparse",
+        sha256 = "525979122bea3641a1dd3abc53f2add19f7f427d507018f8c7caf0693a6e78c8",
+        version = "2.0.0",
+    ),
+    "dx_erlperf": struct(
+        package = "erlperf",
+        sha256 = "c6fc9b9c784facaca01b826b7721efb3bcd7675b228902a182d9e0816f880029",
+        version = "2.3.0",
+    ),
+}
+
 GLIBC_FLOOR = "2.35"
 
 RUNTIME_MIRROR = JAMMY_RUNTIME_MIRROR

erlang/toolchain/repos.bzl

@@ -1,5 +1,6 @@
 """Hermetic Erlang/OTP acquisition and toolchain registration."""
 
+load("//elixir/rules:hex.bzl", "hex_package")
 load("//libs/starlark:deb_closure.bzl", "deb_closure")
 load("//libs/starlark:repo.bzl", "hub_repo", "pinned_archive")
 load(
@@ -13,6 +14,7 @@ load(
     "ERLFMT_VERSION",
     "ERTS_VERSION",
     "GETOPT_VERSION",
+    "HEX_SOURCES",
     "KATANA_CODE_VERSION",
     "OTP_RELEASE",
     "OTP_VERSION",
@@ -220,6 +222,19 @@ erlang_escript_archive(
 )
 """
 
+_HEX_BUILD = """load("@rules_erlang//:erlang_app.bzl", "erlang_app")
+
+package(default_visibility = ["//visibility:public"])
+
+erlang_app(
+    app_name = "{app}",
+    app_version = "{version}",
+    deps = {deps},
+)
+"""
+
+HEX_DEPS = {"dx_erlperf": ["@dx_erlang_argparse//:erlang_app"]}
+
 TOOL_BUILDS = {
     "dx_elvis": _ELVIS_BUILD.format(version = ELVIS_VERSION),
     "dx_elvis_core": _ELVIS_CORE_BUILD.format(version = ELVIS_CORE_VERSION),
@@ -380,6 +395,14 @@ def _erlang_impl(mctx):
             strip_prefix = source.strip_prefix,
             urls = [source.url],
         )
+    for name, hex in sorted(HEX_SOURCES.items()):
+        hex_package(
+            name = name,
+            build_file_content = _HEX_BUILD.format(app = hex.package, version = hex.version, deps = repr(HEX_DEPS.get(name, []))),
+            package = hex.package,
+            sha256 = hex.sha256,
+            version = hex.version,
+        )
     return mctx.extension_metadata(reproducible = True)
 
 erlang = module_extension(

families/beam/elixir/README.md

@@ -69,6 +69,24 @@ escript_archive(
 - An `escript_archive` fails at analysis time when a bundled app ships a NIF; a NIF cannot load from an escript archive.
 - A test that runs no tests fails.
 
+## Benchmarks
+
+`elixir_bench` is an `elixir_test` for `dx bench`. Benchee 1.5.1 comes with rules_dx. Call `DxBench.run/2` once in the test with a map of names to funs. It runs Benchee without formatters and writes `dx-bench.json`, one row per job in ns.
+
+```python
+load("@rules_dx//elixir/rules:defs.bzl", "elixir_bench")
+
+elixir_bench(
+    name = "sum_bench",
+    srcs = [
+        "sum_bench.exs",
+        "test_helper.exs",
+    ],
+)
+```
+
+The target is tagged `dx-bench` and `manual`. Benchee has no coverage-guided fuzzer on the BEAM, so there is no `elixir_fuzz_test`.
+
 ## Hex packages
 
 Declare each Hex package once, pinned to a version and the tarball digest.

families/beam/elixir/tests/fixtures/bench/BUILD.bazel

@@ -0,0 +1,22 @@
+load("@rules_dx//elixir/rules:defs.bzl", "ELIXIR_COMPATIBLE_WITH", "elixir_bench")
+load("@rules_dx//quality:fixtures.bzl", "real_source_target")
+
+exports_files(
+    glob(["*.exs"]) + ["BUILD.bazel"],
+    visibility = ["//visibility:public"],
+)
+
+elixir_bench(
+    name = "sum_bench",
+    srcs = [
+        "sum_bench.exs",
+        "test_helper.exs",
+    ],
+    target_compatible_with = ELIXIR_COMPATIBLE_WITH,
+)
+
+real_source_target(
+    name = "corpus_starlark",
+    starlark_srcs = ["BUILD.bazel"],
+    tags = ["corpus"],
+)

families/beam/elixir/tests/fixtures/bench/sum_bench.exs

@@ -0,0 +1,8 @@
+defmodule SumBench do
+  use ExUnit.Case
+
+  test "sums a list" do
+    list = Enum.to_list(1..1000)
+    DxBench.run(%{"reduce" => fn -> Enum.reduce(list, 0, &+/2) end, "sum" => fn -> Enum.sum(list) end}, time: 0.1, warmup: 0.05)
+  end
+end

families/beam/elixir/tests/fixtures/bench/test_helper.exs

@@ -0,0 +1 @@
+ExUnit.start()

families/beam/erlang/README.md

@@ -111,6 +111,21 @@ The base PLT is built by Bazel from the pinned OTP tree. See
 `docs/cli/commands/quality.md` for the PLT application list and the rules.
 The tool is shared with Elixir: an `elixir_app` adds its own PLT.
 
+## Benchmarks
+
+`erlang_bench` is a `ct_suite` for `dx bench`. erlperf 2.3.0 comes with rules_dx. Call `dx_bench:run/1,2` in a test case with a map of names to funs. It measures each fun with erlperf and writes `dx-bench.json`, one row per job in ops/s. Pass erlperf run options as the second argument.
+
+```python
+load("@rules_dx//erlang/rules:defs.bzl", "erlang_bench")
+
+erlang_bench(
+    name = "sum_bench_SUITE",
+    app_name = "sum",
+)
+```
+
+It needs the `erlang_app` and `test_erlang_app` of the package, like `ct_suite`. The target is tagged `dx-bench` and `manual`. No coverage-guided fuzzer exists for the BEAM, so there is no `erlang_fuzz_test`.
+
 ## Runtime
 
 On Linux the OTP build comes from `builds.hex.pm` for Ubuntu 22.04. Its

families/beam/erlang/tests/fixtures/bench/BUILD.bazel

@@ -0,0 +1,36 @@
+load("@rules_dx//erlang/rules:defs.bzl", "ERLANG_COMPATIBLE_WITH", "erlang_app", "erlang_bench", "test_erlang_app")
+load("@rules_dx//quality:fixtures.bzl", "real_source_target")
+
+exports_files(
+    glob([
+        "src/*",
+        "test/*",
+    ]) + ["BUILD.bazel"],
+    visibility = ["//visibility:public"],
+)
+
+erlang_app(
+    app_description = "Sums a list two ways.",
+    app_name = "sum",
+    app_version = "1.0.0",
+    target_compatible_with = ERLANG_COMPATIBLE_WITH,
+)
+
+test_erlang_app(
+    app_description = "Sums a list two ways.",
+    app_name = "sum",
+    app_version = "1.0.0",
+    target_compatible_with = ERLANG_COMPATIBLE_WITH,
+)
+
+erlang_bench(
+    name = "sum_bench_SUITE",
+    app_name = "sum",
+    target_compatible_with = ERLANG_COMPATIBLE_WITH,
+)
+
+real_source_target(
+    name = "corpus_starlark",
+    starlark_srcs = ["BUILD.bazel"],
+    tags = ["corpus"],
+)

families/beam/erlang/tests/fixtures/bench/src/sum.erl

@@ -0,0 +1,7 @@
+-module(sum).
+
+-export([fold/1, list/1]).
+
+fold(List) -> lists:foldl(fun(X, Acc) -> X + Acc end, 0, List).
+
+list(List) -> lists:sum(List).

families/beam/erlang/tests/fixtures/bench/test/sum_bench_SUITE.erl

@@ -0,0 +1,13 @@
+-module(sum_bench_SUITE).
+
+-export([all/0, sums/1]).
+
+all() ->
+    [sums].
+
+sums(_Config) ->
+    List = lists:seq(1, 1000),
+    dx_bench:run(
+        #{fold => fun() -> sum:fold(List) end, list => fun() -> sum:list(List) end},
+        #{samples => 2, sample_duration => 100, warmup => 0}
+    ).