diff --git a/MODULE.bazel b/MODULE.bazel index 0dc84962b..b69c1c74c 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1149,10 +1149,14 @@ cuda.toolkit( use_repo(cuda, "cuda") erlang = use_extension("//erlang/toolchain:repos.bzl", "erlang") -use_repo(erlang, "dx_elvis", "dx_erlfmt", "dx_otp_windows_arm64", "dx_otp_windows_x86_64", "rules_erlang") +use_repo(erlang, "dx_elvis", "dx_erlang_argparse", "dx_erlfmt", "dx_erlperf", "dx_otp_windows_arm64", "dx_otp_windows_x86_64", "rules_erlang") elixir = use_extension("//elixir/toolchain:extensions.bzl", "elixir") -use_repo(elixir, "dx_credo", "dx_elixir", "rules_elixir") +elixir.lock( + name = "dx_benchee", + lock = "//elixir/bench:mix.lock", +) +use_repo(elixir, "dx_benchee", "dx_credo", "dx_elixir", "rules_elixir") # needed: @rules_powershell (powershell extension) loads @rules_batch bazel_dep(name = "rules_batch", version = "0.1.1") diff --git a/docs/cli/commands/bench.md b/docs/cli/commands/bench.md index 7d2b54606..5eaf729d9 100644 --- a/docs/cli/commands/bench.md +++ b/docs/cli/commands/bench.md @@ -24,7 +24,7 @@ sh_test( `dx bench` selects the `dx-bench` tests inside the scope with `attr(tags, ...)` over `tests()`, then runs them with `--runs_per_test=1 --nocache_test_results --local_test_jobs=1 --test_output=errors --test_env=DX_BENCH=1`. Bazel flags after `--` are forwarded, except the ones dx owns. -Every `_bench` macro (`go_bench`, `rust_bench`, `java_bench`, `kotlin_bench`, `python_bench`, `csharp_bench`, `fsharp_bench`, `d_bench`, `javascript_bench`, `typescript_bench`, `cc_bench`, `zig_bench`, `perl_bench`, `ruby_bench`, `dart_bench`, `php_bench`) declares `` as the manual `dx-bench` test dx runs. Test attributes such as `size` (default `small`), `timeout`, `env`, `args`, `data`, `flaky` and `shard_count` go to ``, the other attributes go to the private `_bin`, and `tags` are merged with `dx-bench` and `manual`. +Every `_bench` macro (`go_bench`, `rust_bench`, `java_bench`, `kotlin_bench`, `python_bench`, `csharp_bench`, `fsharp_bench`, `d_bench`, `javascript_bench`, `typescript_bench`, `cc_bench`, `zig_bench`, `perl_bench`, `ruby_bench`, `dart_bench`, `php_bench`, `elixir_bench`, `erlang_bench`) declares `` as the manual `dx-bench` test dx runs. Test attributes such as `size` (default `small`), `timeout`, `env`, `args`, `data`, `flaky` and `shard_count` go to ``, the other attributes go to the private `_bin`, and `tags` are merged with `dx-bench` and `manual`. Go: `go_bench(name, srcs, embed = ...)` from `//go/rules:defs.bzl` runs the `Benchmark*` functions with `-test.bench=. -test.benchmem` and turns each `ns/op`, `B/op` and `allocs/op` column into one result named ` `. The Go gazelle extension adds `_bench` next to the `go_test` when a test file declares a `Benchmark*` function. diff --git a/elixir/bench/BUILD.bazel b/elixir/bench/BUILD.bazel new file mode 100644 index 000000000..99d5207f6 --- /dev/null +++ b/elixir/bench/BUILD.bazel @@ -0,0 +1,23 @@ +load("//elixir/rules:defs.bzl", "ELIXIR_COMPATIBLE_WITH", "elixir_app") +load("//quality:fixtures.bzl", "real_source_target") + +package(default_visibility = ["//visibility:public"]) + +exports_files(["mix.lock"]) + +elixir_app( + name = "dx_bench", + testonly = True, + srcs = ["lib/dx_bench.ex"], + app_description = "Writes Benchee results as dx-bench.json.", + app_name = "dx_bench", + app_version = "0.0.0", + target_compatible_with = ELIXIR_COMPATIBLE_WITH, + deps = ["@dx_benchee//:benchee"], +) + +real_source_target( + name = "corpus_starlark", + starlark_srcs = ["BUILD.bazel"], + tags = ["corpus"], +) diff --git a/elixir/bench/lib/dx_bench.ex b/elixir/bench/lib/dx_bench.ex new file mode 100644 index 000000000..b902d5734 --- /dev/null +++ b/elixir/bench/lib/dx_bench.ex @@ -0,0 +1,25 @@ +defmodule DxBench do + @moduledoc false + + @doc "Runs Benchee on the jobs and writes dx-bench.json to the test outputs directory." + def run(jobs, opts \\ []) when is_map(jobs) do + defaults = [time: 1, warmup: 0.2, formatters: [], print: [benchmarking: false, configuration: false, fast_warning: false]] + suite = Benchee.run(jobs, Keyword.merge(defaults, opts)) + write(suite.scenarios) + suite + end + + def results(scenarios) do + for scenario <- scenarios do + stats = scenario.run_time_data.statistics + %{name: scenario.name, unit: "ns", mean: stats.average, stddev: stats.std_dev, iterations: stats.sample_size} + end + end + + defp write(scenarios) do + case System.get_env("TEST_UNDECLARED_OUTPUTS_DIR") do + nil -> :ok + dir -> File.write!(Path.join(dir, "dx-bench.json"), JSON.encode!(%{schema: "dx-bench/v1", results: results(scenarios)})) + end + end +end diff --git a/elixir/bench/mix.lock b/elixir/bench/mix.lock new file mode 100644 index 000000000..86e533e66 --- /dev/null +++ b/elixir/bench/mix.lock @@ -0,0 +1,5 @@ +%{ + "benchee": {:hex, :benchee, "1.5.1", "b95cbc36c4b98969a5c592a246e171041eb683c56bad1cb4f49a3b081ba66087", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.1", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a539301f8dfd4efc5c5123bfb9d47ebde20092a863a5b5b16c2a60d2243dfce7"}, + "deep_merge": {:hex, :deep_merge, "1.0.2", "476aa7ea61c54de96220051b998d893869069094da65b96101aebf79416f8a1e", [:mix], [], "hexpm", "737a53cdc9758fedbb608bdc213969e65729466c4ef3cd8e8726d0335dff116c"}, + "statistex": {:hex, :statistex, "1.1.1", "73612aa7f79e53c30569be065fd121e380f1cf57bc4c2da5b41be9246da18df9", [:mix], [], "hexpm", "310c4b49b34adf683de3103639006bed233ab54c08a4add65a531448e653857c"}, +} diff --git a/elixir/rules/defs.bzl b/elixir/rules/defs.bzl index 41016ca48..880558c9b 100644 --- a/elixir/rules/defs.bzl +++ b/elixir/rules/defs.bzl @@ -8,6 +8,7 @@ load("//erlang/rules:defs.bzl", _escript_archive = "erlang_escript_archive", "ru load("//erlang/rules:providers.bzl", "BeamInfo", "DialyzerPltInfo", "ErlangNifInfo") load("//erlang/toolchain:cells.bzl", "otp_compatible_with") load("//erlang/toolchain:pins.bzl", "OTP_DIALYZER_UNPORTED_CELLS") +load("//libs/starlark:fuzz_bench.bzl", "merge_tags") load("//libs/starlark:wrapper.bzl", "dx_executable_forward_rule", "dx_lcov_merger_attr", "dx_library_forward_rule", "dx_upstream_tags", "dx_wrap_test") load("//quality:sources.bzl", "QualitySourcesInfo") load(":kwargs.bzl", _COVER_LOAD = "COVER_LOAD", _ELIXIR_CELLS = "ELIXIR_CELLS", _ELIXIR_COMPATIBLE_WITH = "ELIXIR_COMPATIBLE_WITH", "elixir_app_kwargs", "elixir_cover_opts", "elixir_kwargs") @@ -175,3 +176,13 @@ def elixir_test(name, *, srcs = None, visibility = None, **kwargs): upstream = dict(kwargs, elixir_opts = elixir_cover_opts(kwargs.get("elixir_opts"))) apps = {"apps": kwargs["deps"]} if kwargs.get("deps") else None dx_wrap_test(name, _ex_unit_test, _ex_unit_forward_test, helper_first(srcs), visibility = visibility, upstream_kwargs = upstream, extra_forward_kwargs = apps, **kwargs) + +def elixir_bench(name, srcs, tags = None, deps = None, **kwargs): + """elixir_test run by dx bench: DxBench.run in srcs runs Benchee and writes dx-bench.json.""" + elixir_test( + name = name, + srcs = srcs, + tags = merge_tags(tags, "dx-bench", "manual"), + deps = (deps or []) + [Label("//elixir/bench:dx_bench")], + **kwargs + ) diff --git a/erlang/bench/BUILD.bazel b/erlang/bench/BUILD.bazel new file mode 100644 index 000000000..ebd52febe --- /dev/null +++ b/erlang/bench/BUILD.bazel @@ -0,0 +1,19 @@ +load("//erlang/rules:defs.bzl", "ERLANG_COMPATIBLE_WITH", "erlang_app") +load("//quality:fixtures.bzl", "real_source_target") + +package(default_visibility = ["//visibility:public"]) + +erlang_app( + testonly = True, + app_description = "Writes erlperf results as dx-bench.json.", + app_name = "dx_bench", + app_version = "0.0.0", + target_compatible_with = ERLANG_COMPATIBLE_WITH, + deps = ["@dx_erlperf//:erlang_app"], +) + +real_source_target( + name = "corpus_starlark", + starlark_srcs = ["BUILD.bazel"], + tags = ["corpus"], +) diff --git a/erlang/bench/src/dx_bench.erl b/erlang/bench/src/dx_bench.erl new file mode 100644 index 000000000..d5eb5c2d3 --- /dev/null +++ b/erlang/bench/src/dx_bench.erl @@ -0,0 +1,34 @@ +-module(dx_bench). + +-export([run/1, run/2]). + +%% Runs each fun with erlperf and writes dx-bench.json to the test outputs directory. +-spec run(#{term() => fun(() -> term())}) -> [map()]. +run(Jobs) -> + run(Jobs, #{}). + +-spec run(#{term() => fun(() -> term())}, erlperf:run_options()) -> [map()]. +run(Jobs, Options) -> + {ok, _} = application:ensure_all_started(erlperf), + Defaults = #{report => full, samples => 3, sample_duration => 200, warmup => 1}, + Results = [result(Name, Fun, maps:merge(Defaults, Options)) || {Name, Fun} <- lists:sort(maps:to_list(Jobs))], + write(Results), + Results. + +result(Name, Fun, #{sample_duration := Duration} = Options) -> + [#{result := #{average := Average, stddev := Stddev}}] = erlperf:benchmark([#{runner => Fun}], Options, undefined), + Scale = 1000 / Duration, + #{name => unicode:characters_to_binary(io_lib:format("~tp", [Name])), + unit => <<"ops/s">>, + mean => Average * Scale, + stddev => Stddev * Scale, + iterations => maps:get(samples, Options)}. + +write(Results) -> + case os:getenv("TEST_UNDECLARED_OUTPUTS_DIR") of + false -> + ok; + Dir -> + Document = #{schema => <<"dx-bench/v1">>, results => Results}, + ok = file:write_file(filename:join(Dir, "dx-bench.json"), json:encode(Document)) + end. diff --git a/erlang/rules/defs.bzl b/erlang/rules/defs.bzl index 21671ef12..f15fa8ad4 100644 --- a/erlang/rules/defs.bzl +++ b/erlang/rules/defs.bzl @@ -15,6 +15,7 @@ load("//erlang/rules:providers.bzl", "BeamInfo", "ErlangNifInfo") load("//erlang/toolchain:cells.bzl", "otp_compatible_with") load("//erlang/toolchain:pins.bzl", "OTP_DIALYZER_UNPORTED_CELLS", "OTP_NIF_UNPORTED_CELLS") load("//libs/launcher:defs.bzl", "NATIVE_LAUNCHER_ATTRS", "target_is_windows", "write_native_launcher") +load("//libs/starlark:fuzz_bench.bzl", "merge_tags") load("//libs/starlark:wrapper.bzl", "DX_EXEC_PROVIDES", "dx_binary_forward_kwargs", "dx_binary_upstream_kwargs", "dx_executable_forward_rule", "dx_forward_data_attr", "dx_language_kwargs", "dx_lcov_merger_attr", "dx_library_forward_rule", "dx_native_coverage_attrs", "dx_native_coverage_toolchains", "dx_native_test_coverage", "dx_quality_sources", "dx_setdefaults", "dx_test_forward_kwargs", "dx_test_upstream_kwargs", "dx_upstream_tags", "dx_wrap", "dx_wrap_test") load("//quality:sources.bzl", "QualitySourcesInfo") load(":kwargs.bzl", _ERLANG_CELLS = "ERLANG_CELLS", _ERLANG_COMPATIBLE_WITH = "ERLANG_COMPATIBLE_WITH", "erlang_app_forward_kwargs", "erlang_kwargs") @@ -539,3 +540,13 @@ def erlang_ct_suite_test(name, suite_name = "", visibility = None, **kwargs): visibility = visibility, **dx_test_forward_kwargs(kwargs) ) + +def erlang_bench(name, suite_name = "", tags = None, runtime_deps = None, **kwargs): + """ct_suite run by dx bench: dx_bench:run in the suite runs erlperf and writes dx-bench.json.""" + erlang_ct_suite_test( + name = name, + suite_name = suite_name, + tags = merge_tags(tags, "dx-bench", "manual"), + runtime_deps = (runtime_deps or []) + [Label("//erlang/bench:erlang_app")], + **kwargs + ) diff --git a/erlang/toolchain/pins.bzl b/erlang/toolchain/pins.bzl index de51103fc..cb9197285 100644 --- a/erlang/toolchain/pins.bzl +++ b/erlang/toolchain/pins.bzl @@ -63,6 +63,20 @@ TOOL_SOURCES = { ), } +# Hex releases fetched for erlang_bench: erlperf and its one dependency. +HEX_SOURCES = { + "dx_erlang_argparse": struct( + package = "argparse", + sha256 = "525979122bea3641a1dd3abc53f2add19f7f427d507018f8c7caf0693a6e78c8", + version = "2.0.0", + ), + "dx_erlperf": struct( + package = "erlperf", + sha256 = "c6fc9b9c784facaca01b826b7721efb3bcd7675b228902a182d9e0816f880029", + version = "2.3.0", + ), +} + GLIBC_FLOOR = "2.35" RUNTIME_MIRROR = JAMMY_RUNTIME_MIRROR diff --git a/erlang/toolchain/repos.bzl b/erlang/toolchain/repos.bzl index d171fc25a..350dbd72b 100644 --- a/erlang/toolchain/repos.bzl +++ b/erlang/toolchain/repos.bzl @@ -1,5 +1,6 @@ """Hermetic Erlang/OTP acquisition and toolchain registration.""" +load("//elixir/rules:hex.bzl", "hex_package") load("//libs/starlark:deb_closure.bzl", "deb_closure") load("//libs/starlark:repo.bzl", "hub_repo", "pinned_archive") load( @@ -13,6 +14,7 @@ load( "ERLFMT_VERSION", "ERTS_VERSION", "GETOPT_VERSION", + "HEX_SOURCES", "KATANA_CODE_VERSION", "OTP_RELEASE", "OTP_VERSION", @@ -220,6 +222,19 @@ erlang_escript_archive( ) """ +_HEX_BUILD = """load("@rules_erlang//:erlang_app.bzl", "erlang_app") + +package(default_visibility = ["//visibility:public"]) + +erlang_app( + app_name = "{app}", + app_version = "{version}", + deps = {deps}, +) +""" + +HEX_DEPS = {"dx_erlperf": ["@dx_erlang_argparse//:erlang_app"]} + TOOL_BUILDS = { "dx_elvis": _ELVIS_BUILD.format(version = ELVIS_VERSION), "dx_elvis_core": _ELVIS_CORE_BUILD.format(version = ELVIS_CORE_VERSION), @@ -380,6 +395,14 @@ def _erlang_impl(mctx): strip_prefix = source.strip_prefix, urls = [source.url], ) + for name, hex in sorted(HEX_SOURCES.items()): + hex_package( + name = name, + build_file_content = _HEX_BUILD.format(app = hex.package, version = hex.version, deps = repr(HEX_DEPS.get(name, []))), + package = hex.package, + sha256 = hex.sha256, + version = hex.version, + ) return mctx.extension_metadata(reproducible = True) erlang = module_extension( diff --git a/families/beam/elixir/README.md b/families/beam/elixir/README.md index ca7ab464f..5e6d9014b 100644 --- a/families/beam/elixir/README.md +++ b/families/beam/elixir/README.md @@ -69,6 +69,24 @@ escript_archive( - An `escript_archive` fails at analysis time when a bundled app ships a NIF; a NIF cannot load from an escript archive. - A test that runs no tests fails. +## Benchmarks + +`elixir_bench` is an `elixir_test` for `dx bench`. Benchee 1.5.1 comes with rules_dx. Call `DxBench.run/2` once in the test with a map of names to funs. It runs Benchee without formatters and writes `dx-bench.json`, one row per job in ns. + +```python +load("@rules_dx//elixir/rules:defs.bzl", "elixir_bench") + +elixir_bench( + name = "sum_bench", + srcs = [ + "sum_bench.exs", + "test_helper.exs", + ], +) +``` + +The target is tagged `dx-bench` and `manual`. Benchee has no coverage-guided fuzzer on the BEAM, so there is no `elixir_fuzz_test`. + ## Hex packages Declare each Hex package once, pinned to a version and the tarball digest. diff --git a/families/beam/elixir/tests/fixtures/bench/BUILD.bazel b/families/beam/elixir/tests/fixtures/bench/BUILD.bazel new file mode 100644 index 000000000..be102fda2 --- /dev/null +++ b/families/beam/elixir/tests/fixtures/bench/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_dx//elixir/rules:defs.bzl", "ELIXIR_COMPATIBLE_WITH", "elixir_bench") +load("@rules_dx//quality:fixtures.bzl", "real_source_target") + +exports_files( + glob(["*.exs"]) + ["BUILD.bazel"], + visibility = ["//visibility:public"], +) + +elixir_bench( + name = "sum_bench", + srcs = [ + "sum_bench.exs", + "test_helper.exs", + ], + target_compatible_with = ELIXIR_COMPATIBLE_WITH, +) + +real_source_target( + name = "corpus_starlark", + starlark_srcs = ["BUILD.bazel"], + tags = ["corpus"], +) diff --git a/families/beam/elixir/tests/fixtures/bench/sum_bench.exs b/families/beam/elixir/tests/fixtures/bench/sum_bench.exs new file mode 100644 index 000000000..d5fcd43c4 --- /dev/null +++ b/families/beam/elixir/tests/fixtures/bench/sum_bench.exs @@ -0,0 +1,8 @@ +defmodule SumBench do + use ExUnit.Case + + test "sums a list" do + list = Enum.to_list(1..1000) + DxBench.run(%{"reduce" => fn -> Enum.reduce(list, 0, &+/2) end, "sum" => fn -> Enum.sum(list) end}, time: 0.1, warmup: 0.05) + end +end diff --git a/families/beam/elixir/tests/fixtures/bench/test_helper.exs b/families/beam/elixir/tests/fixtures/bench/test_helper.exs new file mode 100644 index 000000000..869559e70 --- /dev/null +++ b/families/beam/elixir/tests/fixtures/bench/test_helper.exs @@ -0,0 +1 @@ +ExUnit.start() diff --git a/families/beam/erlang/README.md b/families/beam/erlang/README.md index c2b1a545e..ab8deb58c 100644 --- a/families/beam/erlang/README.md +++ b/families/beam/erlang/README.md @@ -111,6 +111,21 @@ The base PLT is built by Bazel from the pinned OTP tree. See `docs/cli/commands/quality.md` for the PLT application list and the rules. The tool is shared with Elixir: an `elixir_app` adds its own PLT. +## Benchmarks + +`erlang_bench` is a `ct_suite` for `dx bench`. erlperf 2.3.0 comes with rules_dx. Call `dx_bench:run/1,2` in a test case with a map of names to funs. It measures each fun with erlperf and writes `dx-bench.json`, one row per job in ops/s. Pass erlperf run options as the second argument. + +```python +load("@rules_dx//erlang/rules:defs.bzl", "erlang_bench") + +erlang_bench( + name = "sum_bench_SUITE", + app_name = "sum", +) +``` + +It needs the `erlang_app` and `test_erlang_app` of the package, like `ct_suite`. The target is tagged `dx-bench` and `manual`. No coverage-guided fuzzer exists for the BEAM, so there is no `erlang_fuzz_test`. + ## Runtime On Linux the OTP build comes from `builds.hex.pm` for Ubuntu 22.04. Its diff --git a/families/beam/erlang/tests/fixtures/bench/BUILD.bazel b/families/beam/erlang/tests/fixtures/bench/BUILD.bazel new file mode 100644 index 000000000..d1211f442 --- /dev/null +++ b/families/beam/erlang/tests/fixtures/bench/BUILD.bazel @@ -0,0 +1,36 @@ +load("@rules_dx//erlang/rules:defs.bzl", "ERLANG_COMPATIBLE_WITH", "erlang_app", "erlang_bench", "test_erlang_app") +load("@rules_dx//quality:fixtures.bzl", "real_source_target") + +exports_files( + glob([ + "src/*", + "test/*", + ]) + ["BUILD.bazel"], + visibility = ["//visibility:public"], +) + +erlang_app( + app_description = "Sums a list two ways.", + app_name = "sum", + app_version = "1.0.0", + target_compatible_with = ERLANG_COMPATIBLE_WITH, +) + +test_erlang_app( + app_description = "Sums a list two ways.", + app_name = "sum", + app_version = "1.0.0", + target_compatible_with = ERLANG_COMPATIBLE_WITH, +) + +erlang_bench( + name = "sum_bench_SUITE", + app_name = "sum", + target_compatible_with = ERLANG_COMPATIBLE_WITH, +) + +real_source_target( + name = "corpus_starlark", + starlark_srcs = ["BUILD.bazel"], + tags = ["corpus"], +) diff --git a/families/beam/erlang/tests/fixtures/bench/src/sum.erl b/families/beam/erlang/tests/fixtures/bench/src/sum.erl new file mode 100644 index 000000000..ee76efc4e --- /dev/null +++ b/families/beam/erlang/tests/fixtures/bench/src/sum.erl @@ -0,0 +1,7 @@ +-module(sum). + +-export([fold/1, list/1]). + +fold(List) -> lists:foldl(fun(X, Acc) -> X + Acc end, 0, List). + +list(List) -> lists:sum(List). diff --git a/families/beam/erlang/tests/fixtures/bench/test/sum_bench_SUITE.erl b/families/beam/erlang/tests/fixtures/bench/test/sum_bench_SUITE.erl new file mode 100644 index 000000000..933dad312 --- /dev/null +++ b/families/beam/erlang/tests/fixtures/bench/test/sum_bench_SUITE.erl @@ -0,0 +1,13 @@ +-module(sum_bench_SUITE). + +-export([all/0, sums/1]). + +all() -> + [sums]. + +sums(_Config) -> + List = lists:seq(1, 1000), + dx_bench:run( + #{fold => fun() -> sum:fold(List) end, list => fun() -> sum:list(List) end}, + #{samples => 2, sample_duration => 100, warmup => 0} + ). diff --git a/tools/ci/BUILD.bazel b/tools/ci/BUILD.bazel index d657f529a..5a7e96672 100644 --- a/tools/ci/BUILD.bazel +++ b/tools/ci/BUILD.bazel @@ -436,10 +436,12 @@ rust_test( "//deploy/release:corpus_starlark", "//deploy/rules:corpus_starlark", "//dockerfile/rules:corpus_starlark", + "//elixir/bench:corpus_starlark", "//elixir/rules:corpus_starlark", "//elixir/toolchain:corpus_starlark", "//elixir/tools:corpus_starlark", "//env:corpus_starlark", + "//erlang/bench:corpus_starlark", "//erlang/rules:corpus_starlark", "//erlang/toolchain:corpus_starlark", "//examples/shared-rust:corpus_starlark",