diff --git a/MODULE.bazel b/MODULE.bazel index 250f125117..0dc84962bc 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -821,7 +821,7 @@ bazel_dep(name = "bazel_features", version = "1.54.1") # Pinned SimpleCov gems Ruby tests load under `bazel coverage`; consumers need them (#3954). ruby_coverage = use_extension("//ruby/coverage:repos.bzl", "ruby_coverage") -use_repo(ruby_coverage, "dx_ruby_coverage_gems") +use_repo(ruby_coverage, "dx_ruby_bench_gems", "dx_ruby_coverage_gems") ruby_tools = use_extension("//ruby/toolchain:repos.bzl", "ruby_tools") use_repo(ruby_tools, "dx_sorbet", "rules_ruby") @@ -1026,7 +1026,11 @@ pub_tools.from_lock( lock = "//dart/tools/dart_apitool:pubspec.lock", on_version_conflict = "upgrade", ) -use_repo(pub_tools, "dart_apitool_deps") +pub_tools.from_lock( + name = "dx_dart_bench_deps", + lock = "//dart/rules/bench:pubspec.lock", +) +use_repo(pub_tools, "dart_apitool_deps", "dx_dart_bench_deps") # floor: equals tested; patched, consumers are pinned by the dx include (#5006) bazel_dep(name = "rules_julia", version = "0.8.0") @@ -1227,7 +1231,7 @@ lua_tools = use_extension("//lua/toolchain:repos.bzl", "lua") use_repo(lua_tools, "dx_luacov", "dx_luacov_reporter_lcov", "dx_luals") php = use_extension("//php/toolchain:repos.bzl", "php") -use_repo(php, "dx_composer", "dx_phpcs", "dx_phpstan", "dx_phpstan_strict_rules", "dx_phpunit", "dx_unused_public", "dx_webmozart_assert") +use_repo(php, "dx_composer", "dx_phpbench", "dx_phpcs", "dx_phpstan", "dx_phpstan_strict_rules", "dx_phpunit", "dx_unused_public", "dx_webmozart_assert") haskell = use_extension("//haskell/toolchain:extensions.bzl", "haskell") haskell.lock( diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 97fb88267f..a956e80881 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -525,7 +525,7 @@ "//dart:extensions.bzl%pub": { "general": { "bzlTransitiveDigest": "VNTf9kwkvE2nQ/+jY5ktqngyMqwf0ge9JSbNN0s5rsc=", - "usagesDigest": "FnGiRM9ZVlSoil3JRzpF3xRK8Q9uy5A5lDXhZT1M1oA=", + "usagesDigest": "YeokpRu115TyqUUIiQaK52ZoI8IdIiqyJ9iHjlLmwJ8=", "recordedInputs": [ "REPO_MAPPING:,dx_rules_dart_pub +dart_tools+dx_rules_dart_pub", "REPO_MAPPING:+dart_tools+dx_rules_dart_pub,bsd_tar_toolchains_darwin_amd64 tar.bzl++toolchains+bsd_tar_toolchains_darwin_amd64", @@ -538,6 +538,7 @@ "REPO_MAPPING:rules_dx_dart+,rules_dx ", "FILE:@@//third_party/dart/pubspec.lock d5a738f744941bbb4f4c2787d00d18d741e1c465fbe950f181884843712971dc", "FILE:@@//dart/tools/dart_apitool/pubspec.lock bcde4406b9a37a9d8df7f185033a0a067bfde50c4b786c71f5d6a38e9d138480", + "FILE:@@//dart/rules/bench/pubspec.lock 1fdb12d41f188d31d09c31a0dd13ee0e902ddc52e6a61f36e5832642c5990e3b", "FILE:@@rules_dx_dart+//dart/tests/padded_archive.pubspec.lock 387ec4ad36dc402444f6d88dc07d91027c1b65bdfa642254599ad4e7e2d513db" ], "generatedRepoSpecs": { @@ -556,6 +557,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -644,6 +646,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -732,6 +735,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -820,6 +824,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -908,6 +913,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -996,6 +1002,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -1084,6 +1091,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -1172,6 +1180,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -1260,6 +1269,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -1348,6 +1358,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -1436,6 +1447,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -1524,6 +1536,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -1612,6 +1625,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -1700,6 +1714,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -1788,6 +1803,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -1876,6 +1892,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -1964,6 +1981,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -2052,6 +2070,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -2140,6 +2159,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -2228,6 +2248,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -2316,6 +2337,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -2404,6 +2426,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -2492,6 +2515,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -2580,6 +2604,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -2668,6 +2693,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -2756,6 +2782,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -2844,6 +2871,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -2932,6 +2960,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -3020,6 +3049,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -3108,6 +3138,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -3196,6 +3227,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -3284,6 +3316,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -3372,6 +3405,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -3460,6 +3494,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -3548,6 +3583,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -3636,6 +3672,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -3724,6 +3761,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -3812,6 +3850,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -3900,6 +3939,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -3988,6 +4028,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -4076,6 +4117,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -4164,6 +4206,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -4252,6 +4295,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -4340,6 +4384,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -4428,6 +4473,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -4516,6 +4562,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -4604,6 +4651,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -4692,6 +4740,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -4780,6 +4829,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -4868,6 +4918,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -4956,6 +5007,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -5044,6 +5096,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -5132,6 +5185,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -5220,6 +5274,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -5308,6 +5363,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -5396,6 +5452,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -5484,6 +5541,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -5572,6 +5630,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -5660,6 +5719,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -5748,6 +5808,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -5836,6 +5897,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -5924,6 +5986,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -6012,6 +6075,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -6100,6 +6164,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -6188,6 +6253,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -6276,6 +6342,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -6364,6 +6431,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -6452,6 +6520,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -6540,6 +6609,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -6628,6 +6698,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -6716,6 +6787,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -6804,6 +6876,96 @@ "archive", "args", "async", + "benchmark_harness", + "boolean_selector", + "characters", + "checked_yaml", + "cli_config", + "collection", + "colorize", + "colorize_lumberdash", + "console", + "convert", + "coverage", + "crypto", + "dart_apitool", + "ffi", + "file", + "freezed_annotation", + "frontend_server_client", + "glob", + "http_multi_server", + "http_parser", + "io", + "json_annotation", + "lints", + "logging", + "lumberdash", + "matcher", + "meta", + "mime", + "nested", + "node_preamble", + "package_config", + "path", + "petitparser", + "plist_parser", + "pool", + "posix", + "pub_semver", + "pubspec_manager", + "pubspec_parse", + "shelf", + "shelf_packages_handler", + "shelf_static", + "shelf_web_socket", + "simple_sparse_list", + "source_map_stack_trace", + "source_maps", + "source_span", + "stack", + "stack_trace", + "stream_channel", + "string_scanner", + "strings", + "term_glyph", + "test", + "test_api", + "test_core", + "tuple", + "typed_data", + "unicode", + "vector_math", + "vm_service", + "watcher", + "web", + "web_socket", + "web_socket_channel", + "webkit_inspection_protocol", + "xml", + "yaml" + ], + "explicit_packages": [], + "code_assets": [], + "ignore_hook": false + } + }, + "dart_pub__benchmark_harness": { + "repoRuleId": "@@+dart_tools+dx_rules_dart_pub//dart/pub:pub_lock_package.bzl%pub_lock_package", + "attributes": { + "package_name": "benchmark_harness", + "version": "2.4.0", + "sha256": "a2d3c4c83cac0126bf38e41eaf7bd9ed4f6635f1ee1a0cbc6f79fa9736c62cbd", + "base_url": "https://pub.dev", + "hub_name": "dart_pub", + "lock_packages": [ + "_fe_analyzer_shared", + "analyzer", + "ansicolor", + "archive", + "args", + "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -6892,6 +7054,7 @@ "archive", "args", "async", + "benchmark_harness", "boolean_selector", "characters", "checked_yaml", @@ -7074,6 +7237,17 @@ ] } }, + "dx_dart_bench_deps": { + "repoRuleId": "@@+dart_tools+dx_rules_dart_pub//dart/pub:pub_lock_hub.bzl%pub_lock_hub", + "attributes": { + "hub_name": "dx_dart_bench_deps", + "spoke_prefix": "dart_pub", + "packages": [ + "args", + "benchmark_harness" + ] + } + }, "dx_dart_padded_archive": { "repoRuleId": "@@+dart_tools+dx_rules_dart_pub//dart/pub:pub_lock_hub.bzl%pub_lock_hub", "attributes": { @@ -7087,7 +7261,8 @@ }, "moduleExtensionMetadata": { "explicitRootModuleDirectDeps": [ - "dart_apitool_deps" + "dart_apitool_deps", + "dx_dart_bench_deps" ], "explicitRootModuleDirectDevDeps": [ "dx_dart_deps" diff --git a/dart/README.md b/dart/README.md index 18c5786741..ee3a488320 100644 --- a/dart/README.md +++ b/dart/README.md @@ -40,6 +40,7 @@ dart_test( | `dart_library` | A package from `.dart` files, importable as `package:` | | `dart_binary` | A compiled executable that runs with `bazel run` | | `dart_test` | Runs one test file with the package:test runner | +| `dart_bench` | `dart_test` for `dx bench`. `main` calls `runBenchmarks` from `package:dx_bench` with `BenchmarkBase` objects; seconds per run. Depend on `@dx_dart_bench_deps//:benchmark_harness` for the base class | | `dart_codegen` | Runs a generator script over one `.dart` file and makes the output files | `dart_js_binary` and `dart_wasm_binary` compile a web app to `.js`, or to `.wasm` plus its `.mjs` loader and `.support.js`. diff --git a/dart/rules/bench/BUILD.bazel b/dart/rules/bench/BUILD.bazel new file mode 100644 index 0000000000..209d3a8b3c --- /dev/null +++ b/dart/rules/bench/BUILD.bazel @@ -0,0 +1,24 @@ +load("//dart/rules:defs.bzl", "dart_library") +load("//quality:fixtures.bzl", "real_source_target") + +# gazelle:dx_external pub dart/rules/bench/pubspec.lock dx_dart_bench_deps + +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "pubspec.lock", + "pubspec.yaml", +]) + +dart_library( + name = "bench", + srcs = ["lib/dx_bench.dart"], + package_name = "dx_bench", + deps = ["@dx_dart_bench_deps//:benchmark_harness"], +) + +real_source_target( + name = "corpus_starlark", + starlark_srcs = ["BUILD.bazel"], + tags = ["corpus"], +) diff --git a/dart/rules/bench/lib/dx_bench.dart b/dart/rules/bench/lib/dx_bench.dart new file mode 100644 index 0000000000..db2032a4e3 --- /dev/null +++ b/dart/rules/bench/lib/dx_bench.dart @@ -0,0 +1,24 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:benchmark_harness/benchmark_harness.dart'; + +/// Measures each benchmark and writes dx-bench.json, in seconds per run. +void runBenchmarks(List benchmarks) { + final results = >[]; + for (final benchmark in benchmarks) { + final micros = benchmark.measure(); + results.add({ + 'name': benchmark.name, + 'unit': 's', + 'mean': micros / 1e6, + 'stddev': 0.0, + 'iterations': 1, + }); + } + final dir = Platform.environment['TEST_UNDECLARED_OUTPUTS_DIR']; + if (dir == null) { + throw StateError('TEST_UNDECLARED_OUTPUTS_DIR is not set'); + } + File('$dir/dx-bench.json').writeAsStringSync(jsonEncode({'schema': 'dx-bench/v1', 'results': results})); +} diff --git a/dart/rules/bench/pubspec.lock b/dart/rules/bench/pubspec.lock new file mode 100644 index 0000000000..c52820a933 --- /dev/null +++ b/dart/rules/bench/pubspec.lock @@ -0,0 +1,21 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + args: + dependency: transitive + description: + name: args + sha256: d0481093c50b1da8910eb0bb301626d4d8eb7284aa739614d2b394ee09e3ea04 + url: "https://pub.dev" + source: hosted + version: "2.7.0" + benchmark_harness: + dependency: "direct main" + description: + name: benchmark_harness + sha256: a2d3c4c83cac0126bf38e41eaf7bd9ed4f6635f1ee1a0cbc6f79fa9736c62cbd + url: "https://pub.dev" + source: hosted + version: "2.4.0" +sdks: + dart: ">=3.10.0 <4.0.0" diff --git a/dart/rules/bench/pubspec.yaml b/dart/rules/bench/pubspec.yaml new file mode 100644 index 0000000000..fb8ecb5647 --- /dev/null +++ b/dart/rules/bench/pubspec.yaml @@ -0,0 +1,6 @@ +name: dx_dart_bench +publish_to: none +environment: + sdk: ^3.10.0 +dependencies: + benchmark_harness: 2.4.0 diff --git a/dart/rules/defs.bzl b/dart/rules/defs.bzl index b520a5db17..2ad2ce91bf 100644 --- a/dart/rules/defs.bzl +++ b/dart/rules/defs.bzl @@ -4,6 +4,7 @@ load("@rules_dart//dart:defs.bzl", _dart_binary = "dart_binary", _dart_codegen = load("@rules_dart//dart:providers.bzl", "DartAnalyzableInfo", "DartCompileInfo", "DartInfo") load("//libs/starlark:cells.bzl", "unsupported_constraint") load("//libs/starlark:entry.bzl", "dx_entry_quality_srcs", "dx_infer_main", "dx_main_candidates") +load("//libs/starlark:fuzz_bench.bzl", "merge_tags") load("//libs/starlark:wrapper.bzl", "DX_EXEC_PROVIDES", "dx_executable_forward_rule", "dx_language_kwargs", "dx_lcov_merger_attr", "dx_library_forward_rule", "dx_wrap", "dx_wrap_binary", "dx_wrap_test") load("//quality:sources.bzl", "QualitySourcesInfo", "QualityToolsInfo") @@ -108,3 +109,14 @@ def dart_test(name, *, main = None, srcs = None, visibility = None, **kwargs): main = dx_infer_main("dart_test", name, main, srcs, dx_main_candidates(name, ".dart", ["test"])) kwargs = dx_language_kwargs(kwargs, DART_DEFERRED_SKIP_TAGS, DART_COMPATIBLE_WITH) dx_wrap_test(name, _dart_test, _dart_forward_test, srcs, visibility = visibility, upstream_kwargs = dict(kwargs, main = main), quality_srcs = dx_entry_quality_srcs(main, srcs), **kwargs) + +def dart_bench(name, main = None, srcs = None, deps = None, tags = None, **kwargs): + """A dart_test run by dx bench: `main` calls runBenchmarks from package:dx_bench with BenchmarkBase objects, which writes dx-bench.json.""" + dart_test( + name = name, + main = main, + srcs = srcs, + deps = (deps or []) + [Label("//dart/rules/bench")], + tags = merge_tags(tags, "dx-bench", "manual"), + **kwargs + ) diff --git a/dart/tests/bench/BUILD.bazel b/dart/tests/bench/BUILD.bazel new file mode 100644 index 0000000000..7d8d6c482a --- /dev/null +++ b/dart/tests/bench/BUILD.bazel @@ -0,0 +1,19 @@ +load("//dart/rules:defs.bzl", "dart_bench") +load("//quality:fixtures.bzl", "real_source_target") + +package( + default_testonly = True, + default_visibility = ["//:__subpackages__"], +) + +dart_bench( + name = "sum_bench", + main = "sum_bench.dart", + deps = ["@dx_dart_bench_deps//:benchmark_harness"], +) + +real_source_target( + name = "corpus_starlark", + starlark_srcs = ["BUILD.bazel"], + tags = ["corpus"], +) diff --git a/dart/tests/bench/sum_bench.dart b/dart/tests/bench/sum_bench.dart new file mode 100644 index 0000000000..8c7d8629bc --- /dev/null +++ b/dart/tests/bench/sum_bench.dart @@ -0,0 +1,19 @@ +import 'package:benchmark_harness/benchmark_harness.dart'; +import 'package:dx_bench/dx_bench.dart'; + +class SumBench extends BenchmarkBase { + SumBench() : super('sum'); + + var total = 0; + + @override + void run() { + for (var i = 0; i < 1000; i++) { + total += i; + } + } +} + +void main() { + runBenchmarks([SumBench()]); +} diff --git a/docs/cli/commands/README.md b/docs/cli/commands/README.md index d8e65728a3..e85073a667 100644 --- a/docs/cli/commands/README.md +++ b/docs/cli/commands/README.md @@ -327,6 +327,7 @@ dx rule that wraps it. | | | `@rules_dx//d/rules:defs.bzl` | `d_bench` | | | | `@rules_dx//d/rules:defs.bzl` | `d_fuzz_test` | | | | `@rules_dx//d/rules:defs.bzl` | `d_quality_srcs` | +| | | `@rules_dx//dart/rules:defs.bzl` | `dart_bench` | | | | `@rules_dx//dart/rules:defs.bzl` | `dart_binary` | | | | `@rules_dx//dart/rules:defs.bzl` | `dart_js_binary` | | | | `@rules_dx//dart/rules:defs.bzl` | `dart_library` | @@ -482,6 +483,8 @@ dx rule that wraps it. | | | `@rules_dx//ocaml/rules:defs.bzl` | `toolchain_choice` | | | | `@rules_dx//ocaml/rules:defs.bzl` | `toolchain_settings` | | | | `@rules_dx//ocaml/tests/native:defs.bzl` | `sys_cell_test` | +| | | `@rules_dx//perl/rules:defs.bzl` | `perl_bench` | +| | | `@rules_dx//php/rules:defs.bzl` | `php_bench` | | | | `@rules_dx//php/rules:defs.bzl` | `php_binary` | | | | `@rules_dx//php/rules:defs.bzl` | `php_library` | | | | `@rules_dx//php/rules:defs.bzl` | `php_test` | @@ -516,6 +519,7 @@ dx rule that wraps it. | | | `@rules_dx//rego/rules:defs.bzl` | `rego_library` | | | | `@rules_dx//rego/rules:defs.bzl` | `rego_test` | | | | `@rules_dx//rst/rules:defs.bzl` | `rst_library` | +| | | `@rules_dx//ruby/rules:defs.bzl` | `ruby_bench` | | | | `@rules_dx//ruby/rules:defs.bzl` | `ruby_coverage_lib_dirs` | | | | `@rules_dx//rust/rules:defs.bzl` | `rust_bench` | | | | `@rules_dx//rust/rules:defs.bzl` | `rust_binary` | diff --git a/docs/cli/commands/bench.md b/docs/cli/commands/bench.md index b569e3b4a8..7d2b546069 100644 --- a/docs/cli/commands/bench.md +++ b/docs/cli/commands/bench.md @@ -24,7 +24,7 @@ sh_test( `dx bench` selects the `dx-bench` tests inside the scope with `attr(tags, ...)` over `tests()`, then runs them with `--runs_per_test=1 --nocache_test_results --local_test_jobs=1 --test_output=errors --test_env=DX_BENCH=1`. Bazel flags after `--` are forwarded, except the ones dx owns. -Every `_bench` macro (`go_bench`, `rust_bench`, `java_bench`, `kotlin_bench`, `python_bench`, `csharp_bench`, `fsharp_bench`, `d_bench`, `javascript_bench`, `typescript_bench`, `cc_bench`, `zig_bench`) declares `` as the manual `dx-bench` test dx runs. Test attributes such as `size` (default `small`), `timeout`, `env`, `args`, `data`, `flaky` and `shard_count` go to ``, the other attributes go to the private `_bin`, and `tags` are merged with `dx-bench` and `manual`. +Every `_bench` macro (`go_bench`, `rust_bench`, `java_bench`, `kotlin_bench`, `python_bench`, `csharp_bench`, `fsharp_bench`, `d_bench`, `javascript_bench`, `typescript_bench`, `cc_bench`, `zig_bench`, `perl_bench`, `ruby_bench`, `dart_bench`, `php_bench`) declares `` as the manual `dx-bench` test dx runs. Test attributes such as `size` (default `small`), `timeout`, `env`, `args`, `data`, `flaky` and `shard_count` go to ``, the other attributes go to the private `_bin`, and `tags` are merged with `dx-bench` and `manual`. Go: `go_bench(name, srcs, embed = ...)` from `//go/rules:defs.bzl` runs the `Benchmark*` functions with `-test.bench=. -test.benchmem` and turns each `ns/op`, `B/op` and `allocs/op` column into one result named ` `. The Go gazelle extension adds `_bench` next to the `go_test` when a test file declares a `Benchmark*` function. @@ -49,3 +49,11 @@ D: `d_bench(name, srcs)` from `//d/rules:defs.bzl` is a `d_binary` that writes ` Rust: `rust_bench` from `//rust/rules:defs.bzl` runs a criterion or `#[bench]` target with `--output-format bencher` and writes `dx-bench.json` in ns/iter. Gazelle turns `[[bench]]` entries with `harness = false` into `rust_bench` targets. JavaScript and TypeScript: `javascript_bench(name, srcs, node_modules)` from `//javascript/rules:defs.bzl` and `typescript_bench` from `//typescript/rules:defs.bzl` run a [tinybench](https://github.com/tinylibs/tinybench) `Bench` that the first source (or `main`) exports as its default export. Each task becomes one row, in milliseconds. List `tinybench` in `package.json` so `node_modules` links it. `typescript_bench` also takes the `typescript_test` attributes such as `deps` and `tsconfig`. + +Perl: `perl_bench` from `//perl/rules:defs.bzl` is a `perl_test` whose script calls `DxBench::run_benchmarks([name => sub { ... }, ...])`. It times each sub with the core `Benchmark` module and reports seconds per call. + +Ruby: `ruby_bench` from `//ruby/rules:defs.bzl` takes `main`, a script that calls `DxBench.run { |x| x.report("name") { ... } }` with the benchmark-ips API. The pinned benchmark-ips gem comes with rules_dx, so no Gemfile entry is needed. Results are in seconds per call. + +Dart: `dart_bench` from `//dart/rules:defs.bzl` takes `main`, a program that calls `runBenchmarks([...])` from `package:dx_bench` with benchmark_harness `BenchmarkBase` objects. The pinned benchmark_harness comes from the `@dx_dart_bench_deps` hub of rules_dx. Results are in seconds per run. + +PHP: `php_bench` from `//php/rules:defs.bzl` takes `benches`, phpbench files with one class per file named like the file, with `#[Revs]` and `#[Iterations]` attributes. The pinned phpbench phar comes with rules_dx and runs on the pinned PHP. Results are in seconds per revolution. diff --git a/families/perl/perl/tests/bench/BUILD.bazel b/families/perl/perl/tests/bench/BUILD.bazel new file mode 100644 index 0000000000..6bb5e11ec9 --- /dev/null +++ b/families/perl/perl/tests/bench/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_dx//perl/rules:defs.bzl", "perl_bench", "perl_test") +load("@rules_dx//quality:fixtures.bzl", "real_source_target") + +package( + default_testonly = True, + default_visibility = ["//visibility:public"], +) + +perl_bench( + name = "sum_bench", + srcs = ["sum_bench.pl"], +) + +perl_test( + name = "bench_report_test", + srcs = ["check_test.pl"], + deps = ["@rules_dx//perl/rules:bench"], +) + +real_source_target( + name = "corpus_starlark", + starlark_srcs = ["BUILD.bazel"], + tags = ["corpus"], +) diff --git a/families/perl/perl/tests/bench/check_test.pl b/families/perl/perl/tests/bench/check_test.pl new file mode 100644 index 0000000000..7cf5da1757 --- /dev/null +++ b/families/perl/perl/tests/bench/check_test.pl @@ -0,0 +1,14 @@ +use strict; +use warnings; +use JSON::PP qw(decode_json); +use Test::More tests => 3; + +my $dir = $ENV{TEST_TMPDIR} // '.'; +$ENV{TEST_UNDECLARED_OUTPUTS_DIR} = $dir; +use DxBench qw(run_benchmarks); + +is( run_benchmarks( [ noop => sub { 1 } ], iterations => 2, rounds => 2 ), 1, 'one case' ); +open my $fh, '<', "$dir/dx-bench.json" or die $!; +my $doc = decode_json( do { local $/; <$fh> } ); +is( $doc->{schema}, 'dx-bench/v1', 'schema' ); +is( $doc->{results}[0]{name}, 'noop', 'name' ); diff --git a/families/perl/perl/tests/bench/sum_bench.pl b/families/perl/perl/tests/bench/sum_bench.pl new file mode 100644 index 0000000000..5e27549705 --- /dev/null +++ b/families/perl/perl/tests/bench/sum_bench.pl @@ -0,0 +1,18 @@ +use strict; +use warnings; +use DxBench qw(run_benchmarks); + +my @numbers = ( 1 .. 1000 ); + +run_benchmarks( + [ + sum_loop => sub { + my $total = 0; + $total += $_ for @numbers; + return $total; + }, + sum_join => sub { return eval join '+', @numbers }, + ], + iterations => 20, + rounds => 3, +); diff --git a/families/php/php/tests/bench/BUILD.bazel b/families/php/php/tests/bench/BUILD.bazel new file mode 100644 index 0000000000..5d3ef0ec5c --- /dev/null +++ b/families/php/php/tests/bench/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_dx//php/rules:defs.bzl", "php_bench") +load("@rules_dx//quality:fixtures.bzl", "real_source_target") + +package( + default_testonly = True, + default_visibility = ["//visibility:public"], +) + +php_bench( + name = "sum_bench", + benches = ["SumBench.php"], +) + +real_source_target( + name = "corpus_starlark", + starlark_srcs = ["BUILD.bazel"], + tags = ["corpus"], +) diff --git a/families/php/php/tests/bench/SumBench.php b/families/php/php/tests/bench/SumBench.php new file mode 100644 index 0000000000..1123d070ae --- /dev/null +++ b/families/php/php/tests/bench/SumBench.php @@ -0,0 +1,26 @@ + sub` pairs, timed with core `Benchmark`; seconds per call | | `perl_binary` | Runs one `.pl` script with `bazel run` | - Library `tags` reach the library; the generated upstream target is diff --git a/perl/rules/BUILD.bazel b/perl/rules/BUILD.bazel index e1121b1439..ff60100bd0 100644 --- a/perl/rules/BUILD.bazel +++ b/perl/rules/BUILD.bazel @@ -5,7 +5,7 @@ load("//libs/starlark:defs.bzl", "expect_equal") load("//libs/starlark:entry.bzl", "dx_entry_quality_srcs") load("//libs/starlark:wrapper.bzl", "dx_language_kwargs") load("//quality:fixtures.bzl", "real_source_target") -load(":defs.bzl", "PERL_DEFERRED_SKIP_TAGS", "PERL_QUALITY_TOOLS", "PERL_SOURCE_EXTS", "PERL_SOURCE_SPECS") +load(":defs.bzl", "PERL_DEFERRED_SKIP_TAGS", "PERL_QUALITY_TOOLS", "PERL_SOURCE_EXTS", "PERL_SOURCE_SPECS", "perl_library") load(":platforms.bzl", "PERL_PLATFORMS") package(default_visibility = ["//visibility:public"]) @@ -18,6 +18,13 @@ exports_files([ "wrapper_tests.bzl", ]) +perl_library( + name = "bench", + srcs = ["lib/DxBench.pm"], + tags = ["manual"], + visibility = ["//visibility:public"], +) + dx_wrapper_contract_tests( name = "wrapper_contract_tests", extra_checks = [ diff --git a/perl/rules/defs.bzl b/perl/rules/defs.bzl index ae761509e6..dad1c2c67f 100644 --- a/perl/rules/defs.bzl +++ b/perl/rules/defs.bzl @@ -3,6 +3,7 @@ load("@rules_perl//perl:defs.bzl", _PerlInfo = "PerlInfo", _perl_binary = "perl_binary", _perl_library = "perl_library", _perl_test = "perl_test") load("//libs/starlark:cells.bzl", "unsupported_constraint") load("//libs/starlark:entry.bzl", "dx_entry_quality_srcs", "dx_infer_main", "dx_main_candidates") +load("//libs/starlark:fuzz_bench.bzl", "merge_tags") load("//libs/starlark:wrapper.bzl", "DX_EXEC_PROVIDES", "dx_executable_forward_rule", "dx_language_kwargs", "dx_lcov_merger_attr", "dx_library_forward_rule", "dx_wrap", "dx_wrap_binary", "dx_wrap_test") load("//quality:sources.bzl", "QualitySourcesInfo", "QualityToolsInfo") @@ -111,3 +112,14 @@ def perl_test(name, *, srcs = None, main = None, visibility = None, **kwargs): kwargs = dx_language_kwargs(kwargs, PERL_DEFERRED_SKIP_TAGS, PERL_COMPATIBLE_WITH) upstream = dict(kwargs, main = main) dx_wrap_test(name, _perl_test, _perl_forward_test, srcs, visibility = visibility, upstream_kwargs = upstream, extra_forward_kwargs = {"deps": kwargs.get("deps", [])}, quality_srcs = dx_entry_quality_srcs(main, srcs), **kwargs) + +def perl_bench(name, srcs = None, main = None, deps = None, tags = None, **kwargs): + """A perl_test run by dx bench: scripts call DxBench::run_benchmarks, which times them with core Benchmark and writes dx-bench.json.""" + perl_test( + name = name, + srcs = srcs, + main = main, + deps = (deps or []) + [Label("//perl/rules:bench")], + tags = merge_tags(tags, "dx-bench", "manual"), + **kwargs + ) diff --git a/perl/rules/lib/DxBench.pm b/perl/rules/lib/DxBench.pm new file mode 100644 index 0000000000..bfd07c5db5 --- /dev/null +++ b/perl/rules/lib/DxBench.pm @@ -0,0 +1,57 @@ +package DxBench; + +use strict; +use warnings; +use Benchmark qw(timeit); +use Exporter qw(import); +use Time::HiRes qw(time); +use JSON::PP (); + +our @EXPORT_OK = qw(run_benchmarks); + +my $DEFAULT_ITERATIONS = 100; +my $DEFAULT_ROUNDS = 5; + +sub _measure { + my ( $code, $iterations, $rounds ) = @_; + my @times; + for ( 1 .. $rounds ) { + my $start = time; + timeit( $iterations, $code ); + push @times, ( time - $start ) / $iterations; + } + my $mean = 0; + $mean += $_ for @times; + $mean /= @times; + my $var = 0; + $var += ( $_ - $mean )**2 for @times; + my $stddev = @times > 1 ? sqrt( $var / ( @times - 1 ) ) : 0; + return ( $mean, $stddev ); +} + +sub run_benchmarks { + my ( $cases, %opts ) = @_; + my $iterations = $opts{iterations} // $DEFAULT_ITERATIONS; + my $rounds = $opts{rounds} // $DEFAULT_ROUNDS; + my @results; + while ( my ( $name, $code ) = splice @{$cases}, 0, 2 ) { + my ( $mean, $stddev ) = _measure( $code, $iterations, $rounds ); + push @results, + { + name => $name, + unit => 's', + mean => $mean + 0, + stddev => $stddev + 0, + iterations => $iterations * $rounds, + }; + } + my $dir = $ENV{TEST_UNDECLARED_OUTPUTS_DIR}; + die "TEST_UNDECLARED_OUTPUTS_DIR is not set\n" unless defined $dir; + open my $fh, '>', "$dir/dx-bench.json" or die "dx-bench.json: $!\n"; + print {$fh} JSON::PP->new->canonical->encode( + { schema => 'dx-bench/v1', results => \@results } ); + close $fh or die "dx-bench.json: $!\n"; + return scalar @results; +} + +1; diff --git a/php/README.md b/php/README.md index 922e53b068..15cec5054d 100644 --- a/php/README.md +++ b/php/README.md @@ -43,6 +43,7 @@ php_binary( | `php_library` | Sources whose classes other targets load; every file is checked with `php -l` at build time | | `php_binary` | Runs `main` with `bazel run` | | `php_test` | Runs PHPUnit test files; a failure or an empty suite fails the test | +| `php_bench` | `php_test` for `dx bench`. `benches` lists phpbench files, one class per file named like the file; the pinned phpbench phar runs them. `php_fuzz_test` is not offered: php-fuzzer is not a pinned phar | - A syntax error fails the build, not the first run. - PHP runs with `-n`, so no `php.ini` from the host is read. Set ini values diff --git a/php/rules/BUILD.bazel b/php/rules/BUILD.bazel index afdd407612..cd7c584052 100644 --- a/php/rules/BUILD.bazel +++ b/php/rules/BUILD.bazel @@ -8,6 +8,7 @@ load(":php.bzl", "PHP_SOURCE_EXTS", "autoloader_suffix", "coverage_launcher_scri package(default_visibility = ["//visibility:public"]) exports_files([ + "BenchTest.php", "defs.bzl", "php.bzl", "providers.bzl", diff --git a/php/rules/BenchTest.php b/php/rules/BenchTest.php new file mode 100644 index 0000000000..934fd2492c --- /dev/null +++ b/php/rules/BenchTest.php @@ -0,0 +1,60 @@ +&1', $lines, $status); + self::assertSame(0, $status, implode("\n", $lines)); + + $suite = simplexml_load_file($dump); + self::assertNotFalse($suite); + $results = []; + foreach ($suite->xpath('//subject') ?: [] as $subject) { + foreach ($subject->variant as $variant) { + $revs = (int) $variant['revs']; + $times = []; + foreach ($variant->iteration as $iteration) { + $times[] = (float) $iteration['time-net'] / $revs / 1e6; + } + if ($times === []) { + continue; + } + $mean = array_sum($times) / count($times); + $variance = 0.0; + foreach ($times as $time) { + $variance += ($time - $mean) ** 2; + } + $results[] = [ + 'name' => (string) $subject['name'], + 'unit' => 's', + 'mean' => $mean, + 'stddev' => sqrt($variance / count($times)), + 'iterations' => count($times) * $revs, + ]; + } + } + self::assertNotSame([], $results, 'phpbench reported no results'); + file_put_contents($out . '/dx-bench.json', json_encode(['schema' => 'dx-bench/v1', 'results' => $results])); + } +} diff --git a/php/rules/defs.bzl b/php/rules/defs.bzl index cf5a5be50a..051edd3e0c 100644 --- a/php/rules/defs.bzl +++ b/php/rules/defs.bzl @@ -2,6 +2,7 @@ load("//libs/starlark:cells.bzl", "cells_compatible_with") load("//libs/starlark:entry.bzl", "dx_entry_quality_srcs", "dx_infer_main", "dx_main_candidates") +load("//libs/starlark:fuzz_bench.bzl", "merge_tags") load("//libs/starlark:wrapper.bzl", "DX_EXEC_PROVIDES", "dx_executable_forward_rule", "dx_language_kwargs", "dx_lcov_merger_attr", "dx_library_forward_rule", "dx_wrap", "dx_wrap_binary", "dx_wrap_test") load("//php/toolchain:pins.bzl", "PHP_CELLS") load("//quality:sources.bzl", "QualitySourcesInfo") @@ -66,3 +67,19 @@ def php_test(name, *, srcs = None, visibility = None, **kwargs): """Runs PHPUnit test files on the pinned PHP and PHPUnit. srcs are the test files and are linted.""" kwargs = dx_language_kwargs(kwargs, PHP_DEFERRED_SKIP_TAGS, PHP_COMPATIBLE_WITH) dx_wrap_test(name, _php_test, _php_forward_test, srcs, visibility = visibility, quality_srcs = dx_entry_quality_srcs(None, srcs), **kwargs) + +def php_bench(name, benches, data = None, env = None, tags = None, **kwargs): + """A php_test run by dx bench: phpbench runs the `benches` files, one class per file named like the file, and the result is written as dx-bench.json.""" + phar = Label("@dx_phpbench//:phpbench.phar") + php_test( + name = name, + srcs = [Label("//php/rules:BenchTest.php")], + data = (data or []) + benches + [phar], + env = dict( + env or {}, + DX_BENCH_FILES = " ".join(["$(rlocationpath %s)" % bench for bench in benches]), + DX_PHPBENCH = "$(rlocationpath %s)" % phar, + ), + tags = merge_tags(tags, "dx-bench", "manual"), + **kwargs + ) diff --git a/php/toolchain/metadata_tests.bzl b/php/toolchain/metadata_tests.bzl index 9952825196..fb6d1b81be 100644 --- a/php/toolchain/metadata_tests.bzl +++ b/php/toolchain/metadata_tests.bzl @@ -6,10 +6,10 @@ load("//libs/starlark:defs.bzl", "expect_contains", "expect_equal", "expect_true load("//php/rules:defs.bzl", "PHP_COMPATIBLE_WITH") load("//php/rules:php.bzl", "lit", "rf", "runfiles_path", "write_launcher") load(":php.bzl", "TOOLCHAIN_TYPE") -load(":pins.bzl", "COMPOSER", "PCOV", "PHPCS", "PHPSTAN", "PHPSTAN_STRICT_RULES", "PHPUNIT", "PHP_CELLS", "PHP_HEADERS", "PHP_VERSION", "STATIC_PHP", "UNUSED_PUBLIC", "WEBMOZART_ASSERT") +load(":pins.bzl", "COMPOSER", "PCOV", "PHPBENCH", "PHPCS", "PHPSTAN", "PHPSTAN_STRICT_RULES", "PHPUNIT", "PHP_CELLS", "PHP_HEADERS", "PHP_VERSION", "STATIC_PHP", "UNUSED_PUBLIC", "WEBMOZART_ASSERT") load(":repos.bzl", "VERSION_FILE", "hub_build", "runtime_build") -PINS = [COMPOSER, PCOV, PHPCS, PHPSTAN, PHPSTAN_STRICT_RULES, PHPUNIT, UNUSED_PUBLIC, WEBMOZART_ASSERT] +PINS = [COMPOSER, PCOV, PHPBENCH, PHPCS, PHPSTAN, PHPSTAN_STRICT_RULES, PHPUNIT, UNUSED_PUBLIC, WEBMOZART_ASSERT] def _checks(): hub = hub_build(PHP_CELLS.keys()) diff --git a/php/toolchain/pins.bzl b/php/toolchain/pins.bzl index c1c5394b1c..9806edc4cc 100644 --- a/php/toolchain/pins.bzl +++ b/php/toolchain/pins.bzl @@ -96,3 +96,9 @@ PHPSTAN = struct( sha256 = "aa5cc7543f0de0ae7997deb24f654a3b2454b6ba4ba2c4552957a175886b60b1", urls = ["https://github.com/phpstan/phpstan/releases/download/2.3.3/phpstan.phar"], ) + +PHPBENCH = struct( + version = "1.7.0", + sha256 = "68563bfbef7eb3af8ebf6b62ce80c9646ab61c9b1ce0011b57fb947f73ac3b37", + urls = ["https://github.com/phpbench/phpbench/releases/download/1.7.0/phpbench.phar"], +) diff --git a/php/toolchain/repos.bzl b/php/toolchain/repos.bzl index 0a61e47bb6..b43fc3ee1c 100644 --- a/php/toolchain/repos.bzl +++ b/php/toolchain/repos.bzl @@ -1,7 +1,7 @@ """Hermetic PHP, Composer and PHPUnit acquisition and toolchain registration.""" load("//libs/starlark:repo.bzl", "hub_repo", "pinned_archive") -load(":pins.bzl", "COMPOSER", "PCOV", "PCRE2_HEADERS", "PHPCS", "PHPSTAN", "PHPSTAN_STRICT_RULES", "PHPUNIT", "PHP_CELLS", "PHP_HEADERS", "PHP_VERSION", "UNUSED_PUBLIC", "WEBMOZART_ASSERT") +load(":pins.bzl", "COMPOSER", "PCOV", "PCRE2_HEADERS", "PHPBENCH", "PHPCS", "PHPSTAN", "PHPSTAN_STRICT_RULES", "PHPUNIT", "PHP_CELLS", "PHP_HEADERS", "PHP_VERSION", "UNUSED_PUBLIC", "WEBMOZART_ASSERT") HUB_NAME = "dx_php_toolchains" @@ -134,6 +134,7 @@ def _php_impl(mctx): for name, file, pin in [ ("dx_composer", "composer.phar", COMPOSER), ("dx_phpunit", "phpunit.phar", PHPUNIT), + ("dx_phpbench", "phpbench.phar", PHPBENCH), ("dx_phpcs", "phpcs.phar", PHPCS), ("dx_phpstan", "phpstan.phar", PHPSTAN), ]: diff --git a/ruby/README.md b/ruby/README.md index fd05795048..6e556f9ffa 100644 --- a/ruby/README.md +++ b/ruby/README.md @@ -47,6 +47,7 @@ ruby_test( | `ruby_library` | `rb_library` | | `ruby_binary` | `rb_binary`. `srcs` is optional | | `ruby_test` | `rb_test`. Set `main` to a runner such as an RSpec bin | +| `ruby_bench` | `ruby_test` for `dx bench`. `main` calls `DxBench.run` with a benchmark-ips block and reports seconds per call. `ruby_fuzz_test` is not offered yet: ruzzy builds a native extension against libFuzzer | ## Gems diff --git a/ruby/coverage/pins.bzl b/ruby/coverage/pins.bzl index 8e8b2670b0..93ad221c39 100644 --- a/ruby/coverage/pins.bzl +++ b/ruby/coverage/pins.bzl @@ -15,6 +15,13 @@ RUBY_COVERAGE_GEMS = [ RUBY_COVERAGE_REPOSITORY = "dx_ruby_coverage_gems" +# benchmark-ips for ruby_bench; pure Ruby, no runtime dependencies. +RUBY_BENCH_GEMS = [ + struct(name = "benchmark-ips", version = "2.14.0", sha256 = "b72bc8a65d525d5906f8cd94270dccf73452ee3257a32b89fbd6684d3e8a9b1d", entry = "benchmark/ips.rb"), +] + +RUBY_BENCH_REPOSITORY = "dx_ruby_bench_gems" + def gem_url(gem): return "https://rubygems.org/downloads/" + gem.name + "-" + gem.version + ".gem" diff --git a/ruby/coverage/repos.bzl b/ruby/coverage/repos.bzl index 9545232554..638d59f42f 100644 --- a/ruby/coverage/repos.bzl +++ b/ruby/coverage/repos.bzl @@ -1,6 +1,6 @@ """Pinned SimpleCov gems that Ruby tests load under `bazel coverage` (#3954).""" -load(":pins.bzl", "RUBY_COVERAGE_GEMS", "RUBY_COVERAGE_REPOSITORY", "gem_dir", "gem_url") +load(":pins.bzl", "RUBY_BENCH_GEMS", "RUBY_BENCH_REPOSITORY", "RUBY_COVERAGE_GEMS", "RUBY_COVERAGE_REPOSITORY", "gem_dir", "gem_url") def _build_file(gems): dirs = [gem_dir(gem) for gem in gems] @@ -14,7 +14,7 @@ def _build_file(gems): lines += [ " \"simplecov-html-*/public/**\",", " \"simplecov-html-*/views/**\",", - " ]),", + " ], allow_empty = True),", " visibility = [\"//visibility:public\"],", ")", "", @@ -31,8 +31,8 @@ def _build_file(gems): ] return "\n".join(lines) -def _ruby_coverage_gems_impl(ctx): - for gem in RUBY_COVERAGE_GEMS: +def _download_gems(ctx, gems): + for gem in gems: url = gem_url(gem) tmp = "_gem_" + gem.name ctx.download_and_extract( @@ -44,26 +44,37 @@ def _ruby_coverage_gems_impl(ctx): ) data = tmp + "/data.tar.gz" if not ctx.path(data).exists: - fail("ruby coverage: '" + url + "' holds no data.tar.gz") + fail("ruby gems: '" + url + "' holds no data.tar.gz") ctx.extract(archive = data, output = gem_dir(gem)) entry = gem_dir(gem) + "/lib/" + gem.entry if not ctx.path(entry).exists: - fail("ruby coverage: '" + url + "' holds no lib/" + gem.entry) + fail("ruby gems: '" + url + "' holds no lib/" + gem.entry) ctx.delete(tmp) - ctx.file("BUILD.bazel", _build_file(RUBY_COVERAGE_GEMS)) + ctx.file("BUILD.bazel", _build_file(gems)) return ctx.repo_metadata(reproducible = True) +def _ruby_coverage_gems_impl(ctx): + return _download_gems(ctx, RUBY_COVERAGE_GEMS) + +def _ruby_bench_gems_impl(ctx): + return _download_gems(ctx, RUBY_BENCH_GEMS) + _ruby_coverage_gems = repository_rule( implementation = _ruby_coverage_gems_impl, ) +_ruby_bench_gems = repository_rule( + implementation = _ruby_bench_gems_impl, +) + def _ruby_coverage_impl(module_ctx): if len(module_ctx.modules) == 0: - fail("ruby coverage: the extension is only reachable from a module graph") + fail("ruby gems: the extension is only reachable from a module graph") _ruby_coverage_gems(name = RUBY_COVERAGE_REPOSITORY) + _ruby_bench_gems(name = RUBY_BENCH_REPOSITORY) return module_ctx.extension_metadata(reproducible = True) ruby_coverage = module_extension( - doc = "Downloads the pinned, checksum-verified SimpleCov gems as @dx_ruby_coverage_gems.", + doc = "Downloads the pinned, checksum-verified SimpleCov gems as @dx_ruby_coverage_gems and benchmark-ips as @dx_ruby_bench_gems.", implementation = _ruby_coverage_impl, ) diff --git a/ruby/rules/BUILD.bazel b/ruby/rules/BUILD.bazel index 4d77ed6372..83aeb002a0 100644 --- a/ruby/rules/BUILD.bazel +++ b/ruby/rules/BUILD.bazel @@ -6,6 +6,7 @@ load(":wrapper_tests.bzl", "ruby_wrapper_contract_tests") package(default_visibility = ["//visibility:public"]) exports_files([ + "bench_main.rb", "defs.bzl", "toolchain_tests.bzl", "wrapper_tests.bzl", diff --git a/ruby/rules/bench_main.rb b/ruby/rules/bench_main.rb new file mode 100644 index 0000000000..7d9e589323 --- /dev/null +++ b/ruby/rules/bench_main.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require "json" + +ips_entry = ENV.fetch("DX_BENCH_IPS") +$LOAD_PATH.unshift(File.dirname(File.dirname(File.join(ENV.fetch("TEST_SRCDIR"), ips_entry)))) +require "benchmark/ips" + +# Runs a benchmark-ips block and writes dx-bench.json for `dx bench`. +module DxBench + def self.run(time: 0.5, warmup: 0.2, &block) + report = Benchmark.ips(time, warmup, true, &block) + results = report.entries.map do |entry| + ips = entry.stats.central_tendency + { + name: entry.label.to_s, + unit: "s", + mean: 1.0 / ips, + stddev: entry.stats.error / (ips * ips), + iterations: entry.iterations + } + end + File.write(File.join(ENV.fetch("TEST_UNDECLARED_OUTPUTS_DIR"), "dx-bench.json"), + JSON.generate(schema: "dx-bench/v1", results: results)) + end +end + +load File.join(ENV.fetch("TEST_SRCDIR"), ENV.fetch("DX_BENCH_SCRIPT")) diff --git a/ruby/rules/defs.bzl b/ruby/rules/defs.bzl index d7e9f06240..8a1ebc9b02 100644 --- a/ruby/rules/defs.bzl +++ b/ruby/rules/defs.bzl @@ -3,6 +3,7 @@ load("@rules_ruby//ruby:defs.bzl", _rb_binary = "rb_binary", _rb_library = "rb_library", _rb_test = "rb_test") load("@rules_ruby//ruby/private:providers.bzl", "BundlerInfo", "RubyFilesInfo") load("//libs/starlark:entry.bzl", "dx_entry_quality_srcs") +load("//libs/starlark:fuzz_bench.bzl", "merge_tags") load("//libs/starlark:wrapper.bzl", "DX_EXEC_PROVIDES", "dx_executable_forward_rule", "dx_lcov_merger_attr", "dx_library_forward_rule", "dx_wrap", "dx_wrap_binary", "dx_wrap_test") load("//quality:sources.bzl", "QualitySourcesInfo") load("//ruby/coverage:pins.bzl", "RUBY_COVERAGE_GEMS", "gem_dir") @@ -107,3 +108,20 @@ def ruby_test(name, *, srcs = None, main = None, visibility = None, **kwargs): """Minimal wrapper over rb_test. Set main to a runner such as an RSpec bin; main is linted unless it is an external label.""" upstream = dict(kwargs, main = main) if main != None else kwargs dx_wrap_test(name, _rb_test, _ruby_forward_test, srcs, visibility = visibility, upstream_kwargs = upstream, quality_srcs = dx_entry_quality_srcs(main, srcs), **kwargs) + +def ruby_bench(name, main, srcs = None, data = None, env = None, tags = None, **kwargs): + """A ruby_test run by dx bench: `main` calls DxBench.run with a benchmark-ips block, which writes dx-bench.json.""" + ips = Label("@dx_ruby_bench_gems//:benchmark-ips-2.14.0/lib/benchmark/ips.rb") + ruby_test( + name = name, + srcs = (srcs or []) + [main, Label("//ruby/rules:bench_main.rb")], + main = Label("//ruby/rules:bench_main.rb"), + data = (data or []) + [ips, Label("@dx_ruby_bench_gems//:lib")], + env = dict( + env or {}, + DX_BENCH_IPS = "$(rlocationpath %s)" % ips, + DX_BENCH_SCRIPT = "$(rlocationpath %s)" % main, + ), + tags = merge_tags(tags, "dx-bench", "manual"), + **kwargs + )