From ef623e2d99510ebc8d26643bf750c17cbc6e4389 Mon Sep 17 00:00:00 2001 From: YuYu1015 Date: Fri, 18 Sep 2026 03:55:14 +0800 Subject: [PATCH 1/4] build: keep generated code out of test coverage --- build.yaml | 9 +++ l10n.yaml | 4 + lib/core/geo/town.g.dart | 2 + .../bug_tracker/domain/bug_thread.g.dart | 2 + .../changelog/domain/release_note.g.dart | 2 + .../disaster_map/domain/aed_detail.g.dart | 2 + .../domain/restroom_detail.g.dart | 2 + .../disaster_map/domain/shelter_detail.g.dart | 2 + .../domain/earthquake_report.g.dart | 2 + lib/features/earthquake/domain/eew.g.dart | 2 + .../domain/partial_earthquake_report.g.dart | 2 + lib/features/earthquake/domain/rts.g.dart | 2 + .../domain/release_highlight.g.dart | 2 + .../typhoon/domain/storm_circle.g.dart | 2 + .../typhoon/domain/typhoon_cyclone.g.dart | 2 + .../typhoon/domain/typhoon_potential.g.dart | 2 + .../typhoon/domain/typhoon_track.g.dart | 2 + .../typhoon/domain/typhoon_warning.g.dart | 2 + .../weather/domain/weather_forecast.g.dart | 2 + .../weather/domain/weather_realtime.g.dart | 2 + .../weather/domain/weather_station.g.dart | 2 + lib/l10n/gen/app_localizations.dart | 1 + lib/l10n/gen/app_localizations_en.dart | 2 + lib/l10n/gen/app_localizations_fil.dart | 2 + lib/l10n/gen/app_localizations_id.dart | 2 + lib/l10n/gen/app_localizations_ja.dart | 2 + lib/l10n/gen/app_localizations_ko.dart | 2 + lib/l10n/gen/app_localizations_th.dart | 2 + lib/l10n/gen/app_localizations_vi.dart | 2 + lib/l10n/gen/app_localizations_yue.dart | 2 + lib/l10n/gen/app_localizations_zh.dart | 2 + test/tool/generated_code_coverage_test.dart | 77 +++++++++++++++++++ 32 files changed, 147 insertions(+) create mode 100644 test/tool/generated_code_coverage_test.dart diff --git a/build.yaml b/build.yaml index f5b3e0a22..0e36ac781 100644 --- a/build.yaml +++ b/build.yaml @@ -7,3 +7,12 @@ targets: # Nested models serialize via their own toJson (so round-trips and # local caching produce plain JSON, not embedded objects). explicit_to_json: true + # Generated code is not code anyone tests into shape. `flutter test` + # leaves out every file that says `// coverage:ignore-file`, so CI's + # report and the editor's agree without a list of suffixes kept in step. + # freezed writes the comment into `.freezed.dart` on its own; this puts + # it into every `.g.dart`. test/tool/generated_code_coverage_test.dart + # fails when a generated file comes out without it. + source_gen:combining_builder: + options: + preamble: "// coverage:ignore-file" diff --git a/l10n.yaml b/l10n.yaml index 3b86e8805..05cbc3075 100644 --- a/l10n.yaml +++ b/l10n.yaml @@ -13,3 +13,7 @@ output-dir: lib/l10n/gen output-localization-file: app_localizations.dart output-class: AppLocalizations nullable-getter: false +# Keeps the generated localizations out of test coverage: thousands of getters +# that a string test either reaches or does not would move the total whenever a +# translation is added. `flutter test` honours the comment (see build.yaml). +header: "// coverage:ignore-file" diff --git a/lib/core/geo/town.g.dart b/lib/core/geo/town.g.dart index 92c569a42..50da18229 100644 --- a/lib/core/geo/town.g.dart +++ b/lib/core/geo/town.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'town.dart'; // ************************************************************************** diff --git a/lib/features/bug_tracker/domain/bug_thread.g.dart b/lib/features/bug_tracker/domain/bug_thread.g.dart index 7ce5eccb5..798fee1f1 100644 --- a/lib/features/bug_tracker/domain/bug_thread.g.dart +++ b/lib/features/bug_tracker/domain/bug_thread.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'bug_thread.dart'; // ************************************************************************** diff --git a/lib/features/changelog/domain/release_note.g.dart b/lib/features/changelog/domain/release_note.g.dart index 68af439bc..20327b572 100644 --- a/lib/features/changelog/domain/release_note.g.dart +++ b/lib/features/changelog/domain/release_note.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'release_note.dart'; // ************************************************************************** diff --git a/lib/features/disaster_map/domain/aed_detail.g.dart b/lib/features/disaster_map/domain/aed_detail.g.dart index 780f98ff8..00976bb50 100644 --- a/lib/features/disaster_map/domain/aed_detail.g.dart +++ b/lib/features/disaster_map/domain/aed_detail.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'aed_detail.dart'; // ************************************************************************** diff --git a/lib/features/disaster_map/domain/restroom_detail.g.dart b/lib/features/disaster_map/domain/restroom_detail.g.dart index a08e3ea88..17acad0f4 100644 --- a/lib/features/disaster_map/domain/restroom_detail.g.dart +++ b/lib/features/disaster_map/domain/restroom_detail.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'restroom_detail.dart'; // ************************************************************************** diff --git a/lib/features/disaster_map/domain/shelter_detail.g.dart b/lib/features/disaster_map/domain/shelter_detail.g.dart index 0ea56a666..9465950a4 100644 --- a/lib/features/disaster_map/domain/shelter_detail.g.dart +++ b/lib/features/disaster_map/domain/shelter_detail.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'shelter_detail.dart'; // ************************************************************************** diff --git a/lib/features/earthquake/domain/earthquake_report.g.dart b/lib/features/earthquake/domain/earthquake_report.g.dart index c7e62994d..0986d18b2 100644 --- a/lib/features/earthquake/domain/earthquake_report.g.dart +++ b/lib/features/earthquake/domain/earthquake_report.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'earthquake_report.dart'; // ************************************************************************** diff --git a/lib/features/earthquake/domain/eew.g.dart b/lib/features/earthquake/domain/eew.g.dart index 57b424ccf..26ce13c33 100644 --- a/lib/features/earthquake/domain/eew.g.dart +++ b/lib/features/earthquake/domain/eew.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'eew.dart'; // ************************************************************************** diff --git a/lib/features/earthquake/domain/partial_earthquake_report.g.dart b/lib/features/earthquake/domain/partial_earthquake_report.g.dart index 12a9a7e4d..6fa36632f 100644 --- a/lib/features/earthquake/domain/partial_earthquake_report.g.dart +++ b/lib/features/earthquake/domain/partial_earthquake_report.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'partial_earthquake_report.dart'; // ************************************************************************** diff --git a/lib/features/earthquake/domain/rts.g.dart b/lib/features/earthquake/domain/rts.g.dart index d926e3ccc..45b2683ea 100644 --- a/lib/features/earthquake/domain/rts.g.dart +++ b/lib/features/earthquake/domain/rts.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'rts.dart'; // ************************************************************************** diff --git a/lib/features/release_highlights/domain/release_highlight.g.dart b/lib/features/release_highlights/domain/release_highlight.g.dart index b0a3d5191..4bd61e49e 100644 --- a/lib/features/release_highlights/domain/release_highlight.g.dart +++ b/lib/features/release_highlights/domain/release_highlight.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'release_highlight.dart'; // ************************************************************************** diff --git a/lib/features/typhoon/domain/storm_circle.g.dart b/lib/features/typhoon/domain/storm_circle.g.dart index 20fe0f24d..11d3c18ae 100644 --- a/lib/features/typhoon/domain/storm_circle.g.dart +++ b/lib/features/typhoon/domain/storm_circle.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'storm_circle.dart'; // ************************************************************************** diff --git a/lib/features/typhoon/domain/typhoon_cyclone.g.dart b/lib/features/typhoon/domain/typhoon_cyclone.g.dart index dad3b1eff..e9643a742 100644 --- a/lib/features/typhoon/domain/typhoon_cyclone.g.dart +++ b/lib/features/typhoon/domain/typhoon_cyclone.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'typhoon_cyclone.dart'; // ************************************************************************** diff --git a/lib/features/typhoon/domain/typhoon_potential.g.dart b/lib/features/typhoon/domain/typhoon_potential.g.dart index 37c24611c..6323c46c0 100644 --- a/lib/features/typhoon/domain/typhoon_potential.g.dart +++ b/lib/features/typhoon/domain/typhoon_potential.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'typhoon_potential.dart'; // ************************************************************************** diff --git a/lib/features/typhoon/domain/typhoon_track.g.dart b/lib/features/typhoon/domain/typhoon_track.g.dart index 5d4efaa5c..e99804558 100644 --- a/lib/features/typhoon/domain/typhoon_track.g.dart +++ b/lib/features/typhoon/domain/typhoon_track.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'typhoon_track.dart'; // ************************************************************************** diff --git a/lib/features/typhoon/domain/typhoon_warning.g.dart b/lib/features/typhoon/domain/typhoon_warning.g.dart index 383f7062f..f685ede4a 100644 --- a/lib/features/typhoon/domain/typhoon_warning.g.dart +++ b/lib/features/typhoon/domain/typhoon_warning.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'typhoon_warning.dart'; // ************************************************************************** diff --git a/lib/features/weather/domain/weather_forecast.g.dart b/lib/features/weather/domain/weather_forecast.g.dart index 9b184e882..264b03ddf 100644 --- a/lib/features/weather/domain/weather_forecast.g.dart +++ b/lib/features/weather/domain/weather_forecast.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'weather_forecast.dart'; // ************************************************************************** diff --git a/lib/features/weather/domain/weather_realtime.g.dart b/lib/features/weather/domain/weather_realtime.g.dart index d056755ee..b399691f5 100644 --- a/lib/features/weather/domain/weather_realtime.g.dart +++ b/lib/features/weather/domain/weather_realtime.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'weather_realtime.dart'; // ************************************************************************** diff --git a/lib/features/weather/domain/weather_station.g.dart b/lib/features/weather/domain/weather_station.g.dart index 4c475c7b3..1ef2c63a1 100644 --- a/lib/features/weather/domain/weather_station.g.dart +++ b/lib/features/weather/domain/weather_station.g.dart @@ -1,5 +1,7 @@ // GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file + part of 'weather_station.dart'; // ************************************************************************** diff --git a/lib/l10n/gen/app_localizations.dart b/lib/l10n/gen/app_localizations.dart index 4bddd0ac8..cbd4e6d8b 100644 --- a/lib/l10n/gen/app_localizations.dart +++ b/lib/l10n/gen/app_localizations.dart @@ -1,3 +1,4 @@ +// coverage:ignore-file import 'dart:async'; import 'package:flutter/foundation.dart'; diff --git a/lib/l10n/gen/app_localizations_en.dart b/lib/l10n/gen/app_localizations_en.dart index c3d0a0cd8..5c4c09634 100644 --- a/lib/l10n/gen/app_localizations_en.dart +++ b/lib/l10n/gen/app_localizations_en.dart @@ -1,3 +1,5 @@ +// coverage:ignore-file + // ignore: unused_import import 'package:intl/intl.dart' as intl; diff --git a/lib/l10n/gen/app_localizations_fil.dart b/lib/l10n/gen/app_localizations_fil.dart index ae4e1ced7..4ec39a8b7 100644 --- a/lib/l10n/gen/app_localizations_fil.dart +++ b/lib/l10n/gen/app_localizations_fil.dart @@ -1,3 +1,5 @@ +// coverage:ignore-file + // ignore: unused_import import 'package:intl/intl.dart' as intl; diff --git a/lib/l10n/gen/app_localizations_id.dart b/lib/l10n/gen/app_localizations_id.dart index 03de16cbf..88f76f260 100644 --- a/lib/l10n/gen/app_localizations_id.dart +++ b/lib/l10n/gen/app_localizations_id.dart @@ -1,3 +1,5 @@ +// coverage:ignore-file + // ignore: unused_import import 'package:intl/intl.dart' as intl; diff --git a/lib/l10n/gen/app_localizations_ja.dart b/lib/l10n/gen/app_localizations_ja.dart index ab50d6433..0a5d20c47 100644 --- a/lib/l10n/gen/app_localizations_ja.dart +++ b/lib/l10n/gen/app_localizations_ja.dart @@ -1,3 +1,5 @@ +// coverage:ignore-file + // ignore: unused_import import 'package:intl/intl.dart' as intl; diff --git a/lib/l10n/gen/app_localizations_ko.dart b/lib/l10n/gen/app_localizations_ko.dart index f17fff947..10bdb53b1 100644 --- a/lib/l10n/gen/app_localizations_ko.dart +++ b/lib/l10n/gen/app_localizations_ko.dart @@ -1,3 +1,5 @@ +// coverage:ignore-file + // ignore: unused_import import 'package:intl/intl.dart' as intl; diff --git a/lib/l10n/gen/app_localizations_th.dart b/lib/l10n/gen/app_localizations_th.dart index 0da0c91b7..3309032ce 100644 --- a/lib/l10n/gen/app_localizations_th.dart +++ b/lib/l10n/gen/app_localizations_th.dart @@ -1,3 +1,5 @@ +// coverage:ignore-file + // ignore: unused_import import 'package:intl/intl.dart' as intl; diff --git a/lib/l10n/gen/app_localizations_vi.dart b/lib/l10n/gen/app_localizations_vi.dart index 5f7c2ceed..19393318d 100644 --- a/lib/l10n/gen/app_localizations_vi.dart +++ b/lib/l10n/gen/app_localizations_vi.dart @@ -1,3 +1,5 @@ +// coverage:ignore-file + // ignore: unused_import import 'package:intl/intl.dart' as intl; diff --git a/lib/l10n/gen/app_localizations_yue.dart b/lib/l10n/gen/app_localizations_yue.dart index 4bdca7328..0ca2786d4 100644 --- a/lib/l10n/gen/app_localizations_yue.dart +++ b/lib/l10n/gen/app_localizations_yue.dart @@ -1,3 +1,5 @@ +// coverage:ignore-file + // ignore: unused_import import 'package:intl/intl.dart' as intl; diff --git a/lib/l10n/gen/app_localizations_zh.dart b/lib/l10n/gen/app_localizations_zh.dart index eadfe6ae8..fd620a528 100644 --- a/lib/l10n/gen/app_localizations_zh.dart +++ b/lib/l10n/gen/app_localizations_zh.dart @@ -1,3 +1,5 @@ +// coverage:ignore-file + // ignore: unused_import import 'package:intl/intl.dart' as intl; diff --git a/test/tool/generated_code_coverage_test.dart b/test/tool/generated_code_coverage_test.dart new file mode 100644 index 000000000..11538c11d --- /dev/null +++ b/test/tool/generated_code_coverage_test.dart @@ -0,0 +1,77 @@ +/// Generated code stays out of test coverage because every generated file says +/// so itself. +/// +/// `flutter test --coverage` leaves out each file that carries +/// `// coverage:ignore-file`, and that comment is the only exclusion there is: +/// CI's report and the editor's coverage view both show what the test runner +/// collected. A generated file without it breaks nothing else. It quietly adds +/// thousands of lines nobody writes tests for, and moves the total whenever a +/// translation or a model changes. +/// +/// freezed writes the comment on its own. build.yaml has source_gen add it to +/// every `.g.dart`, and l10n.yaml has gen-l10n add it to the localizations. The +/// files tool/gen/ and tool/release/build_info.sh write hold constants only, +/// which have no lines to count, so they are not held to it. +library; + +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; + +const _marker = '// coverage:ignore-file'; + +/// The first line of everything build_runner writes, `.g.dart` and +/// `.freezed.dart` alike. +const _buildRunnerHeader = '// GENERATED CODE - DO NOT MODIFY BY HAND'; + +void main() { + final root = Directory.current.path; + final sources = Directory('$root/lib') + .listSync(recursive: true) + .whereType() + .where((file) => file.path.endsWith('.dart')) + .toList(); + + final fromBuildRunner = [ + for (final file in sources) + if (file.readAsLinesSync().firstOrNull == _buildRunnerHeader) file, + ]; + final fromGenL10n = [ + for (final file in sources) + if (file.path.startsWith('$root/lib/l10n/gen/')) file, + ]; + + test('there is generated code to check', () { + // A move or a renamed header that emptied these lists would let every + // test below pass without looking at anything. + expect( + fromBuildRunner.where((f) => f.path.endsWith('.g.dart')), + isNotEmpty, + ); + expect( + fromBuildRunner.where((f) => f.path.endsWith('.freezed.dart')), + isNotEmpty, + ); + expect(fromGenL10n, isNotEmpty); + }); + + for (final (generator, files, regenerate) in [ + ('build_runner', fromBuildRunner, 'tool/dev/codegen.sh'), + ('gen-l10n', fromGenL10n, 'tool/dev/l10n.sh'), + ]) { + test('everything $generator writes is left out of coverage', () { + final unmarked = [ + for (final file in files) + if (!file.readAsLinesSync().contains(_marker)) + file.path.substring(root.length + 1), + ]; + expect( + unmarked, + isEmpty, + reason: + 'missing `$_marker`. Regenerate with $regenerate; the comment ' + 'comes from build.yaml and l10n.yaml.', + ); + }); + } +} From 3acc83b6bb87efe6eceb3187d6a2cda6772fde81 Mon Sep 17 00:00:00 2001 From: YuYu1015 Date: Fri, 18 Sep 2026 03:55:14 +0800 Subject: [PATCH 2/4] chore(tool): run the test suite with line coverage for lib/ --- AGENTS.md | 30 ++++++ test/tool/coverage_script_test.dart | 150 ++++++++++++++++++++++++++++ tool/check.sh | 19 ++-- tool/check/tooling.sh | 1 + tool/dev/_lib.sh | 13 +++ tool/dev/coverage.sh | 97 ++++++++++++++++++ 6 files changed, 299 insertions(+), 11 deletions(-) create mode 100644 test/tool/coverage_script_test.dart create mode 100755 tool/dev/coverage.sh diff --git a/AGENTS.md b/AGENTS.md index d7a18b4ae..ee1ee4740 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,6 +47,7 @@ tool/dev/analyze.sh |---|---| | Start the app | `tool/run.sh` (see [Running](#running)) | | Run the tests | `tool/dev/test.sh` | +| Test coverage, into `coverage/` | `tool/dev/coverage.sh` (see [Coverage](#coverage)) | | Format + analyze | `tool/dev/analyze.sh` | | Reformat in place | `tool/dev/format.sh` | | Resolve dependencies | `tool/dev/deps.sh` (`--offline` when pub.dev stalls) | @@ -125,6 +126,35 @@ a target Dart file and fails with `Target file "ios" not found`. open "$(xcode-select -p)/../Applications/DeviceHub.app" ``` +## Coverage + +```sh +tool/dev/coverage.sh +``` + +It runs the test suite with coverage and writes `coverage/lcov.info`, plus +`coverage/html/` where `genhtml` is installed (`brew install lcov`). Every +library in `lib/` is counted, including the ones no test imports; leaving those +out used to make the total read higher than the truth. + +Generated code is left out by the standard `// coverage:ignore-file` comment, +which `flutter test` itself honours, so every way of measuring leaves out the +same files. freezed writes it, `build.yaml` has source_gen add it to every +`.g.dart`, and `l10n.yaml` has gen-l10n add it to the localizations. +`test/tool/generated_code_coverage_test.dart` fails when a generated file comes +out without it. + +Nobody has to remember to run it: + +- **`tool/check.sh`** runs it too, because it is what CI runs. + +**In VS Code**, the Dart extension runs it from the Testing view: **Run Tests +with Coverage** puts a bar on every folder and file under Test Coverage, a +percentage beside each file in the Explorer, and the lines that ran in the +editor. That is the extension's own run, so its numbers are close to this +script's rather than equal to them: it leaves out the libraries no test imports, +and it also counts the lines its branch data names. + ## Commits The full specification, with examples, is **[commit.md](commit.md)**. The parts diff --git a/test/tool/coverage_script_test.dart b/test/tool/coverage_script_test.dart new file mode 100644 index 000000000..255d94953 --- /dev/null +++ b/test/tool/coverage_script_test.dart @@ -0,0 +1,150 @@ +/// `tool/dev/coverage.sh`, end to end, against a fake toolchain. +/// +/// The script is thin, but each thing it decides is a way to report a coverage +/// run that did not happen: a failed suite recorded as a pass, or a refresh +/// that never happens after an edit. These run the real script and `_lib.sh` in +/// a throwaway repository, with `mise` stubbed so `flutter test` is a fixture +/// instead of a minute. +library; + +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; + +/// Writes the lcov a real run would have written, counts how often it was +/// called, and exits with `FAKE_EXIT`. +/// +/// The lcov is written *before* a failing exit, and that order is the point: +/// `flutter test` collects coverage and only then exits with the suite's status +/// (flutter_tools `TestCommand`), so a failed run still leaves a +/// complete-looking file behind. A fake that wrote nothing on failure would let +/// a script that stamped that run as a pass go unnoticed. +const _fakeMise = r'''#!/bin/sh +echo run >> "$FAKE_LOG" +path=coverage/lcov.info +while [ $# -gt 0 ]; do + if [ "$1" = --coverage-path ]; then path="$2"; fi + shift +done +mkdir -p "$(dirname "$path")" +cat > "$path" < dir.deleteSync(recursive: true)); + Process.runSync('git', ['init', '-q', dir.path]); + for (final path in ['tool/dev/_lib.sh', 'tool/dev/coverage.sh']) { + final target = File('${dir.path}/$path') + ..parent.createSync(recursive: true); + File('${Directory.current.path}/$path').copySync(target.path); + } + Process.runSync('chmod', ['+x', '${dir.path}/tool/dev/coverage.sh']); + File('${dir.path}/lib/a.dart') + ..parent.createSync(recursive: true) + ..writeAsStringSync('int a() => 1;\n'); + File('${dir.path}/lib/a_part.dart') + .writeAsStringSync("part of 'a.dart';\n"); + File('${dir.path}/test/a_test.dart') + ..parent.createSync(recursive: true) + ..writeAsStringSync('void main() {}\n'); + // What a checkout ignores; the report must not invalidate the cache key it + // is computed alongside. + File('${dir.path}/.gitignore') + .writeAsStringSync('/coverage/\n/.dart_tool/\n'); + + final bin = Directory('${dir.path}/bin')..createSync(); + final mise = File('${bin.path}/mise')..writeAsStringSync(_fakeMise); + Process.runSync('chmod', ['+x', mise.path]); + return Repo._(dir); + } + + File get lcov => File('${dir.path}/coverage/lcov.info'); + File get _log => File('${dir.path}/fake.log'); + int get flutterRuns => _log.existsSync() ? _log.readAsLinesSync().length : 0; + + ProcessResult run({bool ifChanged = false, int exit = 0}) => Process.runSync( + '${dir.path}/tool/dev/coverage.sh', + [if (ifChanged) '--if-changed'], + workingDirectory: dir.path, + environment: { + 'PATH': '${dir.path}/bin:${Platform.environment['PATH']}', + // Past require_mise: the fake is not an SDK install to inspect. + 'DPIP_MISE_CHECKED': '1', + 'DPIP_NO_CACHE': '', + 'FAKE_LOG': _log.path, + 'FAKE_EXIT': '$exit', + }, + ); +} + +void main() { + test('a run writes the report and imports every library', () { + final repo = Repo.create(); + + final result = repo.run(); + + expect(result.exitCode, 0, reason: '${result.stderr}'); + expect(repo.lcov.readAsStringSync(), contains('SF:lib/a.dart')); + final all = File( + '${repo.dir.path}/.dart_tool/dpip_coverage/all_libraries_test.dart', + ).readAsStringSync(); + expect(all, contains("import 'package:dpip/a.dart' as l0;")); + expect( + all, + isNot(contains('a_part.dart')), + reason: '`part of` cannot be imported', + ); + }); + + test('--if-changed skips an unchanged tree and reruns after an edit', () { + final repo = Repo.create(); + expect(repo.run().exitCode, 0); + expect(repo.flutterRuns, 1); + + expect(repo.run(ifChanged: true).exitCode, 0); + expect(repo.flutterRuns, 1, reason: 'nothing the suite reads has changed'); + + File('${repo.dir.path}/lib/a.dart').writeAsStringSync('int a() => 2;\n'); + expect(repo.run(ifChanged: true).exitCode, 0); + expect(repo.flutterRuns, 2, reason: 'an edit is exactly when to refresh'); + }); + + test('--if-changed still runs when there is no report to keep', () { + final repo = Repo.create(); + expect(repo.run().exitCode, 0); + repo.lcov.deleteSync(); + + expect(repo.run(ifChanged: true).exitCode, 0); + + expect(repo.flutterRuns, 2); + expect(repo.lcov.existsSync(), isTrue); + }); + + test('a failing suite fails, and its tree is not stamped as passing', () { + final repo = Repo.create(); + expect(repo.run().exitCode, 0); + File('${repo.dir.path}/lib/a.dart').writeAsStringSync('int a() => 3;\n'); + + final failed = repo.run(exit: 1); + + expect(failed.exitCode, isNot(0)); + // The same tree again: had the failure been stamped as a pass, this would + // be skipped. + expect(repo.run(ifChanged: true).exitCode, 0); + expect(repo.flutterRuns, 3); + }); +} diff --git a/tool/check.sh b/tool/check.sh index adabb2cbf..b1e521399 100755 --- a/tool/check.sh +++ b/tool/check.sh @@ -30,15 +30,6 @@ source "$(dirname "${BASH_SOURCE[0]}")/dev/_lib.sh" cd "$(repo_root)" -# What each cached step reads. Anything a step's result depends on has to be in -# its list, or the cache will happily hand back an answer about a file that has -# since changed — the one failure mode of a cache like this, and a silent one. -readonly -a CODE_INPUTS=( - lib test tool - pubspec.yaml pubspec.lock analysis_options.yaml l10n.yaml build.yaml -) -readonly -a TEST_INPUTS=("${CODE_INPUTS[@]}" assets shaders) - step 'format + analyze' cached analyze "$(cache_key "${CODE_INPUTS[@]}")" tool/dev/analyze.sh @@ -91,7 +82,13 @@ codegen_check() { } cached codegen "$(cache_key "${CODE_INPUTS[@]}")" codegen_check -step 'test' -cached test "$(cache_key "${TEST_INPUTS[@]}")" tool/dev/test.sh +step 'test (with coverage)' +# The suite as CI runs it. CI's test step is tool/dev/coverage.sh, and running a +# different command here is how this list and ci.yml would start to disagree: +# the coverage run also compiles one test that imports every library in lib/, +# and a file that does not build on the test VM fails there and nowhere in +# tool/dev/test.sh. It caches itself on the same inputs the suite reads, and a +# run leaves the report in coverage/ current as a side effect. +tool/dev/coverage.sh --if-changed step 'all gates passed' diff --git a/tool/check/tooling.sh b/tool/check/tooling.sh index 776d4e3ad..28d015072 100755 --- a/tool/check/tooling.sh +++ b/tool/check/tooling.sh @@ -107,6 +107,7 @@ version: tool/run.sh start the app tool/dev/test.sh the test suite + tool/dev/coverage.sh the test suite, with coverage tool/dev/analyze.sh format + analyzer tool/dev/deps.sh dependencies tool/check.sh every gate CI runs diff --git a/tool/dev/_lib.sh b/tool/dev/_lib.sh index c64064478..a37e6a32d 100755 --- a/tool/dev/_lib.sh +++ b/tool/dev/_lib.sh @@ -136,6 +136,19 @@ cache_key() { shasum -a 256 | cut -d' ' -f1 } +# What each cached step reads. Anything a step's result depends on has to be in +# its list, or the cache will happily hand back an answer about a file that has +# since changed — the one failure mode of a cache like this, and a silent one. +# +# Here rather than in tool/check.sh because two scripts key on them: the gate, +# and tool/dev/coverage.sh. Two copies of this list would be two lists by the +# next edit. +CODE_INPUTS=( + lib test tool + pubspec.yaml pubspec.lock analysis_options.yaml l10n.yaml build.yaml +) +TEST_INPUTS=("${CODE_INPUTS[@]}" assets shaders) + # cached # # `DPIP_NO_CACHE=1` forces a run — for when you suspect the cache itself, which diff --git a/tool/dev/coverage.sh b/tool/dev/coverage.sh new file mode 100755 index 000000000..c266821f2 --- /dev/null +++ b/tool/dev/coverage.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# The test suite, with line coverage for lib/. +# +# tool/dev/coverage.sh # always runs +# tool/dev/coverage.sh --if-changed # skips when nothing it reads changed +# +# It writes `coverage/lcov.info` — what CI uploads to Codecov, what the editor +# reads, and what genhtml turns into `coverage/html/` wherever lcov is installed +# (`brew install lcov`). Both are gitignored. +# +# Generated code is not in it: `flutter test` leaves out every file marked +# `// coverage:ignore-file`, which build.yaml and l10n.yaml put into everything +# the generators write. +# +# This *is* the test suite, not a run beside it: CI's test step is this script, +# and so is tool/check.sh's. A failing test fails here exactly as it does in +# tool/dev/test.sh. +# +# `--if-changed` is what tool/check.sh uses: the answer depends only on the +# files the suite reads, so an unchanged tree keeps the report it already has +# and costs a hash instead of a minute. +source "$(dirname "${BASH_SOURCE[0]}")/_lib.sh" +cd "$(repo_root)" + +if_changed='' +case "${1:-}" in +'') ;; +--if-changed) if_changed=1 ;; +*) + printf 'usage: tool/dev/coverage.sh [--if-changed]\n' >&2 + exit 2 + ;; +esac + +readonly out=coverage +readonly work=.dart_tool/dpip_coverage +mkdir -p "$out" "$work" + +# Every library in lib/, as the test runner sees them. +# +# Generated files are imported like the rest; their `// coverage:ignore-file` is +# what keeps them out of the numbers. `part of` files cannot be imported on +# their own and are reached through their library anyway. +libraries() { + git ls-files -co --exclude-standard -z -- 'lib/*.dart' | + xargs -0 grep -L '^part of' -- || true +} + +# Every step checks its own status. This runs as `cached`'s command, which calls +# it on the left of a `||` — and bash switches `set -e` off for everything +# executed there, functions included. Without the explicit returns a failing +# suite would fall through, the function would end on a status of 0, and the +# cache would record a pass for a tree whose tests fail. +run_coverage() { + # A library no test imports is not in the VM's coverage at all — not at 0%, + # absent — so a total built from the suite alone leaves it out and reads + # higher than the truth. When this was written, 85 of 489 hand-written files + # were missing that way, bootstrap.dart among them. One generated test that + # imports everything puts every executable line on the books, and running it + # beside the suite costs a few seconds of compile, not a second test run. + # + # Under .dart_tool rather than test/: it is regenerated on every run, belongs + # to no one, and nothing that walks test/ should ever find it. + local all="$work/all_libraries_test.dart" + { + printf '// Generated by tool/dev/coverage.sh on every run. Not committed.\n' + printf "import 'package:flutter_test/flutter_test.dart';\n" + local i=0 file + while IFS= read -r file; do + printf "import 'package:dpip/%s' as l%d;\n" "${file#lib/}" "$i" + i=$((i + 1)) + done < <(libraries) + printf "\nvoid main() => test('every library is loaded', () {});\n" + } >"$all" || return + + # `--coverage` writes coverage/lcov.info, and covers this package alone: the + # default for `--coverage-package` is the current package's name. + pinned flutter test --coverage test "$all" || return + + # The browsable report, where lcov is installed — one `brew install lcov` + # away, and CI does not need it because Codecov renders the same numbers. A + # missing or failing genhtml costs the pages and nothing else. + if command -v genhtml >/dev/null 2>&1; then + rm -rf "$out/html" + genhtml --quiet --output-directory "$out/html" "$out/lcov.info" >/dev/null || + printf 'coverage: genhtml failed; the html report was skipped\n' >&2 + fi +} + +key="$(cache_key "${TEST_INPUTS[@]}")" +if [[ -n $if_changed && -f $out/lcov.info ]]; then + cached coverage "$key" run_coverage +else + # Asked for by name, or nothing to show yet: run, and still leave the stamp + # so the next check over the same tree knows it has nothing to do. + DPIP_NO_CACHE=1 cached coverage "$key" run_coverage +fi From 2d8be8d5a1f6ee4daac48f59b7ed6177293edf7a Mon Sep 17 00:00:00 2001 From: YuYu1015 Date: Fri, 18 Sep 2026 03:55:15 +0800 Subject: [PATCH 3/4] ci: upload test coverage to Codecov --- .github/workflows/ci.yml | 32 +++++++++++++++++++++++++++++--- AGENTS.md | 5 +++++ codecov.yml | 25 +++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 codecov.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a28b3736e..221af5dcc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,11 @@ concurrency: jobs: build: runs-on: ubuntu-latest + # Naming any permission sets every one not listed to none, so the checkout's + # read is spelled out beside the OIDC token the Codecov upload signs with. + permissions: + contents: read + id-token: write steps: # Full history: the commit gate walks the range this push or PR adds, and # a shallow clone has no base to compare against. @@ -171,8 +176,29 @@ jobs: # The ETag-cache tests run SQLite on the host via sqflite_common_ffi; # ubuntu ships libsqlite3.so.0 but not the unversioned symlink its loader # needs, so install the dev package. - - name: Install SQLite (sqflite_common_ffi host tests) + - name: Install SQLite run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev - - name: Test - run: bash tool/dev/test.sh + # The test suite, run once, with coverage. The coverage run *is* the test + # run — a failing test fails this step exactly as tool/dev/test.sh would — + # so the lcov below costs a few seconds of instrumentation rather than a + # second pass over the suite. + - name: Test (with coverage) + run: bash tool/dev/coverage.sh + + # Codecov is where the badge in the README, its sunburst graph and the + # pull-request comment all come from; codecov.yml holds the settings. + # + # OIDC rather than a token: there is nothing to store and nothing to + # rotate. A pull request from a fork cannot mint one, so the upload is + # allowed to fail rather than fail the run — by this point the suite has + # passed, which is what the check is for. Uploads from forks need the + # tokenless opt-out for public repositories, in the Codecov org settings. + - name: Upload coverage to Codecov + if: ${{ !cancelled() && hashFiles('coverage/lcov.info') != '' }} + uses: codecov/codecov-action@v5 + with: + use_oidc: true + files: coverage/lcov.info + disable_search: true + fail_ci_if_error: false diff --git a/AGENTS.md b/AGENTS.md index ee1ee4740..9fb1413b7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -146,6 +146,11 @@ out without it. Nobody has to remember to run it: +- **CI** runs it as the test step and uploads the lcov to + [Codecov](https://app.codecov.io/gh/ExpTechTW/DPIP), which is where the + README's badge and sunburst graph, the per-file view and the pull-request + comment all come from. `codecov.yml` holds its settings; the upload signs + with an OIDC token, so there is no secret to keep. - **`tool/check.sh`** runs it too, because it is what CI runs. **In VS Code**, the Dart extension runs it from the Testing view: **Run Tests diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..e12d5cf0d --- /dev/null +++ b/codecov.yml @@ -0,0 +1,25 @@ +# What Codecov does with the lcov CI uploads (.github/workflows/ci.yml). +# +# There is no ignore list here on purpose. Generated code never reaches the +# report: `flutter test` leaves out every file marked `// coverage:ignore-file`, +# which build.yaml and l10n.yaml put into everything the generators write. One +# exclusion, honoured by CI, the editor and anybody running the suite by hand. +coverage: + status: + # The total, against the base commit. A percent of drift is noise — which + # tests happened to load which libraries — rather than a regression, so it + # takes more than that to turn the check red. + project: + default: + target: auto + threshold: 1% + + # The lines a pull request adds or changes. It reports and does not block: + # a gate that fails half the pull requests in a codebase still climbing out + # of 55% teaches people to ignore the check rather than to write tests. + patch: + default: + informational: true + +comment: + layout: condensed_header, condensed_files, condensed_footer From 72990bafe06c719ad1d1aff5f3799be673e860bc Mon Sep 17 00:00:00 2001 From: YuYu1015 Date: Fri, 18 Sep 2026 03:55:15 +0800 Subject: [PATCH 4/4] docs: show the test coverage badge and graph in the README --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 4aebc13c9..6d7b376dc 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ [![正式版](https://img.shields.io/github/v/release/exptechtw/dpip?label=%E6%AD%A3%E5%BC%8F%E7%89%88&color=1B8A50)](https://github.com/ExpTechTW/DPIP/releases/latest) [![測試版](https://img.shields.io/github/v/tag/exptechtw/dpip?sort=date&label=%E6%B8%AC%E8%A9%A6%E7%89%88&color=orange)](https://github.com/ExpTechTW/DPIP/releases) [![CI](https://img.shields.io/github/actions/workflow/status/ExpTechTW/DPIP/ci.yml?branch=main&label=CI)](https://github.com/ExpTechTW/DPIP/actions/workflows/ci.yml) +[![測試覆蓋率](https://img.shields.io/codecov/c/github/ExpTechTW/DPIP/main?label=%E6%B8%AC%E8%A9%A6%E8%A6%86%E8%93%8B%E7%8E%87)](https://app.codecov.io/gh/ExpTechTW/DPIP) [![Discord](https://img.shields.io/discord/926545182407688273?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/5dbHqV8ees) [官網](https://exptech.dev) • [更新日誌](https://github.com/ExpTechTW/DPIP/releases) • [開發文件](AGENTS.md) @@ -126,6 +127,7 @@ tool/dev/build.sh ios # iOS(不含簽章) | 要做什麼 | 指令 | |---|---| | 跑測試 | `tool/dev/test.sh` | +| 跑測試並算覆蓋率 | `tool/dev/coverage.sh` | | 格式化 + 靜態分析 | `tool/dev/analyze.sh` | | 只格式化 | `tool/dev/format.sh` | | 重新產生 l10n | `tool/dev/l10n.sh` | @@ -133,6 +135,12 @@ tool/dev/build.sh ios # iOS(不含簽章) | 砍掉重建 | `tool/dev/clean.sh` | | 跑完 CI 會跑的每一道關卡 | `tool/check.sh` | +覆蓋率由 CI 上傳到 [Codecov](https://app.codecov.io/gh/ExpTechTW/DPIP)。下圖由內到外是資料夾層級,每個扇形是一個檔案,大小是行數、顏色是覆蓋率: + +[![測試覆蓋率旭日圖](https://codecov.io/gh/ExpTechTW/DPIP/branch/main/graphs/sunburst.svg)](https://app.codecov.io/gh/ExpTechTW/DPIP) + +產生出來的程式碼不算在內:`flutter test` 會跳過標了 `// coverage:ignore-file` 的檔案,而 `build.yaml` 與 `l10n.yaml` 讓產生器自動加上那一行。 + **mise 是必要條件,不是建議。** 沒有 mise 就不能建置這個專案 —— 腳本會直接拒絕執行並告訴你怎麼裝。 **絕對不要自己打 `flutter`、`dart` 或 `mise exec`。** 工具鏈只在 `tool/dev/_lib.sh` 一個地方指定。理由不是整潔:shell 的 PATH 只解析一次,`mise activate` 會把它快取起來,所以升級工具鏈之後舊的 SDK 還留在 PATH 上 —— 而**用錯 SDK 一樣建得起來、跑得起來、測試也會過**,差別要到幾天後變成一個沒人重現得出來的失敗才浮現。