From 77da82b6489c52752c7cd1a9a23cec2be986bf30 Mon Sep 17 00:00:00 2001 From: edde746 <86283021+edde746@users.noreply.github.com> Date: Sat, 25 Jul 2026 16:17:25 +0200 Subject: [PATCH] fix(ci): make cross-platform checks deterministic --- .gitattributes | 13 ++ .github/workflows/ci.yml | 1 + CONTRIBUTING.md | 17 +- scripts/check_apple_spm_locks.py | 3 +- scripts/ci_checks.sh | 1 + scripts/ci_website_checks.sh | 1 + scripts/generate_relay_protocol.py | 2 +- scripts/run_maestro_ci.py | 63 +++++- scripts/test_check_apple_spm_locks.py | 11 +- scripts/test_check_codegen.py | 86 ++++++-- scripts/test_generate_relay_protocol.py | 19 ++ scripts/test_run_maestro.py | 213 +++++++++++++++++++- scripts/test_verify_runtime_inputs.py | 57 +++++- test/scripts/generate_ducet_ranks_test.dart | 23 ++- 14 files changed, 458 insertions(+), 52 deletions(-) diff --git a/.gitattributes b/.gitattributes index e4f44113..3048a062 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,3 +2,16 @@ *.Dockerfile text eol=lf Dockerfile text eol=lf *.bat text eol=lf + +# Byte-exact generated Dart checked by scripts/check_codegen.py. +lib/data/ducet_order.dart text eol=lf +lib/data/hid_key_labels.dart text eol=lf +lib/data/iso_639_data.dart text eol=lf +lib/**/*.g.dart text eol=lf +lib/**/*.freezed.dart text eol=lf + +# Byte-exact wakelock inputs hashed in packages/wakelock_plus/provenance.json. +packages/wakelock_plus/pigeons/messages.dart text eol=lf +packages/wakelock_plus/android/src/main/kotlin/dev/fluttercommunity/plus/wakelock/WakelockPlusMessages.g.kt text eol=lf +packages/wakelock_plus/ios/wakelock_plus/Sources/wakelock_plus/include/wakelock_plus/messages.g.h text eol=lf +packages/wakelock_plus/ios/wakelock_plus/Sources/wakelock_plus/messages.g.m text eol=lf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb7a75f2..da84beb7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,6 +73,7 @@ jobs: python3 scripts/test_run_maestro.py python3 scripts/test_maestro_flow_contracts.py python3 scripts/test_maestro_jellyfin_proxy.py + python3 scripts/test_maestro_real_jellyfin.py python3 scripts/check_update_packages_workflow.py python3 scripts/test_pubspec_version.py python3 scripts/test_clean_translations.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2f2cd2f0..54fd82ae 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,8 +63,21 @@ Use `--skip-build` to reuse the debug APK and `--skip-jellyfin-build` to reuse t `--device ` when multiple devices are connected; physical devices also require `--adb-reverse`. Top-level flows live in `.maestro/flows/`, shared setup in `.maestro/subflows/`, and focused regressions in -`.maestro/regression_flows/`. CI runs the same suites from `.github/workflows/e2e.yml` and uploads diagnostics on -failure. +`.maestro/regression_flows/`. Automatic PR groups are declared in `scripts/run_maestro_ci.py::GROUPS`. Every top-level +regression flow must be registered either there or in `DESTRUCTIVE_MANUAL_TARGETS`; reusable subflows are not +independent tests. A manual-only classification must state why the flow cannot run automatically and must not be +described as CI coverage. `.github/workflows/e2e.yml` runs only automatic targets and uploads diagnostics on failure. + +The profile-isolation and profile-teardown regressions create and remove profile connections, so they are a destructive +manual target rather than automatic PR coverage. Run them only against the pre-seeded Jellyfin fixture and a disposable +emulator, using the required opt-in: + +```bash +python3 scripts/run_maestro_ci.py profile-regressions --disposable-emulator +``` + +The target refuses to start without `--disposable-emulator`. Each profile flow writes to its own Jellyfin log and +diagnostics directory under `build/maestro-profile-regressions/`. ### Production container image updates diff --git a/scripts/check_apple_spm_locks.py b/scripts/check_apple_spm_locks.py index f2bf3a68..a2c38c53 100755 --- a/scripts/check_apple_spm_locks.py +++ b/scripts/check_apple_spm_locks.py @@ -8,6 +8,7 @@ import json import re from pathlib import Path from urllib.parse import unquote, urlparse +from urllib.request import url2pathname LOCK_PAIRS = { "iOS": ( @@ -75,7 +76,7 @@ def _resolved_sentry_root(root: Path, errors: list[str]) -> Path | None: uri = package["rootUri"] parsed = urlparse(uri) if parsed.scheme == "file": - return Path(unquote(parsed.path)) + return Path(url2pathname(parsed.path)) if parsed.scheme: errors.append(f"{config_path}: unsupported sentry_flutter root URI {uri!r}") return None diff --git a/scripts/ci_checks.sh b/scripts/ci_checks.sh index 4ed46376..47ac4ed2 100755 --- a/scripts/ci_checks.sh +++ b/scripts/ci_checks.sh @@ -104,6 +104,7 @@ if python3 scripts/check_build_workflow.py && python3 scripts/test_run_maestro.py && python3 scripts/test_maestro_flow_contracts.py && python3 scripts/test_maestro_jellyfin_proxy.py && + python3 scripts/test_maestro_real_jellyfin.py && python3 scripts/test_check_icon_consistency.py; then ok "workflow and script guards passed" else diff --git a/scripts/ci_website_checks.sh b/scripts/ci_website_checks.sh index 901a978f..56e8ca11 100755 --- a/scripts/ci_website_checks.sh +++ b/scripts/ci_website_checks.sh @@ -5,6 +5,7 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT_DIR/website" bun install --frozen-lockfile +bun run test bun run check bun run build python3 "$ROOT_DIR/scripts/test_check_bun_audit.py" diff --git a/scripts/generate_relay_protocol.py b/scripts/generate_relay_protocol.py index 0876c243..d15ca6fc 100755 --- a/scripts/generate_relay_protocol.py +++ b/scripts/generate_relay_protocol.py @@ -143,7 +143,7 @@ def main() -> None: spec = json.loads(SPEC_PATH.read_text(encoding="utf-8")) dart_output = dart_source(spec) go_output = go_source(spec) - DART_PATH.write_text(dart_output, encoding="utf-8") + DART_PATH.write_text(dart_output, encoding="utf-8", newline="\n") GO_PATH.write_text(go_output, encoding="utf-8") diff --git a/scripts/run_maestro_ci.py b/scripts/run_maestro_ci.py index 077acb5a..22504f09 100755 --- a/scripts/run_maestro_ci.py +++ b/scripts/run_maestro_ci.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -"""Run the Maestro groups assigned to each CI emulator.""" +"""Run automatic Maestro groups and guarded destructive manual targets.""" from __future__ import annotations import argparse +import sys from collections.abc import Sequence import run_maestro @@ -104,6 +105,30 @@ GROUPS: dict[str, tuple[tuple[str, ...], ...]] = { } +DESTRUCTIVE_MANUAL_TARGETS: dict[str, tuple[tuple[str, ...], ...]] = { + "profile-regressions": ( + ( + "basic", + "--flow", + ".maestro/regression_flows/01_profile_switch_isolation.yaml", + "--jellyfin-log", + "build/maestro-profile-regressions/profile-switch-isolation-jellyfin.log", + "--diagnostics-dir", + "build/maestro-profile-regressions/profile-switch-isolation-diagnostics", + ), + ( + "basic", + "--flow", + ".maestro/regression_flows/02_profile_teardown.yaml", + "--jellyfin-log", + "build/maestro-profile-regressions/profile-teardown-jellyfin.log", + "--diagnostics-dir", + "build/maestro-profile-regressions/profile-teardown-diagnostics", + ), + ), +} + + def run_android_15_instrumentation() -> None: print("==> Android 15 filtered instrumentation", flush=True) run_maestro._run_checked( @@ -119,9 +144,9 @@ def run_android_15_instrumentation() -> None: ) -def run_group(name: str) -> int: +def run_recipes(recipes: tuple[tuple[str, ...], ...]) -> int: failed = False - for arguments in GROUPS[name]: + for arguments in recipes: print(f"==> Maestro {' '.join(arguments)}", flush=True) exit_status = run_maestro.main(arguments) if exit_status >= 128: @@ -130,7 +155,21 @@ def run_group(name: str) -> int: return 1 if failed else 0 -def run_target(name: str) -> int: +def run_group(name: str) -> int: + return run_recipes(GROUPS[name]) + + +def run_target(name: str, *, disposable_emulator: bool = False) -> int: + if name in DESTRUCTIVE_MANUAL_TARGETS: + if not disposable_emulator: + print( + f"Refusing destructive manual target {name!r}: " + "re-run with --disposable-emulator only on a disposable emulator.", + file=sys.stderr, + ) + return 2 + print(f"==> DESTRUCTIVE manual Maestro target: {name}", flush=True) + return run_recipes(DESTRUCTIVE_MANUAL_TARGETS[name]) if name == ANDROID_15_INSTRUMENTATION_TARGET: run_android_15_instrumentation() return 0 @@ -139,9 +178,21 @@ def run_target(name: str) -> int: def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("target", choices=(*GROUPS, ANDROID_15_INSTRUMENTATION_TARGET)) + parser.add_argument( + "target", + choices=( + *GROUPS, + ANDROID_15_INSTRUMENTATION_TARGET, + *DESTRUCTIVE_MANUAL_TARGETS, + ), + ) + parser.add_argument( + "--disposable-emulator", + action="store_true", + help="opt in to a destructive manual target on a disposable emulator", + ) args = parser.parse_args(argv) - return run_target(args.target) + return run_target(args.target, disposable_emulator=args.disposable_emulator) if __name__ == "__main__": diff --git a/scripts/test_check_apple_spm_locks.py b/scripts/test_check_apple_spm_locks.py index 46ecf1ab..d54dff64 100755 --- a/scripts/test_check_apple_spm_locks.py +++ b/scripts/test_check_apple_spm_locks.py @@ -29,14 +29,14 @@ class AppleSpmLockCheckerTest(unittest.TestCase): def tearDown(self) -> None: self.temporary.cleanup() - def _write_package_config(self) -> None: + def _write_package_config(self, root_uri: str = "../packages/sentry_flutter") -> None: path = self.root / ".dart_tool/package_config.json" - path.parent.mkdir(parents=True) + path.parent.mkdir(parents=True, exist_ok=True) path.write_text( json.dumps( { "configVersion": 2, - "packages": [{"name": "sentry_flutter", "rootUri": "../packages/sentry_flutter"}], + "packages": [{"name": "sentry_flutter", "rootUri": root_uri}], } ), encoding="utf-8", @@ -84,6 +84,11 @@ class AppleSpmLockCheckerTest(unittest.TestCase): self.assertEqual(0, completed.returncode, completed.stdout + completed.stderr) self.assertIn("Apple SwiftPM locks match", completed.stdout) + def test_absolute_file_uri_resolves_sentry_manifests(self) -> None: + self._write_package_config((self.root / "packages/sentry_flutter").resolve().as_uri()) + + self.assertEqual([], CHECKER.validate(self.root)) + def test_reports_project_workspace_version_mismatch(self) -> None: workspace = CHECKER.LOCK_PAIRS["iOS"][1] self._write_lock(workspace, "8.58.0", "old") diff --git a/scripts/test_check_codegen.py b/scripts/test_check_codegen.py index 4699ce21..8335f07a 100644 --- a/scripts/test_check_codegen.py +++ b/scripts/test_check_codegen.py @@ -29,71 +29,97 @@ def executable(path: Path, contents: str) -> None: class CodegenCheckTest(unittest.TestCase): def setUp(self) -> None: self.temp = tempfile.TemporaryDirectory() - self.root = Path(self.temp.name) + temporary_root = Path(self.temp.name) + repository = temporary_root / "repository" + repository.mkdir() self.codegen_temp = Path(tempfile.mkdtemp(prefix="plezy-codegen-test-temp-")) - subprocess.run(["git", "init", "-q", str(self.root)], check=True) - subprocess.run(["git", "config", "user.email", "fixture@example.invalid"], cwd=self.root, check=True) - subprocess.run(["git", "config", "user.name", "Fixture"], cwd=self.root, check=True) + subprocess.run(["git", "init", "-q"], cwd=repository, check=True) + subprocess.run(["git", "config", "user.email", "fixture@example.invalid"], cwd=repository, check=True) + subprocess.run(["git", "config", "user.name", "Fixture"], cwd=repository, check=True) + subprocess.run(["git", "config", "core.autocrlf", "false"], cwd=repository, check=True) - (self.root / "scripts").mkdir() - shutil.copy2(SCRIPT_DIR / "codegen.sh", self.root / "scripts" / "codegen.sh") - shutil.copy2(SCRIPT_DIR / "check_codegen.py", self.root / "scripts" / "check_codegen.py") - (self.root / "scripts" / "generate_relay_protocol.py").write_text("fixture\n", encoding="utf-8") - (self.root / "source.txt").write_text("version one\n", encoding="utf-8") + shutil.copy2(SCRIPT_DIR.parent / ".gitattributes", repository / ".gitattributes") + + (repository / "scripts").mkdir() + shutil.copy2(SCRIPT_DIR / "codegen.sh", repository / "scripts" / "codegen.sh") + shutil.copy2(SCRIPT_DIR / "check_codegen.py", repository / "scripts" / "check_codegen.py") + (repository / "scripts" / "generate_relay_protocol.py").write_text("fixture\n", encoding="utf-8") + (repository / "source.txt").write_text("version one\n", encoding="utf-8") for relative in GENERATED_PATHS: - path = self.root / relative + path = repository / relative path.parent.mkdir(parents=True, exist_ok=True) path.write_text("version one\n", encoding="utf-8") - self.bin = self.root / "fake-bin" + self.bin = temporary_root / "fake-bin" self.bin.mkdir() executable( self.bin / "python3", """#!/usr/bin/env bash +write_lf() { tr -d '\\r' < source.txt > "$1"; } if [[ "$1" == *check_codegen.py ]]; then exec "$REAL_PYTHON" "$@"; fi mkdir -p lib/watch_together/services server -cp source.txt lib/watch_together/services/relay_protocol.g.dart +write_lf lib/watch_together/services/relay_protocol.g.dart cp source.txt server/relay_protocol_gen.go """, ) executable( self.bin / "dart", """#!/usr/bin/env bash +write_lf() { tr -d '\\r' < source.txt > "$1"; } if [ "${FAIL_DART:-0}" -ne 0 ]; then exit "$FAIL_DART"; fi case "$*" in *"generate_ducet_ranks.dart") mkdir -p lib/data - cp source.txt lib/data/ducet_order.dart + write_lf lib/data/ducet_order.dart ;; *"generate_hid_key_labels.dart") mkdir -p lib/data - cp source.txt lib/data/hid_key_labels.dart + write_lf lib/data/hid_key_labels.dart ;; *"generate_iso_639_data.dart") mkdir -p lib/data - cp source.txt lib/data/iso_639_data.dart + write_lf lib/data/iso_639_data.dart ;; "run slang") mkdir -p lib/i18n - cp source.txt lib/i18n/strings.g.dart + write_lf lib/i18n/strings.g.dart ;; *"build_runner"*) mkdir -p lib/models - cp source.txt lib/models/model.g.dart - cp source.txt lib/models/model.freezed.dart + write_lf lib/models/model.g.dart + write_lf lib/models/model.freezed.dart printf '%s\n' "$*" > build-runner-args.txt ;; esac """, ) - subprocess.run(["git", "add", "."], cwd=self.root, check=True) - subprocess.run(["git", "commit", "-qm", "fixture"], cwd=self.root, check=True) + subprocess.run( + ["git", "add", ".gitattributes", "scripts", "source.txt", "lib", "server"], + cwd=repository, + check=True, + ) + subprocess.run(["git", "commit", "-qm", "fixture"], cwd=repository, check=True) + + self.root = temporary_root / "worktree" + subprocess.run( + [ + "git", + "clone", + "-q", + "-c", + "core.autocrlf=true", + str(repository), + str(self.root), + ], + check=True, + ) self.env = os.environ | { "PATH": f"{self.bin}:{os.environ['PATH']}", "REAL_PYTHON": sys.executable, "TMPDIR": str(self.codegen_temp), } + def tearDown(self) -> None: self.temp.cleanup() shutil.rmtree(self.codegen_temp, ignore_errors=True) @@ -139,6 +165,25 @@ esac self.assertEqual(worktrees.count("worktree "), 1) self.assertEqual(list(self.codegen_temp.glob("plezy-codegen-check-*")), []) + def test_crlf_worktree_keeps_generated_dart_lf_and_comparisons_byte_exact(self) -> None: + self.assertIn(b"\r\n", (self.root / "source.txt").read_bytes()) + dart_outputs = [ + self.root / relative + for relative in GENERATED_PATHS + if relative.endswith(".dart") + ] + self.assertTrue(dart_outputs) + self.assertTrue(all(b"\r\n" not in path.read_bytes() for path in dart_outputs)) + self.assertEqual(self.run_codegen("--check").returncode, 0) + + drifted = self.root / "lib/models/model.g.dart" + drifted.write_bytes(drifted.read_bytes().replace(b"\n", b"\r\n")) + result = self.run_codegen("--check") + + self.assertEqual(result.returncode, 1) + self.assertIn("lib/models/model.g.dart", result.stderr) + self.assertIn(b"\r\n", drifted.read_bytes()) + def test_stale_check_reports_sorted_paths_without_changing_caller(self) -> None: (self.root / "source.txt").write_text("version two\n", encoding="utf-8") before = self.generated_state() @@ -253,6 +298,5 @@ esac "run build_runner build --build-filter=lib/models/**\n", ) - if __name__ == "__main__": unittest.main() diff --git a/scripts/test_generate_relay_protocol.py b/scripts/test_generate_relay_protocol.py index 92f2db8c..7a07ceab 100644 --- a/scripts/test_generate_relay_protocol.py +++ b/scripts/test_generate_relay_protocol.py @@ -22,6 +22,25 @@ class RelayProtocolGeneratorTest(unittest.TestCase): ) self.assertIn("func validRelayID(value string, maxLength int) bool", go_output) + def test_main_writes_canonical_lf_dart_output(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + spec_path = root / "relay_protocol.json" + dart_path = root / "relay_protocol.g.dart" + go_path = root / "relay_protocol_gen.go" + spec_path.write_text(json.dumps(self.spec), encoding="utf-8") + + with ( + mock.patch.object(generator, "SPEC_PATH", spec_path), + mock.patch.object(generator, "DART_PATH", dart_path), + mock.patch.object(generator, "GO_PATH", go_path), + ): + generator.main() + + dart_bytes = dart_path.read_bytes() + self.assertIn(b"\n", dart_bytes) + self.assertNotIn(b"\r\n", dart_bytes) + def test_changed_pattern_fails_before_writing_either_target(self) -> None: changed_spec = copy.deepcopy(self.spec) changed_spec["idPattern"] = r"^[A-Za-z0-9_.-]+$" diff --git a/scripts/test_run_maestro.py b/scripts/test_run_maestro.py index 14e13cc7..0d889d84 100755 --- a/scripts/test_run_maestro.py +++ b/scripts/test_run_maestro.py @@ -2,10 +2,12 @@ from __future__ import annotations +import ast from contextlib import redirect_stderr, redirect_stdout from dataclasses import replace import io from pathlib import Path +import shlex import subprocess import sys import unittest @@ -17,6 +19,112 @@ import run_maestro # noqa: E402 import run_maestro_ci # noqa: E402 +ROOT_DIR = Path(__file__).resolve().parent.parent +SCRIPTS_DIR = ROOT_DIR / "scripts" +LOCAL_SCRIPT_TEST_DISPATCHER = SCRIPTS_DIR / "ci_checks.sh" +CI_SCRIPT_TEST_DISPATCHER = ROOT_DIR / ".github/workflows/ci.yml" +E2E_WORKFLOW = ROOT_DIR / ".github/workflows/e2e.yml" +SCRIPT_TEST_DISPATCHERS = ( + LOCAL_SCRIPT_TEST_DISPATCHER, + CI_SCRIPT_TEST_DISPATCHER, + SCRIPTS_DIR / "ci_website_checks.sh", +) +REGRESSION_FLOWS_DIR = ROOT_DIR / ".maestro/regression_flows" + + +def _is_main_guard(expression: ast.expr) -> bool: + if ( + not isinstance(expression, ast.Compare) + or len(expression.ops) != 1 + or not isinstance(expression.ops[0], ast.Eq) + or len(expression.comparators) != 1 + ): + return False + left = expression.left + right = expression.comparators[0] + return ( + isinstance(left, ast.Name) + and left.id == "__name__" + and isinstance(right, ast.Constant) + and right.value == "__main__" + ) or ( + isinstance(right, ast.Name) + and right.id == "__name__" + and isinstance(left, ast.Constant) + and left.value == "__main__" + ) + + +def _executable_script_tests() -> set[str]: + executable = set() + for path in SCRIPTS_DIR.glob("test_*.py"): + module = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + if any(isinstance(node, ast.If) and _is_main_guard(node.test) for node in module.body): + executable.add(path.name) + return executable + + +def _dispatched_script_tests(path: Path) -> list[str]: + dispatched = [] + for line in path.read_text(encoding="utf-8").splitlines(): + try: + command = shlex.split(line.strip(), comments=True) + except ValueError: + continue + if len(command) < 2 or Path(command[0]).name not in {"python", "python3"}: + continue + script_name = Path(command[1]).name + if script_name.startswith("test_") and script_name.endswith(".py"): + dispatched.append(script_name) + return dispatched + + +def _dispatched_maestro_ci_targets(path: Path) -> set[str]: + targets = set() + for line in path.read_text(encoding="utf-8").splitlines(): + try: + command = shlex.split(line.strip(), comments=True) + except ValueError: + continue + for index, token in enumerate(command[:-1]): + if Path(token).name == "run_maestro_ci.py": + targets.add(command[index + 1]) + return targets + + +def _registered_regression_flows( + targets: dict[str, tuple[tuple[str, ...], ...]], +) -> set[str]: + registered = set() + for recipes in targets.values(): + for arguments in recipes: + flow_target = run_maestro.parse_config(arguments, {}).flow_target + if flow_target.parent == REGRESSION_FLOWS_DIR: + registered.add(flow_target.relative_to(ROOT_DIR).as_posix()) + return registered + + +class ScriptTestDispatchTests(unittest.TestCase): + def test_every_executable_script_test_has_a_dispatcher(self) -> None: + dispatched = { + script + for dispatcher in SCRIPT_TEST_DISPATCHERS + for script in _dispatched_script_tests(dispatcher) + } + + self.assertSetEqual(dispatched, _executable_script_tests()) + + def test_real_jellyfin_fixture_test_is_in_local_and_ci_guards(self) -> None: + for dispatcher in (LOCAL_SCRIPT_TEST_DISPATCHER, CI_SCRIPT_TEST_DISPATCHER): + with self.subTest(dispatcher=dispatcher): + self.assertEqual( + _dispatched_script_tests(dispatcher).count( + "test_maestro_real_jellyfin.py" + ), + 1, + ) + + class ParseConfigTests(unittest.TestCase): def test_basic_defaults(self) -> None: config = run_maestro.parse_config([], {}) @@ -198,11 +306,106 @@ class CiGroupTests(unittest.TestCase): self.assertEqual(exit_status, 0) instrumentation.assert_called_once_with() - def test_group_recipes_are_valid_runner_invocations(self) -> None: - for recipes in run_maestro_ci.GROUPS.values(): - for arguments in recipes: - with self.subTest(arguments=arguments): - run_maestro.parse_config(arguments, {}) + def test_all_automatic_and_manual_recipes_are_valid_runner_invocations( + self, + ) -> None: + target_sets = ( + ("automatic", run_maestro_ci.GROUPS), + ("destructive-manual", run_maestro_ci.DESTRUCTIVE_MANUAL_TARGETS), + ) + for classification, targets in target_sets: + for target, recipes in targets.items(): + for arguments in recipes: + with self.subTest( + classification=classification, + target=target, + arguments=arguments, + ): + run_maestro.parse_config(arguments, {}) + + def test_top_level_regression_flow_inventory_is_complete(self) -> None: + inventory = { + path.relative_to(ROOT_DIR).as_posix() + for path in REGRESSION_FLOWS_DIR.glob("*.yaml") + } + automatic = _registered_regression_flows(run_maestro_ci.GROUPS) + destructive_manual = _registered_regression_flows( + run_maestro_ci.DESTRUCTIVE_MANUAL_TARGETS + ) + + self.assertFalse(automatic & destructive_manual) + self.assertSetEqual(automatic | destructive_manual, inventory) + + def test_profile_regressions_are_only_in_the_destructive_manual_target(self) -> None: + profile_flows = { + ".maestro/regression_flows/01_profile_switch_isolation.yaml", + ".maestro/regression_flows/02_profile_teardown.yaml", + } + manual_profile_flows = _registered_regression_flows( + { + "profile-regressions": run_maestro_ci.DESTRUCTIVE_MANUAL_TARGETS[ + "profile-regressions" + ] + } + ) + + self.assertSetEqual(manual_profile_flows, profile_flows) + self.assertTrue( + profile_flows.isdisjoint( + _registered_regression_flows(run_maestro_ci.GROUPS) + ) + ) + + def test_destructive_manual_targets_are_not_automatic_pr_targets(self) -> None: + automatic_pr_targets = _dispatched_maestro_ci_targets(E2E_WORKFLOW) + + self.assertTrue( + set(run_maestro_ci.DESTRUCTIVE_MANUAL_TARGETS).isdisjoint( + automatic_pr_targets + ) + ) + + def test_destructive_manual_recipes_have_distinct_diagnostics(self) -> None: + for target, recipes in run_maestro_ci.DESTRUCTIVE_MANUAL_TARGETS.items(): + with self.subTest(target=target): + configs = [ + run_maestro.parse_config(arguments, {}) for arguments in recipes + ] + self.assertEqual( + len({config.jellyfin_log for config in configs}), len(configs) + ) + self.assertEqual( + len({config.diagnostics_dir for config in configs}), len(configs) + ) + + def test_destructive_manual_target_requires_disposable_emulator_opt_in( + self, + ) -> None: + with ( + patch.object(run_maestro_ci.run_maestro, "main") as run, + redirect_stderr(io.StringIO()) as error_output, + ): + exit_status = run_maestro_ci.main(["profile-regressions"]) + + self.assertEqual(exit_status, 2) + run.assert_not_called() + self.assertIn("Refusing destructive manual target", error_output.getvalue()) + self.assertIn("--disposable-emulator", error_output.getvalue()) + + def test_destructive_manual_target_runs_after_explicit_opt_in(self) -> None: + recipes = run_maestro_ci.DESTRUCTIVE_MANUAL_TARGETS["profile-regressions"] + with ( + patch.object(run_maestro_ci.run_maestro, "main", return_value=0) as run, + redirect_stdout(io.StringIO()), + ): + exit_status = run_maestro_ci.main( + ["profile-regressions", "--disposable-emulator"] + ) + + self.assertEqual(exit_status, 0) + self.assertEqual( + [invocation.args[0] for invocation in run.call_args_list], list(recipes) + ) def test_group_stops_after_interruption(self) -> None: with ( diff --git a/scripts/test_verify_runtime_inputs.py b/scripts/test_verify_runtime_inputs.py index 93bac87c..8f5f6597 100755 --- a/scripts/test_verify_runtime_inputs.py +++ b/scripts/test_verify_runtime_inputs.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import hashlib import importlib.util import json import shutil @@ -34,12 +35,35 @@ FIXTURES = ( class RuntimeInputVerifierTest(unittest.TestCase): def setUp(self) -> None: self.temporary = tempfile.TemporaryDirectory() - self.root = Path(self.temporary.name) - for relative in FIXTURES: + temporary_root = Path(self.temporary.name) + repository = temporary_root / "repository" + repository.mkdir() + for relative in (".gitattributes", *FIXTURES): source = REPOSITORY / relative - destination = self.root / relative + destination = repository / relative destination.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(source, destination) + (repository / "crlf-control.txt").write_bytes(b"control\n") + subprocess.run(["git", "init", "-q"], cwd=repository, check=True) + subprocess.run(["git", "config", "user.email", "fixture@example.invalid"], cwd=repository, check=True) + subprocess.run(["git", "config", "user.name", "Fixture"], cwd=repository, check=True) + subprocess.run(["git", "config", "core.autocrlf", "false"], cwd=repository, check=True) + subprocess.run(["git", "add", "."], cwd=repository, check=True) + subprocess.run(["git", "commit", "-qm", "fixture"], cwd=repository, check=True) + + self.root = temporary_root / "worktree" + subprocess.run( + [ + "git", + "clone", + "-q", + "-c", + "core.autocrlf=true", + str(repository), + str(self.root), + ], + check=True, + ) def tearDown(self) -> None: self.temporary.cleanup() @@ -61,6 +85,33 @@ class RuntimeInputVerifierTest(unittest.TestCase): self.assertEqual(0, completed.returncode, completed.stdout + completed.stderr) self.assertIn("verified offline", completed.stdout) + def test_crlf_worktree_preserves_canonical_lf_provenance_inputs(self) -> None: + self.assertIn(b"\r\n", (self.root / "crlf-control.txt").read_bytes()) + provenance = self._json("packages/wakelock_plus/provenance.json") + + for relative, expected in provenance["artifacts"].items(): + contents = (self.root / "packages/wakelock_plus" / relative).read_bytes() + self.assertNotIn(b"\r\n", contents, relative) + self.assertEqual(expected, hashlib.sha256(contents).hexdigest(), relative) + + self.assertEqual([], CHECKER.validate(self.root)) + + def test_crlf_artifact_drift_is_not_normalized_before_hashing(self) -> None: + relative = "pigeons/messages.dart" + path = self.root / "packages/wakelock_plus" / relative + canonical = path.read_bytes() + self.assertIn(b"\n", canonical) + path.write_bytes(canonical.replace(b"\n", b"\r\n")) + + errors = CHECKER.validate(self.root) + + self.assertTrue(any(str(path) in error and "SHA-256 drift" in error for error in errors)) + provenance = self._json("packages/wakelock_plus/provenance.json") + self.assertNotEqual( + provenance["artifacts"][relative], + hashlib.sha256(path.read_bytes()).hexdigest(), + ) + def test_rejects_linux_cmake_checksum_drift(self) -> None: path = self.root / "linux/CMakeLists.txt" path.write_text(path.read_text(encoding="utf-8").replace("9fe4d6f5", "0fe4d6f5"), encoding="utf-8") diff --git a/test/scripts/generate_ducet_ranks_test.dart b/test/scripts/generate_ducet_ranks_test.dart index eb93d51d..776927d8 100644 --- a/test/scripts/generate_ducet_ranks_test.dart +++ b/test/scripts/generate_ducet_ranks_test.dart @@ -3,6 +3,7 @@ import 'dart:io'; import 'package:crypto/crypto.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as path; import '../../scripts/generate_ducet_ranks.dart'; @@ -187,9 +188,9 @@ void main() { test('a rejected second download preserves the first cache and output', () async { const allKeysText = '0041 ; [.0100.0020.0002]\n'; const fractionalText = '[radical 1=⼀一:一]\n'; - final cache = Directory('${temporaryDirectory.path}/cache')..createSync(); - final marker = File('${cache.path}/marker')..writeAsStringSync('keep'); - final output = File('${temporaryDirectory.path}/ducet_order.dart')..writeAsStringSync('old output'); + final cache = Directory(path.join(temporaryDirectory.path, 'cache'))..createSync(); + final marker = File(path.join(cache.path, 'marker'))..writeAsStringSync('keep'); + final output = File(path.join(temporaryDirectory.path, 'ducet_order.dart'))..writeAsStringSync('old output'); var request = 0; await expectLater( @@ -208,7 +209,7 @@ void main() { ); expect(request, 2); - expect(cache.listSync().map((entity) => entity.path).toList(), [marker.path]); + expect(cache.listSync().map((entity) => path.basename(entity.path)).toList(), ['marker']); expect(marker.readAsStringSync(), 'keep'); expect(output.readAsStringSync(), 'old output'); }); @@ -219,11 +220,12 @@ void main() { 0041 ; [.0101.0020.0002] '''; const fractionalText = '[radical 1=⼀一:一]\n'; - final allKeysFile = File('${temporaryDirectory.path}/allkeys.txt')..writeAsStringSync(duplicateAllKeys); - final fractionalFile = File('${temporaryDirectory.path}/FractionalUCA.txt')..writeAsStringSync(fractionalText); - final cache = Directory('${temporaryDirectory.path}/cache')..createSync(); - final marker = File('${cache.path}/marker')..writeAsStringSync('keep'); - final output = File('${temporaryDirectory.path}/ducet_order.dart')..writeAsStringSync('old output'); + final allKeysFile = File(path.join(temporaryDirectory.path, 'allkeys.txt'))..writeAsStringSync(duplicateAllKeys); + final fractionalFile = File(path.join(temporaryDirectory.path, 'FractionalUCA.txt')) + ..writeAsStringSync(fractionalText); + final cache = Directory(path.join(temporaryDirectory.path, 'cache'))..createSync(); + final marker = File(path.join(cache.path, 'marker'))..writeAsStringSync('keep'); + final output = File(path.join(temporaryDirectory.path, 'ducet_order.dart'))..writeAsStringSync('old output'); await expectLater( generateDucetRanks( @@ -240,7 +242,8 @@ void main() { expect(allKeysFile.readAsStringSync(), duplicateAllKeys); expect(fractionalFile.readAsStringSync(), fractionalText); - expect(cache.listSync().map((entity) => entity.path).toList(), [marker.path]); + expect(cache.listSync().map((entity) => path.basename(entity.path)).toList(), ['marker']); + expect(marker.readAsStringSync(), 'keep'); expect(output.readAsStringSync(), 'old output'); });