fix(ci): make cross-platform checks deterministic

This commit is contained in:
edde746
2026-07-25 16:17:25 +02:00
parent 39ddfd2bd9
commit 77da82b648
14 changed files with 458 additions and 52 deletions
+13
View File
@@ -2,3 +2,16 @@
*.Dockerfile text eol=lf *.Dockerfile text eol=lf
Dockerfile text eol=lf Dockerfile text eol=lf
*.bat text eol=lf *.bat text eol=lf
# Byte-exact generated Dart checked by scripts/check_codegen.py.
lib/data/ducet_order.dart text eol=lf
lib/data/hid_key_labels.dart text eol=lf
lib/data/iso_639_data.dart text eol=lf
lib/**/*.g.dart text eol=lf
lib/**/*.freezed.dart text eol=lf
# Byte-exact wakelock inputs hashed in packages/wakelock_plus/provenance.json.
packages/wakelock_plus/pigeons/messages.dart text eol=lf
packages/wakelock_plus/android/src/main/kotlin/dev/fluttercommunity/plus/wakelock/WakelockPlusMessages.g.kt text eol=lf
packages/wakelock_plus/ios/wakelock_plus/Sources/wakelock_plus/include/wakelock_plus/messages.g.h text eol=lf
packages/wakelock_plus/ios/wakelock_plus/Sources/wakelock_plus/messages.g.m text eol=lf
+1
View File
@@ -73,6 +73,7 @@ jobs:
python3 scripts/test_run_maestro.py python3 scripts/test_run_maestro.py
python3 scripts/test_maestro_flow_contracts.py python3 scripts/test_maestro_flow_contracts.py
python3 scripts/test_maestro_jellyfin_proxy.py python3 scripts/test_maestro_jellyfin_proxy.py
python3 scripts/test_maestro_real_jellyfin.py
python3 scripts/check_update_packages_workflow.py python3 scripts/check_update_packages_workflow.py
python3 scripts/test_pubspec_version.py python3 scripts/test_pubspec_version.py
python3 scripts/test_clean_translations.py python3 scripts/test_clean_translations.py
+15 -2
View File
@@ -63,8 +63,21 @@ Use `--skip-build` to reuse the debug APK and `--skip-jellyfin-build` to reuse t
`--device <adb-serial>` when multiple devices are connected; physical devices also require `--adb-reverse`. `--device <adb-serial>` when multiple devices are connected; physical devices also require `--adb-reverse`.
Top-level flows live in `.maestro/flows/`, shared setup in `.maestro/subflows/`, and focused regressions in Top-level flows live in `.maestro/flows/`, shared setup in `.maestro/subflows/`, and focused regressions in
`.maestro/regression_flows/`. CI runs the same suites from `.github/workflows/e2e.yml` and uploads diagnostics on `.maestro/regression_flows/`. Automatic PR groups are declared in `scripts/run_maestro_ci.py::GROUPS`. Every top-level
failure. regression flow must be registered either there or in `DESTRUCTIVE_MANUAL_TARGETS`; reusable subflows are not
independent tests. A manual-only classification must state why the flow cannot run automatically and must not be
described as CI coverage. `.github/workflows/e2e.yml` runs only automatic targets and uploads diagnostics on failure.
The profile-isolation and profile-teardown regressions create and remove profile connections, so they are a destructive
manual target rather than automatic PR coverage. Run them only against the pre-seeded Jellyfin fixture and a disposable
emulator, using the required opt-in:
```bash
python3 scripts/run_maestro_ci.py profile-regressions --disposable-emulator
```
The target refuses to start without `--disposable-emulator`. Each profile flow writes to its own Jellyfin log and
diagnostics directory under `build/maestro-profile-regressions/`.
### Production container image updates ### Production container image updates
+2 -1
View File
@@ -8,6 +8,7 @@ import json
import re import re
from pathlib import Path from pathlib import Path
from urllib.parse import unquote, urlparse from urllib.parse import unquote, urlparse
from urllib.request import url2pathname
LOCK_PAIRS = { LOCK_PAIRS = {
"iOS": ( "iOS": (
@@ -75,7 +76,7 @@ def _resolved_sentry_root(root: Path, errors: list[str]) -> Path | None:
uri = package["rootUri"] uri = package["rootUri"]
parsed = urlparse(uri) parsed = urlparse(uri)
if parsed.scheme == "file": if parsed.scheme == "file":
return Path(unquote(parsed.path)) return Path(url2pathname(parsed.path))
if parsed.scheme: if parsed.scheme:
errors.append(f"{config_path}: unsupported sentry_flutter root URI {uri!r}") errors.append(f"{config_path}: unsupported sentry_flutter root URI {uri!r}")
return None return None
+1
View File
@@ -104,6 +104,7 @@ if python3 scripts/check_build_workflow.py &&
python3 scripts/test_run_maestro.py && python3 scripts/test_run_maestro.py &&
python3 scripts/test_maestro_flow_contracts.py && python3 scripts/test_maestro_flow_contracts.py &&
python3 scripts/test_maestro_jellyfin_proxy.py && python3 scripts/test_maestro_jellyfin_proxy.py &&
python3 scripts/test_maestro_real_jellyfin.py &&
python3 scripts/test_check_icon_consistency.py; then python3 scripts/test_check_icon_consistency.py; then
ok "workflow and script guards passed" ok "workflow and script guards passed"
else else
+1
View File
@@ -5,6 +5,7 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT_DIR/website" cd "$ROOT_DIR/website"
bun install --frozen-lockfile bun install --frozen-lockfile
bun run test
bun run check bun run check
bun run build bun run build
python3 "$ROOT_DIR/scripts/test_check_bun_audit.py" python3 "$ROOT_DIR/scripts/test_check_bun_audit.py"
+1 -1
View File
@@ -143,7 +143,7 @@ def main() -> None:
spec = json.loads(SPEC_PATH.read_text(encoding="utf-8")) spec = json.loads(SPEC_PATH.read_text(encoding="utf-8"))
dart_output = dart_source(spec) dart_output = dart_source(spec)
go_output = go_source(spec) go_output = go_source(spec)
DART_PATH.write_text(dart_output, encoding="utf-8") DART_PATH.write_text(dart_output, encoding="utf-8", newline="\n")
GO_PATH.write_text(go_output, encoding="utf-8") GO_PATH.write_text(go_output, encoding="utf-8")
+57 -6
View File
@@ -1,9 +1,10 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Run the Maestro groups assigned to each CI emulator.""" """Run automatic Maestro groups and guarded destructive manual targets."""
from __future__ import annotations from __future__ import annotations
import argparse import argparse
import sys
from collections.abc import Sequence from collections.abc import Sequence
import run_maestro import run_maestro
@@ -104,6 +105,30 @@ GROUPS: dict[str, tuple[tuple[str, ...], ...]] = {
} }
DESTRUCTIVE_MANUAL_TARGETS: dict[str, tuple[tuple[str, ...], ...]] = {
"profile-regressions": (
(
"basic",
"--flow",
".maestro/regression_flows/01_profile_switch_isolation.yaml",
"--jellyfin-log",
"build/maestro-profile-regressions/profile-switch-isolation-jellyfin.log",
"--diagnostics-dir",
"build/maestro-profile-regressions/profile-switch-isolation-diagnostics",
),
(
"basic",
"--flow",
".maestro/regression_flows/02_profile_teardown.yaml",
"--jellyfin-log",
"build/maestro-profile-regressions/profile-teardown-jellyfin.log",
"--diagnostics-dir",
"build/maestro-profile-regressions/profile-teardown-diagnostics",
),
),
}
def run_android_15_instrumentation() -> None: def run_android_15_instrumentation() -> None:
print("==> Android 15 filtered instrumentation", flush=True) print("==> Android 15 filtered instrumentation", flush=True)
run_maestro._run_checked( run_maestro._run_checked(
@@ -119,9 +144,9 @@ def run_android_15_instrumentation() -> None:
) )
def run_group(name: str) -> int: def run_recipes(recipes: tuple[tuple[str, ...], ...]) -> int:
failed = False failed = False
for arguments in GROUPS[name]: for arguments in recipes:
print(f"==> Maestro {' '.join(arguments)}", flush=True) print(f"==> Maestro {' '.join(arguments)}", flush=True)
exit_status = run_maestro.main(arguments) exit_status = run_maestro.main(arguments)
if exit_status >= 128: if exit_status >= 128:
@@ -130,7 +155,21 @@ def run_group(name: str) -> int:
return 1 if failed else 0 return 1 if failed else 0
def run_target(name: str) -> int: def run_group(name: str) -> int:
return run_recipes(GROUPS[name])
def run_target(name: str, *, disposable_emulator: bool = False) -> int:
if name in DESTRUCTIVE_MANUAL_TARGETS:
if not disposable_emulator:
print(
f"Refusing destructive manual target {name!r}: "
"re-run with --disposable-emulator only on a disposable emulator.",
file=sys.stderr,
)
return 2
print(f"==> DESTRUCTIVE manual Maestro target: {name}", flush=True)
return run_recipes(DESTRUCTIVE_MANUAL_TARGETS[name])
if name == ANDROID_15_INSTRUMENTATION_TARGET: if name == ANDROID_15_INSTRUMENTATION_TARGET:
run_android_15_instrumentation() run_android_15_instrumentation()
return 0 return 0
@@ -139,9 +178,21 @@ def run_target(name: str) -> int:
def main(argv: Sequence[str] | None = None) -> int: def main(argv: Sequence[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__) parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("target", choices=(*GROUPS, ANDROID_15_INSTRUMENTATION_TARGET)) parser.add_argument(
"target",
choices=(
*GROUPS,
ANDROID_15_INSTRUMENTATION_TARGET,
*DESTRUCTIVE_MANUAL_TARGETS,
),
)
parser.add_argument(
"--disposable-emulator",
action="store_true",
help="opt in to a destructive manual target on a disposable emulator",
)
args = parser.parse_args(argv) args = parser.parse_args(argv)
return run_target(args.target) return run_target(args.target, disposable_emulator=args.disposable_emulator)
if __name__ == "__main__": if __name__ == "__main__":
+8 -3
View File
@@ -29,14 +29,14 @@ class AppleSpmLockCheckerTest(unittest.TestCase):
def tearDown(self) -> None: def tearDown(self) -> None:
self.temporary.cleanup() self.temporary.cleanup()
def _write_package_config(self) -> None: def _write_package_config(self, root_uri: str = "../packages/sentry_flutter") -> None:
path = self.root / ".dart_tool/package_config.json" path = self.root / ".dart_tool/package_config.json"
path.parent.mkdir(parents=True) path.parent.mkdir(parents=True, exist_ok=True)
path.write_text( path.write_text(
json.dumps( json.dumps(
{ {
"configVersion": 2, "configVersion": 2,
"packages": [{"name": "sentry_flutter", "rootUri": "../packages/sentry_flutter"}], "packages": [{"name": "sentry_flutter", "rootUri": root_uri}],
} }
), ),
encoding="utf-8", encoding="utf-8",
@@ -84,6 +84,11 @@ class AppleSpmLockCheckerTest(unittest.TestCase):
self.assertEqual(0, completed.returncode, completed.stdout + completed.stderr) self.assertEqual(0, completed.returncode, completed.stdout + completed.stderr)
self.assertIn("Apple SwiftPM locks match", completed.stdout) self.assertIn("Apple SwiftPM locks match", completed.stdout)
def test_absolute_file_uri_resolves_sentry_manifests(self) -> None:
self._write_package_config((self.root / "packages/sentry_flutter").resolve().as_uri())
self.assertEqual([], CHECKER.validate(self.root))
def test_reports_project_workspace_version_mismatch(self) -> None: def test_reports_project_workspace_version_mismatch(self) -> None:
workspace = CHECKER.LOCK_PAIRS["iOS"][1] workspace = CHECKER.LOCK_PAIRS["iOS"][1]
self._write_lock(workspace, "8.58.0", "old") self._write_lock(workspace, "8.58.0", "old")
+65 -21
View File
@@ -29,71 +29,97 @@ def executable(path: Path, contents: str) -> None:
class CodegenCheckTest(unittest.TestCase): class CodegenCheckTest(unittest.TestCase):
def setUp(self) -> None: def setUp(self) -> None:
self.temp = tempfile.TemporaryDirectory() self.temp = tempfile.TemporaryDirectory()
self.root = Path(self.temp.name) temporary_root = Path(self.temp.name)
repository = temporary_root / "repository"
repository.mkdir()
self.codegen_temp = Path(tempfile.mkdtemp(prefix="plezy-codegen-test-temp-")) self.codegen_temp = Path(tempfile.mkdtemp(prefix="plezy-codegen-test-temp-"))
subprocess.run(["git", "init", "-q", str(self.root)], check=True) subprocess.run(["git", "init", "-q"], cwd=repository, check=True)
subprocess.run(["git", "config", "user.email", "fixture@example.invalid"], cwd=self.root, check=True) subprocess.run(["git", "config", "user.email", "fixture@example.invalid"], cwd=repository, check=True)
subprocess.run(["git", "config", "user.name", "Fixture"], cwd=self.root, check=True) subprocess.run(["git", "config", "user.name", "Fixture"], cwd=repository, check=True)
subprocess.run(["git", "config", "core.autocrlf", "false"], cwd=repository, check=True)
(self.root / "scripts").mkdir() shutil.copy2(SCRIPT_DIR.parent / ".gitattributes", repository / ".gitattributes")
shutil.copy2(SCRIPT_DIR / "codegen.sh", self.root / "scripts" / "codegen.sh")
shutil.copy2(SCRIPT_DIR / "check_codegen.py", self.root / "scripts" / "check_codegen.py") (repository / "scripts").mkdir()
(self.root / "scripts" / "generate_relay_protocol.py").write_text("fixture\n", encoding="utf-8") shutil.copy2(SCRIPT_DIR / "codegen.sh", repository / "scripts" / "codegen.sh")
(self.root / "source.txt").write_text("version one\n", encoding="utf-8") shutil.copy2(SCRIPT_DIR / "check_codegen.py", repository / "scripts" / "check_codegen.py")
(repository / "scripts" / "generate_relay_protocol.py").write_text("fixture\n", encoding="utf-8")
(repository / "source.txt").write_text("version one\n", encoding="utf-8")
for relative in GENERATED_PATHS: for relative in GENERATED_PATHS:
path = self.root / relative path = repository / relative
path.parent.mkdir(parents=True, exist_ok=True) path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("version one\n", encoding="utf-8") path.write_text("version one\n", encoding="utf-8")
self.bin = self.root / "fake-bin" self.bin = temporary_root / "fake-bin"
self.bin.mkdir() self.bin.mkdir()
executable( executable(
self.bin / "python3", self.bin / "python3",
"""#!/usr/bin/env bash """#!/usr/bin/env bash
write_lf() { tr -d '\\r' < source.txt > "$1"; }
if [[ "$1" == *check_codegen.py ]]; then exec "$REAL_PYTHON" "$@"; fi if [[ "$1" == *check_codegen.py ]]; then exec "$REAL_PYTHON" "$@"; fi
mkdir -p lib/watch_together/services server mkdir -p lib/watch_together/services server
cp source.txt lib/watch_together/services/relay_protocol.g.dart write_lf lib/watch_together/services/relay_protocol.g.dart
cp source.txt server/relay_protocol_gen.go cp source.txt server/relay_protocol_gen.go
""", """,
) )
executable( executable(
self.bin / "dart", self.bin / "dart",
"""#!/usr/bin/env bash """#!/usr/bin/env bash
write_lf() { tr -d '\\r' < source.txt > "$1"; }
if [ "${FAIL_DART:-0}" -ne 0 ]; then exit "$FAIL_DART"; fi if [ "${FAIL_DART:-0}" -ne 0 ]; then exit "$FAIL_DART"; fi
case "$*" in case "$*" in
*"generate_ducet_ranks.dart") *"generate_ducet_ranks.dart")
mkdir -p lib/data mkdir -p lib/data
cp source.txt lib/data/ducet_order.dart write_lf lib/data/ducet_order.dart
;; ;;
*"generate_hid_key_labels.dart") *"generate_hid_key_labels.dart")
mkdir -p lib/data mkdir -p lib/data
cp source.txt lib/data/hid_key_labels.dart write_lf lib/data/hid_key_labels.dart
;; ;;
*"generate_iso_639_data.dart") *"generate_iso_639_data.dart")
mkdir -p lib/data mkdir -p lib/data
cp source.txt lib/data/iso_639_data.dart write_lf lib/data/iso_639_data.dart
;; ;;
"run slang") "run slang")
mkdir -p lib/i18n mkdir -p lib/i18n
cp source.txt lib/i18n/strings.g.dart write_lf lib/i18n/strings.g.dart
;; ;;
*"build_runner"*) *"build_runner"*)
mkdir -p lib/models mkdir -p lib/models
cp source.txt lib/models/model.g.dart write_lf lib/models/model.g.dart
cp source.txt lib/models/model.freezed.dart write_lf lib/models/model.freezed.dart
printf '%s\n' "$*" > build-runner-args.txt printf '%s\n' "$*" > build-runner-args.txt
;; ;;
esac esac
""", """,
) )
subprocess.run(["git", "add", "."], cwd=self.root, check=True) subprocess.run(
subprocess.run(["git", "commit", "-qm", "fixture"], cwd=self.root, check=True) ["git", "add", ".gitattributes", "scripts", "source.txt", "lib", "server"],
cwd=repository,
check=True,
)
subprocess.run(["git", "commit", "-qm", "fixture"], cwd=repository, check=True)
self.root = temporary_root / "worktree"
subprocess.run(
[
"git",
"clone",
"-q",
"-c",
"core.autocrlf=true",
str(repository),
str(self.root),
],
check=True,
)
self.env = os.environ | { self.env = os.environ | {
"PATH": f"{self.bin}:{os.environ['PATH']}", "PATH": f"{self.bin}:{os.environ['PATH']}",
"REAL_PYTHON": sys.executable, "REAL_PYTHON": sys.executable,
"TMPDIR": str(self.codegen_temp), "TMPDIR": str(self.codegen_temp),
} }
def tearDown(self) -> None: def tearDown(self) -> None:
self.temp.cleanup() self.temp.cleanup()
shutil.rmtree(self.codegen_temp, ignore_errors=True) shutil.rmtree(self.codegen_temp, ignore_errors=True)
@@ -139,6 +165,25 @@ esac
self.assertEqual(worktrees.count("worktree "), 1) self.assertEqual(worktrees.count("worktree "), 1)
self.assertEqual(list(self.codegen_temp.glob("plezy-codegen-check-*")), []) self.assertEqual(list(self.codegen_temp.glob("plezy-codegen-check-*")), [])
def test_crlf_worktree_keeps_generated_dart_lf_and_comparisons_byte_exact(self) -> None:
self.assertIn(b"\r\n", (self.root / "source.txt").read_bytes())
dart_outputs = [
self.root / relative
for relative in GENERATED_PATHS
if relative.endswith(".dart")
]
self.assertTrue(dart_outputs)
self.assertTrue(all(b"\r\n" not in path.read_bytes() for path in dart_outputs))
self.assertEqual(self.run_codegen("--check").returncode, 0)
drifted = self.root / "lib/models/model.g.dart"
drifted.write_bytes(drifted.read_bytes().replace(b"\n", b"\r\n"))
result = self.run_codegen("--check")
self.assertEqual(result.returncode, 1)
self.assertIn("lib/models/model.g.dart", result.stderr)
self.assertIn(b"\r\n", drifted.read_bytes())
def test_stale_check_reports_sorted_paths_without_changing_caller(self) -> None: def test_stale_check_reports_sorted_paths_without_changing_caller(self) -> None:
(self.root / "source.txt").write_text("version two\n", encoding="utf-8") (self.root / "source.txt").write_text("version two\n", encoding="utf-8")
before = self.generated_state() before = self.generated_state()
@@ -253,6 +298,5 @@ esac
"run build_runner build --build-filter=lib/models/**\n", "run build_runner build --build-filter=lib/models/**\n",
) )
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+19
View File
@@ -22,6 +22,25 @@ class RelayProtocolGeneratorTest(unittest.TestCase):
) )
self.assertIn("func validRelayID(value string, maxLength int) bool", go_output) self.assertIn("func validRelayID(value string, maxLength int) bool", go_output)
def test_main_writes_canonical_lf_dart_output(self) -> None:
with tempfile.TemporaryDirectory() as temporary_directory:
root = Path(temporary_directory)
spec_path = root / "relay_protocol.json"
dart_path = root / "relay_protocol.g.dart"
go_path = root / "relay_protocol_gen.go"
spec_path.write_text(json.dumps(self.spec), encoding="utf-8")
with (
mock.patch.object(generator, "SPEC_PATH", spec_path),
mock.patch.object(generator, "DART_PATH", dart_path),
mock.patch.object(generator, "GO_PATH", go_path),
):
generator.main()
dart_bytes = dart_path.read_bytes()
self.assertIn(b"\n", dart_bytes)
self.assertNotIn(b"\r\n", dart_bytes)
def test_changed_pattern_fails_before_writing_either_target(self) -> None: def test_changed_pattern_fails_before_writing_either_target(self) -> None:
changed_spec = copy.deepcopy(self.spec) changed_spec = copy.deepcopy(self.spec)
changed_spec["idPattern"] = r"^[A-Za-z0-9_.-]+$" changed_spec["idPattern"] = r"^[A-Za-z0-9_.-]+$"
+208 -5
View File
@@ -2,10 +2,12 @@
from __future__ import annotations from __future__ import annotations
import ast
from contextlib import redirect_stderr, redirect_stdout from contextlib import redirect_stderr, redirect_stdout
from dataclasses import replace from dataclasses import replace
import io import io
from pathlib import Path from pathlib import Path
import shlex
import subprocess import subprocess
import sys import sys
import unittest import unittest
@@ -17,6 +19,112 @@ import run_maestro # noqa: E402
import run_maestro_ci # noqa: E402 import run_maestro_ci # noqa: E402
ROOT_DIR = Path(__file__).resolve().parent.parent
SCRIPTS_DIR = ROOT_DIR / "scripts"
LOCAL_SCRIPT_TEST_DISPATCHER = SCRIPTS_DIR / "ci_checks.sh"
CI_SCRIPT_TEST_DISPATCHER = ROOT_DIR / ".github/workflows/ci.yml"
E2E_WORKFLOW = ROOT_DIR / ".github/workflows/e2e.yml"
SCRIPT_TEST_DISPATCHERS = (
LOCAL_SCRIPT_TEST_DISPATCHER,
CI_SCRIPT_TEST_DISPATCHER,
SCRIPTS_DIR / "ci_website_checks.sh",
)
REGRESSION_FLOWS_DIR = ROOT_DIR / ".maestro/regression_flows"
def _is_main_guard(expression: ast.expr) -> bool:
if (
not isinstance(expression, ast.Compare)
or len(expression.ops) != 1
or not isinstance(expression.ops[0], ast.Eq)
or len(expression.comparators) != 1
):
return False
left = expression.left
right = expression.comparators[0]
return (
isinstance(left, ast.Name)
and left.id == "__name__"
and isinstance(right, ast.Constant)
and right.value == "__main__"
) or (
isinstance(right, ast.Name)
and right.id == "__name__"
and isinstance(left, ast.Constant)
and left.value == "__main__"
)
def _executable_script_tests() -> set[str]:
executable = set()
for path in SCRIPTS_DIR.glob("test_*.py"):
module = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
if any(isinstance(node, ast.If) and _is_main_guard(node.test) for node in module.body):
executable.add(path.name)
return executable
def _dispatched_script_tests(path: Path) -> list[str]:
dispatched = []
for line in path.read_text(encoding="utf-8").splitlines():
try:
command = shlex.split(line.strip(), comments=True)
except ValueError:
continue
if len(command) < 2 or Path(command[0]).name not in {"python", "python3"}:
continue
script_name = Path(command[1]).name
if script_name.startswith("test_") and script_name.endswith(".py"):
dispatched.append(script_name)
return dispatched
def _dispatched_maestro_ci_targets(path: Path) -> set[str]:
targets = set()
for line in path.read_text(encoding="utf-8").splitlines():
try:
command = shlex.split(line.strip(), comments=True)
except ValueError:
continue
for index, token in enumerate(command[:-1]):
if Path(token).name == "run_maestro_ci.py":
targets.add(command[index + 1])
return targets
def _registered_regression_flows(
targets: dict[str, tuple[tuple[str, ...], ...]],
) -> set[str]:
registered = set()
for recipes in targets.values():
for arguments in recipes:
flow_target = run_maestro.parse_config(arguments, {}).flow_target
if flow_target.parent == REGRESSION_FLOWS_DIR:
registered.add(flow_target.relative_to(ROOT_DIR).as_posix())
return registered
class ScriptTestDispatchTests(unittest.TestCase):
def test_every_executable_script_test_has_a_dispatcher(self) -> None:
dispatched = {
script
for dispatcher in SCRIPT_TEST_DISPATCHERS
for script in _dispatched_script_tests(dispatcher)
}
self.assertSetEqual(dispatched, _executable_script_tests())
def test_real_jellyfin_fixture_test_is_in_local_and_ci_guards(self) -> None:
for dispatcher in (LOCAL_SCRIPT_TEST_DISPATCHER, CI_SCRIPT_TEST_DISPATCHER):
with self.subTest(dispatcher=dispatcher):
self.assertEqual(
_dispatched_script_tests(dispatcher).count(
"test_maestro_real_jellyfin.py"
),
1,
)
class ParseConfigTests(unittest.TestCase): class ParseConfigTests(unittest.TestCase):
def test_basic_defaults(self) -> None: def test_basic_defaults(self) -> None:
config = run_maestro.parse_config([], {}) config = run_maestro.parse_config([], {})
@@ -198,11 +306,106 @@ class CiGroupTests(unittest.TestCase):
self.assertEqual(exit_status, 0) self.assertEqual(exit_status, 0)
instrumentation.assert_called_once_with() instrumentation.assert_called_once_with()
def test_group_recipes_are_valid_runner_invocations(self) -> None: def test_all_automatic_and_manual_recipes_are_valid_runner_invocations(
for recipes in run_maestro_ci.GROUPS.values(): self,
for arguments in recipes: ) -> None:
with self.subTest(arguments=arguments): target_sets = (
run_maestro.parse_config(arguments, {}) ("automatic", run_maestro_ci.GROUPS),
("destructive-manual", run_maestro_ci.DESTRUCTIVE_MANUAL_TARGETS),
)
for classification, targets in target_sets:
for target, recipes in targets.items():
for arguments in recipes:
with self.subTest(
classification=classification,
target=target,
arguments=arguments,
):
run_maestro.parse_config(arguments, {})
def test_top_level_regression_flow_inventory_is_complete(self) -> None:
inventory = {
path.relative_to(ROOT_DIR).as_posix()
for path in REGRESSION_FLOWS_DIR.glob("*.yaml")
}
automatic = _registered_regression_flows(run_maestro_ci.GROUPS)
destructive_manual = _registered_regression_flows(
run_maestro_ci.DESTRUCTIVE_MANUAL_TARGETS
)
self.assertFalse(automatic & destructive_manual)
self.assertSetEqual(automatic | destructive_manual, inventory)
def test_profile_regressions_are_only_in_the_destructive_manual_target(self) -> None:
profile_flows = {
".maestro/regression_flows/01_profile_switch_isolation.yaml",
".maestro/regression_flows/02_profile_teardown.yaml",
}
manual_profile_flows = _registered_regression_flows(
{
"profile-regressions": run_maestro_ci.DESTRUCTIVE_MANUAL_TARGETS[
"profile-regressions"
]
}
)
self.assertSetEqual(manual_profile_flows, profile_flows)
self.assertTrue(
profile_flows.isdisjoint(
_registered_regression_flows(run_maestro_ci.GROUPS)
)
)
def test_destructive_manual_targets_are_not_automatic_pr_targets(self) -> None:
automatic_pr_targets = _dispatched_maestro_ci_targets(E2E_WORKFLOW)
self.assertTrue(
set(run_maestro_ci.DESTRUCTIVE_MANUAL_TARGETS).isdisjoint(
automatic_pr_targets
)
)
def test_destructive_manual_recipes_have_distinct_diagnostics(self) -> None:
for target, recipes in run_maestro_ci.DESTRUCTIVE_MANUAL_TARGETS.items():
with self.subTest(target=target):
configs = [
run_maestro.parse_config(arguments, {}) for arguments in recipes
]
self.assertEqual(
len({config.jellyfin_log for config in configs}), len(configs)
)
self.assertEqual(
len({config.diagnostics_dir for config in configs}), len(configs)
)
def test_destructive_manual_target_requires_disposable_emulator_opt_in(
self,
) -> None:
with (
patch.object(run_maestro_ci.run_maestro, "main") as run,
redirect_stderr(io.StringIO()) as error_output,
):
exit_status = run_maestro_ci.main(["profile-regressions"])
self.assertEqual(exit_status, 2)
run.assert_not_called()
self.assertIn("Refusing destructive manual target", error_output.getvalue())
self.assertIn("--disposable-emulator", error_output.getvalue())
def test_destructive_manual_target_runs_after_explicit_opt_in(self) -> None:
recipes = run_maestro_ci.DESTRUCTIVE_MANUAL_TARGETS["profile-regressions"]
with (
patch.object(run_maestro_ci.run_maestro, "main", return_value=0) as run,
redirect_stdout(io.StringIO()),
):
exit_status = run_maestro_ci.main(
["profile-regressions", "--disposable-emulator"]
)
self.assertEqual(exit_status, 0)
self.assertEqual(
[invocation.args[0] for invocation in run.call_args_list], list(recipes)
)
def test_group_stops_after_interruption(self) -> None: def test_group_stops_after_interruption(self) -> None:
with ( with (
+54 -3
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import hashlib
import importlib.util import importlib.util
import json import json
import shutil import shutil
@@ -34,12 +35,35 @@ FIXTURES = (
class RuntimeInputVerifierTest(unittest.TestCase): class RuntimeInputVerifierTest(unittest.TestCase):
def setUp(self) -> None: def setUp(self) -> None:
self.temporary = tempfile.TemporaryDirectory() self.temporary = tempfile.TemporaryDirectory()
self.root = Path(self.temporary.name) temporary_root = Path(self.temporary.name)
for relative in FIXTURES: repository = temporary_root / "repository"
repository.mkdir()
for relative in (".gitattributes", *FIXTURES):
source = REPOSITORY / relative source = REPOSITORY / relative
destination = self.root / relative destination = repository / relative
destination.parent.mkdir(parents=True, exist_ok=True) destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source, destination) shutil.copy2(source, destination)
(repository / "crlf-control.txt").write_bytes(b"control\n")
subprocess.run(["git", "init", "-q"], cwd=repository, check=True)
subprocess.run(["git", "config", "user.email", "fixture@example.invalid"], cwd=repository, check=True)
subprocess.run(["git", "config", "user.name", "Fixture"], cwd=repository, check=True)
subprocess.run(["git", "config", "core.autocrlf", "false"], cwd=repository, check=True)
subprocess.run(["git", "add", "."], cwd=repository, check=True)
subprocess.run(["git", "commit", "-qm", "fixture"], cwd=repository, check=True)
self.root = temporary_root / "worktree"
subprocess.run(
[
"git",
"clone",
"-q",
"-c",
"core.autocrlf=true",
str(repository),
str(self.root),
],
check=True,
)
def tearDown(self) -> None: def tearDown(self) -> None:
self.temporary.cleanup() self.temporary.cleanup()
@@ -61,6 +85,33 @@ class RuntimeInputVerifierTest(unittest.TestCase):
self.assertEqual(0, completed.returncode, completed.stdout + completed.stderr) self.assertEqual(0, completed.returncode, completed.stdout + completed.stderr)
self.assertIn("verified offline", completed.stdout) self.assertIn("verified offline", completed.stdout)
def test_crlf_worktree_preserves_canonical_lf_provenance_inputs(self) -> None:
self.assertIn(b"\r\n", (self.root / "crlf-control.txt").read_bytes())
provenance = self._json("packages/wakelock_plus/provenance.json")
for relative, expected in provenance["artifacts"].items():
contents = (self.root / "packages/wakelock_plus" / relative).read_bytes()
self.assertNotIn(b"\r\n", contents, relative)
self.assertEqual(expected, hashlib.sha256(contents).hexdigest(), relative)
self.assertEqual([], CHECKER.validate(self.root))
def test_crlf_artifact_drift_is_not_normalized_before_hashing(self) -> None:
relative = "pigeons/messages.dart"
path = self.root / "packages/wakelock_plus" / relative
canonical = path.read_bytes()
self.assertIn(b"\n", canonical)
path.write_bytes(canonical.replace(b"\n", b"\r\n"))
errors = CHECKER.validate(self.root)
self.assertTrue(any(str(path) in error and "SHA-256 drift" in error for error in errors))
provenance = self._json("packages/wakelock_plus/provenance.json")
self.assertNotEqual(
provenance["artifacts"][relative],
hashlib.sha256(path.read_bytes()).hexdigest(),
)
def test_rejects_linux_cmake_checksum_drift(self) -> None: def test_rejects_linux_cmake_checksum_drift(self) -> None:
path = self.root / "linux/CMakeLists.txt" path = self.root / "linux/CMakeLists.txt"
path.write_text(path.read_text(encoding="utf-8").replace("9fe4d6f5", "0fe4d6f5"), encoding="utf-8") path.write_text(path.read_text(encoding="utf-8").replace("9fe4d6f5", "0fe4d6f5"), encoding="utf-8")
+13 -10
View File
@@ -3,6 +3,7 @@ import 'dart:io';
import 'package:crypto/crypto.dart'; import 'package:crypto/crypto.dart';
import 'package:flutter_test/flutter_test.dart'; import 'package:flutter_test/flutter_test.dart';
import 'package:path/path.dart' as path;
import '../../scripts/generate_ducet_ranks.dart'; import '../../scripts/generate_ducet_ranks.dart';
@@ -187,9 +188,9 @@ void main() {
test('a rejected second download preserves the first cache and output', () async { test('a rejected second download preserves the first cache and output', () async {
const allKeysText = '0041 ; [.0100.0020.0002]\n'; const allKeysText = '0041 ; [.0100.0020.0002]\n';
const fractionalText = '[radical 1=⼀一:一]\n'; const fractionalText = '[radical 1=⼀一:一]\n';
final cache = Directory('${temporaryDirectory.path}/cache')..createSync(); final cache = Directory(path.join(temporaryDirectory.path, 'cache'))..createSync();
final marker = File('${cache.path}/marker')..writeAsStringSync('keep'); final marker = File(path.join(cache.path, 'marker'))..writeAsStringSync('keep');
final output = File('${temporaryDirectory.path}/ducet_order.dart')..writeAsStringSync('old output'); final output = File(path.join(temporaryDirectory.path, 'ducet_order.dart'))..writeAsStringSync('old output');
var request = 0; var request = 0;
await expectLater( await expectLater(
@@ -208,7 +209,7 @@ void main() {
); );
expect(request, 2); expect(request, 2);
expect(cache.listSync().map((entity) => entity.path).toList(), [marker.path]); expect(cache.listSync().map((entity) => path.basename(entity.path)).toList(), ['marker']);
expect(marker.readAsStringSync(), 'keep'); expect(marker.readAsStringSync(), 'keep');
expect(output.readAsStringSync(), 'old output'); expect(output.readAsStringSync(), 'old output');
}); });
@@ -219,11 +220,12 @@ void main() {
0041 ; [.0101.0020.0002] 0041 ; [.0101.0020.0002]
'''; ''';
const fractionalText = '[radical 1=⼀一:一]\n'; const fractionalText = '[radical 1=⼀一:一]\n';
final allKeysFile = File('${temporaryDirectory.path}/allkeys.txt')..writeAsStringSync(duplicateAllKeys); final allKeysFile = File(path.join(temporaryDirectory.path, 'allkeys.txt'))..writeAsStringSync(duplicateAllKeys);
final fractionalFile = File('${temporaryDirectory.path}/FractionalUCA.txt')..writeAsStringSync(fractionalText); final fractionalFile = File(path.join(temporaryDirectory.path, 'FractionalUCA.txt'))
final cache = Directory('${temporaryDirectory.path}/cache')..createSync(); ..writeAsStringSync(fractionalText);
final marker = File('${cache.path}/marker')..writeAsStringSync('keep'); final cache = Directory(path.join(temporaryDirectory.path, 'cache'))..createSync();
final output = File('${temporaryDirectory.path}/ducet_order.dart')..writeAsStringSync('old output'); final marker = File(path.join(cache.path, 'marker'))..writeAsStringSync('keep');
final output = File(path.join(temporaryDirectory.path, 'ducet_order.dart'))..writeAsStringSync('old output');
await expectLater( await expectLater(
generateDucetRanks( generateDucetRanks(
@@ -240,7 +242,8 @@ void main() {
expect(allKeysFile.readAsStringSync(), duplicateAllKeys); expect(allKeysFile.readAsStringSync(), duplicateAllKeys);
expect(fractionalFile.readAsStringSync(), fractionalText); expect(fractionalFile.readAsStringSync(), fractionalText);
expect(cache.listSync().map((entity) => entity.path).toList(), [marker.path]); expect(cache.listSync().map((entity) => path.basename(entity.path)).toList(), ['marker']);
expect(marker.readAsStringSync(), 'keep');
expect(output.readAsStringSync(), 'old output'); expect(output.readAsStringSync(), 'old output');
}); });