diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f6a974e..be2c04d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,9 @@ jobs: - name: Verify workflow and script guards run: | python3 scripts/check_build_workflow.py + python3 scripts/check_workflow_security.py + python3 scripts/test_check_workflow_security.py + python3 scripts/test_run_maestro.py python3 scripts/check_update_packages_workflow.py python3 scripts/test_pubspec_version.py python3 scripts/test_clean_translations.py diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 0f954436..438aa3b8 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -199,40 +199,7 @@ jobs: emulator-options: >- -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none - script: | - status=0 - run_flow() { - "$@" || status=1 - } - - run_flow python3 scripts/run_maestro.py basic - run_flow python3 scripts/run_maestro.py catalog - run_flow python3 scripts/run_maestro.py media - run_flow python3 scripts/run_maestro.py basic \ - --flow .maestro/regression_flows/03_tv_library_focus.yaml \ - --jellyfin-log build/maestro-tv/library-focus.log \ - --diagnostics-dir build/maestro-tv/library-focus-diagnostics - run_flow python3 scripts/run_maestro.py basic \ - --flow .maestro/regression_flows/04_tv_player_keys.yaml \ - --jellyfin-log build/maestro-tv/player-keys.log \ - --diagnostics-dir build/maestro-tv/player-keys-diagnostics - run_flow python3 scripts/run_maestro.py basic \ - --flow .maestro/regression_flows/05_tv_next_episode_back.yaml \ - --jellyfin-log build/maestro-tv/next-episode.log \ - --diagnostics-dir build/maestro-tv/next-episode-diagnostics - run_flow python3 scripts/run_maestro.py basic \ - --fault music-failure \ - --flow .maestro/real_flows/02_music_browse.yaml \ - --jellyfin-log build/maestro-recovery/music-jellyfin.log \ - --proxy-journal build/maestro-recovery/music-proxy-journal.jsonl \ - --diagnostics-dir build/maestro-recovery/music-diagnostics - run_flow python3 scripts/run_maestro.py basic \ - --fault recovery \ - --flow .maestro/regression_flows/06_playback_recovery.yaml \ - --jellyfin-log build/maestro-recovery/jellyfin.log \ - --proxy-journal build/maestro-recovery/proxy-journal.jsonl \ - --diagnostics-dir build/maestro-recovery/diagnostics - exit "$status" + script: python3 scripts/run_maestro_ci.py android-15 - name: Run Android 9 legacy playback id: api28 @@ -253,11 +220,7 @@ jobs: emulator-options: >- -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none - script: | - python3 scripts/run_maestro.py basic \ - --flow .maestro/flows/05_playback.yaml \ - --jellyfin-log build/maestro-legacy/jellyfin.log \ - --diagnostics-dir build/maestro-legacy/diagnostics + script: python3 scripts/run_maestro_ci.py android-9 # Fork PRs may restore trusted caches, but only trusted events publish cache entries. - name: Save Pub dependency cache if: github.event_name != 'pull_request' && steps.pub-cache.outputs.cache-hit != 'true' diff --git a/scripts/run_maestro_ci.py b/scripts/run_maestro_ci.py new file mode 100755 index 00000000..7997e325 --- /dev/null +++ b/scripts/run_maestro_ci.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Run the Maestro groups assigned to each CI emulator.""" + +from __future__ import annotations + +import argparse +from collections.abc import Sequence + +import run_maestro + + +GROUPS: dict[str, tuple[tuple[str, ...], ...]] = { + "android-15": ( + ("basic",), + ("catalog",), + ("media",), + ( + "basic", + "--flow", + ".maestro/regression_flows/03_tv_library_focus.yaml", + "--jellyfin-log", + "build/maestro-tv/library-focus.log", + "--diagnostics-dir", + "build/maestro-tv/library-focus-diagnostics", + ), + ( + "basic", + "--flow", + ".maestro/regression_flows/04_tv_player_keys.yaml", + "--jellyfin-log", + "build/maestro-tv/player-keys.log", + "--diagnostics-dir", + "build/maestro-tv/player-keys-diagnostics", + ), + ( + "basic", + "--flow", + ".maestro/regression_flows/05_tv_next_episode_back.yaml", + "--jellyfin-log", + "build/maestro-tv/next-episode.log", + "--diagnostics-dir", + "build/maestro-tv/next-episode-diagnostics", + ), + ( + "basic", + "--fault", + "music-failure", + "--flow", + ".maestro/real_flows/02_music_browse.yaml", + "--jellyfin-log", + "build/maestro-recovery/music-jellyfin.log", + "--proxy-journal", + "build/maestro-recovery/music-proxy-journal.jsonl", + "--diagnostics-dir", + "build/maestro-recovery/music-diagnostics", + ), + ( + "basic", + "--fault", + "recovery", + "--flow", + ".maestro/regression_flows/06_playback_recovery.yaml", + "--jellyfin-log", + "build/maestro-recovery/jellyfin.log", + "--proxy-journal", + "build/maestro-recovery/proxy-journal.jsonl", + "--diagnostics-dir", + "build/maestro-recovery/diagnostics", + ), + ), + "android-9": ( + ( + "basic", + "--flow", + ".maestro/flows/05_playback.yaml", + "--jellyfin-log", + "build/maestro-legacy/jellyfin.log", + "--diagnostics-dir", + "build/maestro-legacy/diagnostics", + ), + ), +} + + +def run_group(name: str) -> int: + failed = False + for arguments in GROUPS[name]: + print(f"==> Maestro {' '.join(arguments)}", flush=True) + exit_status = run_maestro.main(arguments) + if exit_status >= 128: + return exit_status + failed = failed or exit_status != 0 + return 1 if failed else 0 + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("group", choices=GROUPS) + args = parser.parse_args(argv) + return run_group(args.group) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_run_maestro.py b/scripts/test_run_maestro.py index 67d75725..501a5d16 100755 --- a/scripts/test_run_maestro.py +++ b/scripts/test_run_maestro.py @@ -2,7 +2,7 @@ from __future__ import annotations -from contextlib import redirect_stderr +from contextlib import redirect_stderr, redirect_stdout from dataclasses import replace import io from pathlib import Path @@ -14,6 +14,7 @@ from unittest.mock import patch sys.path.insert(0, str(Path(__file__).resolve().parent)) import run_maestro # noqa: E402 +import run_maestro_ci # noqa: E402 class ParseConfigTests(unittest.TestCase): @@ -134,5 +135,36 @@ class LifecycleTests(unittest.TestCase): sleep.assert_called_once_with(5) +class CiGroupTests(unittest.TestCase): + def test_android_15_group_runs_every_suite_after_failure(self) -> None: + expected_runs = len(run_maestro_ci.GROUPS["android-15"]) + statuses = [0, 1, *([0] * (expected_runs - 2))] + + with ( + patch.object(run_maestro_ci.run_maestro, "main", side_effect=statuses) as run, + redirect_stdout(io.StringIO()), + ): + exit_status = run_maestro_ci.run_group("android-15") + + self.assertEqual(exit_status, 1) + self.assertEqual(run.call_count, expected_runs) + + def test_group_recipes_are_valid_runner_invocations(self) -> None: + for recipes in run_maestro_ci.GROUPS.values(): + for arguments in recipes: + with self.subTest(arguments=arguments): + run_maestro.parse_config(arguments, {}) + + def test_group_stops_after_interruption(self) -> None: + with ( + patch.object(run_maestro_ci.run_maestro, "main", return_value=143) as run, + redirect_stdout(io.StringIO()), + ): + exit_status = run_maestro_ci.run_group("android-15") + + self.assertEqual(exit_status, 143) + run.assert_called_once_with(("basic",)) + + if __name__ == "__main__": unittest.main()