diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml deleted file mode 100644 index 483e7e79..00000000 --- a/.github/workflows/e2e.yml +++ /dev/null @@ -1,324 +0,0 @@ -name: E2E - Maestro - -on: - push: - branches: - - main - # Keep untrusted code on the read-only pull_request event. Never use pull_request_target here. - pull_request: - branches: - - main - workflow_dispatch: - -concurrency: - group: maestro-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - android-maestro: - name: Android Maestro suites - runs-on: ubuntu-latest - timeout-minutes: 150 - permissions: - contents: read - env: - FLUTTER_VERSION: "3.44.0" - MAESTRO_VERSION: "2.6.1" - MAESTRO_SHA256: "3440825f514f537c6a96bcf5de995780c2a4a7f83a43208fdc95d4f1fecfad3b" - MAESTRO_CLI_NO_ANALYTICS: "1" - - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - persist-credentials: false - - - name: Enable KVM - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ - | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - - name: Setup Java - uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5 - with: - distribution: temurin - java-version: "21" - - - name: Setup Flutter - uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 # v2 - with: - channel: stable - flutter-version: ${{ env.FLUTTER_VERSION }} - cache: true - pub-cache: false - - - name: Cache Pub dependencies - id: pub-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: ~/.pub-cache - key: ${{ runner.os }}-pub-v3-${{ hashFiles('**/pubspec.yaml', '**/pubspec.lock') }} - - - name: Cache Gradle - id: gradle-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-e2e-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle-e2e- - ${{ runner.os }}-gradle- - - - name: Install dependencies - run: flutter pub get - - - name: Build debug APK - run: flutter build apk --debug --dart-define=PLEZY_MAESTRO_E2E=true - - - name: Cache Maestro CLI - id: maestro-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: | - ~/.maestro/bin - ~/.maestro/lib - key: ${{ runner.os }}-maestro-${{ env.MAESTRO_VERSION }} - - - name: Install verified Maestro CLI - if: steps.maestro-cache.outputs.cache-hit != 'true' - run: | - archive="$RUNNER_TEMP/maestro.zip" - staging="$RUNNER_TEMP/maestro-install" - curl --fail --location --retry 3 \ - --output "$archive" \ - "https://github.com/mobile-dev-inc/maestro/releases/download/cli-${MAESTRO_VERSION}/maestro.zip" - echo "${MAESTRO_SHA256} ${archive}" | sha256sum --check - - rm -rf "$staging" "$HOME/.maestro/bin" "$HOME/.maestro/lib" - mkdir -p "$staging" "$HOME/.maestro" - unzip -q "$archive" -d "$staging" - mv "$staging/maestro/bin" "$HOME/.maestro/bin" - mv "$staging/maestro/lib" "$HOME/.maestro/lib" - - - name: Add Maestro to PATH - run: | - echo "$HOME/.maestro/bin" >> "$GITHUB_PATH" - "$HOME/.maestro/bin/maestro" --version - - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 - - - name: Build deterministic Jellyfin image for pull request - if: github.event_name == 'pull_request' - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 - with: - context: . - file: .maestro/jellyfin-demo/Dockerfile - load: true - tags: plezy-jellyfin-demo:local - cache-from: type=gha,scope=maestro-jellyfin - - - name: Build and cache deterministic Jellyfin image - if: github.event_name != 'pull_request' - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 - with: - context: . - file: .maestro/jellyfin-demo/Dockerfile - load: true - tags: plezy-jellyfin-demo:local - cache-from: type=gha,scope=maestro-jellyfin - cache-to: type=gha,mode=max,scope=maestro-jellyfin - - - name: Cache Android 15 AVD - id: api35-avd-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: | - ~/.android/avd/maestro-api35.avd - ~/.android/avd/maestro-api35.ini - key: ${{ runner.os }}-avd-v1-api35-x86_64-pixel_6 - - - name: Generate Android 15 AVD snapshot - if: steps.api35-avd-cache.outputs.cache-hit != 'true' - uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2 - with: - api-level: 35 - arch: x86_64 - profile: pixel_6 - avd-name: maestro-api35 - force-avd-creation: false - disable-animations: false - emulator-options: >- - -no-window -gpu swiftshader_indirect -noaudio - -no-boot-anim -camera-back none - script: echo "Generated Android 15 AVD snapshot." - - - name: Cache Android 9 AVD - id: api28-avd-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: | - ~/.android/avd/maestro-api28.avd - ~/.android/avd/maestro-api28.ini - key: ${{ runner.os }}-avd-v1-api28-x86-pixel_2-playstore - - - name: Generate Android 9 AVD snapshot - if: steps.api28-avd-cache.outputs.cache-hit != 'true' - uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2 - with: - api-level: 28 - target: google_apis_playstore - arch: x86 - profile: pixel_2 - avd-name: maestro-api28 - force-avd-creation: false - disable-animations: false - emulator-options: >- - -no-window -gpu swiftshader_indirect -noaudio - -no-boot-anim -camera-back none - script: echo "Generated Android 9 AVD snapshot." - - - name: Run Android 15 instrumentation - id: api35-instrumentation - continue-on-error: true - uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2 - with: - api-level: 35 - arch: x86_64 - profile: pixel_6 - avd-name: maestro-api35-instrumentation - force-avd-creation: false - disable-animations: true - emulator-options: >- - -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio - -no-boot-anim -camera-back none - script: python3 scripts/run_maestro_ci.py android-15-instrumentation - - - name: Run Android R8 reachability - id: api35-r8 - continue-on-error: true - uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2 - with: - api-level: 35 - arch: x86_64 - profile: pixel_6 - avd-name: maestro-api35-instrumentation - force-avd-creation: false - disable-animations: true - emulator-options: >- - -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio - -no-boot-anim -camera-back none - script: python3 scripts/run_maestro_ci.py android-r8-reachability - - - name: Run Android 15 suites - id: api35 - continue-on-error: true - uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2 - env: - MAESTRO_SKIP_BUILD: "1" - MAESTRO_SKIP_JELLYFIN_BUILD: "1" - MAESTRO_DEVICE_ID: emulator-5554 - with: - api-level: 35 - arch: x86_64 - profile: pixel_6 - avd-name: maestro-api35 - force-avd-creation: false - disable-animations: true - emulator-options: >- - -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio - -no-boot-anim -camera-back none - script: python3 scripts/run_maestro_ci.py android-15 - - - name: Run Android 9 legacy playback - id: api28 - continue-on-error: true - uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2 - env: - MAESTRO_SKIP_BUILD: "1" - MAESTRO_SKIP_JELLYFIN_BUILD: "1" - MAESTRO_DEVICE_ID: emulator-5554 - with: - api-level: 28 - target: google_apis_playstore - arch: x86 - profile: pixel_2 - avd-name: maestro-api28 - force-avd-creation: false - disable-animations: true - emulator-options: >- - -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio - -no-boot-anim -camera-back none - script: python3 scripts/run_maestro_ci.py android-9 - # Fork PRs may restore trusted caches, but only trusted events publish cache entries. - - name: Save Pub dependency cache - if: github.event_name != 'pull_request' && steps.pub-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: ~/.pub-cache - key: ${{ steps.pub-cache.outputs.cache-primary-key }} - - - name: Save Gradle cache - if: github.event_name != 'pull_request' && steps.gradle-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ steps.gradle-cache.outputs.cache-primary-key }} - - - name: Save Maestro CLI cache - if: github.event_name != 'pull_request' && steps.maestro-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: | - ~/.maestro/bin - ~/.maestro/lib - key: ${{ steps.maestro-cache.outputs.cache-primary-key }} - - - name: Save Android 15 AVD cache - if: github.event_name != 'pull_request' && steps.api35-avd-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: | - ~/.android/avd/maestro-api35.avd - ~/.android/avd/maestro-api35.ini - key: ${{ steps.api35-avd-cache.outputs.cache-primary-key }} - - - name: Save Android 9 AVD cache - if: github.event_name != 'pull_request' && steps.api28-avd-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: | - ~/.android/avd/maestro-api28.avd - ~/.android/avd/maestro-api28.ini - key: ${{ steps.api28-avd-cache.outputs.cache-primary-key }} - - - name: Upload Maestro diagnostics - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: maestro-android-diagnostics - path: | - build/maestro - build/maestro-offline - build/maestro-real-jellyfin - build/maestro-media - build/maestro-tv - build/maestro-recovery - build/maestro-legacy - ~/.maestro/tests - if-no-files-found: ignore - retention-days: 7 - - - name: Report suite failures - if: ${{ always() && (steps.api35-instrumentation.outcome != 'success' || steps.api35-r8.outcome != 'success' || steps.api35.outcome != 'success' || steps.api28.outcome != 'success') }} - run: | - echo "Android 15 instrumentation outcome: ${{ steps.api35-instrumentation.outcome }}" - echo "Android R8 reachability outcome: ${{ steps.api35-r8.outcome }}" - echo "Android 15 outcome: ${{ steps.api35.outcome }}" - echo "Android 9 outcome: ${{ steps.api28.outcome }}" - exit 1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0c1bbf8c..3d807248 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,13 +66,12 @@ Use `--skip-build` to reuse the debug APK and `--skip-jellyfin-build` to reuse t `--device ` when multiple devices are connected; physical devices also require `--adb-reverse`. Top-level flows live in `.maestro/flows/`, shared setup in `.maestro/subflows/`, and focused regressions in -`.maestro/regression_flows/`. Automatic PR groups are declared in `scripts/run_maestro_ci.py::GROUPS`. Every top-level +`.maestro/regression_flows/`. Automatic groups are declared in `scripts/run_maestro_ci.py::GROUPS`. Every top-level regression flow must be registered either there or in `DESTRUCTIVE_MANUAL_TARGETS`; reusable subflows are not -independent tests. A manual-only classification must state why the flow cannot run automatically and must not be -described as CI coverage. `.github/workflows/e2e.yml` runs only automatic targets and uploads diagnostics on failure. +independent tests. A manual-only classification must state why the flow cannot run automatically. The profile-isolation and profile-teardown regressions create and remove profile connections, so they are a destructive -manual target rather than automatic PR coverage. Run them only against the pre-seeded Jellyfin fixture and a disposable +manual target rather than an automatic group. Run them only against the pre-seeded Jellyfin fixture and a disposable emulator, using the required opt-in: ```bash diff --git a/scripts/test_run_maestro.py b/scripts/test_run_maestro.py index 6e86b65f..4fbc332e 100755 --- a/scripts/test_run_maestro.py +++ b/scripts/test_run_maestro.py @@ -27,7 +27,6 @@ CI_SCRIPT_TEST_DISPATCHER = ROOT_DIR / ".github/workflows/ci.yml" # The guard roster both dispatchers above delegate to. It discovers the script # tests by glob, so a new scripts/test_*.py is picked up without being listed. GUARD_SCRIPT_TEST_DISPATCHER = SCRIPTS_DIR / "ci_guard_checks.sh" -E2E_WORKFLOW = ROOT_DIR / ".github/workflows/e2e.yml" SCRIPT_TEST_DISPATCHERS = ( LOCAL_SCRIPT_TEST_DISPATCHER, CI_SCRIPT_TEST_DISPATCHER, @@ -90,19 +89,6 @@ def _dispatched_script_tests(path: Path) -> list[str]: return dispatched -def _dispatched_maestro_ci_targets(path: Path) -> set[str]: - targets = set() - for line in path.read_text(encoding="utf-8").splitlines(): - try: - command = shlex.split(line.strip(), comments=True) - except ValueError: - continue - for index, token in enumerate(command[:-1]): - if Path(token).name == "run_maestro_ci.py": - targets.add(command[index + 1]) - return targets - - def _registered_regression_flows( targets: dict[str, tuple[tuple[str, ...], ...]], ) -> set[str]: @@ -372,15 +358,6 @@ class CiGroupTests(unittest.TestCase): ) ) - def test_destructive_manual_targets_are_not_automatic_pr_targets(self) -> None: - automatic_pr_targets = _dispatched_maestro_ci_targets(E2E_WORKFLOW) - - self.assertTrue( - set(run_maestro_ci.DESTRUCTIVE_MANUAL_TARGETS).isdisjoint( - automatic_pr_targets - ) - ) - def test_destructive_manual_recipes_have_distinct_diagnostics(self) -> None: for target, recipes in run_maestro_ci.DESTRUCTIVE_MANUAL_TARGETS.items(): with self.subTest(target=target):