diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 29e5cbc9..f19e810d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -15,11 +15,15 @@ concurrency: jobs: android-maestro: - name: Android basic, catalog, and codec flows + name: Android Maestro suites runs-on: ubuntu-latest - timeout-minutes: 65 + timeout-minutes: 150 permissions: contents: read + env: + FLUTTER_VERSION: "3.44.0" + MAESTRO_VERSION: "2.6.1" + MAESTRO_CLI_NO_ANALYTICS: "1" steps: - name: Checkout code @@ -42,7 +46,7 @@ jobs: uses: subosito/flutter-action@v2 with: channel: stable - flutter-version: "3.44.0" + flutter-version: ${{ env.FLUTTER_VERSION }} cache: true pub-cache: false @@ -55,7 +59,9 @@ jobs: - name: Cache Gradle uses: actions/cache@v4 with: - path: ~/.gradle/caches + path: | + ~/.gradle/caches + ~/.gradle/wrapper key: ${{ runner.os }}-gradle-e2e-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} restore-keys: | ${{ runner.os }}-gradle-e2e- @@ -64,33 +70,166 @@ jobs: - name: Install dependencies run: flutter pub get - - name: Install Maestro - run: | - curl -fsSL "https://get.maestro.mobile.dev" | bash - echo "$HOME/.maestro/bin" >> "$GITHUB_PATH" - - name: Build debug APK run: flutter build apk --debug - - name: Build deterministic Jellyfin image - run: python3 scripts/run_maestro.py build-image + - name: Cache Maestro CLI + id: maestro-cache + uses: actions/cache@v4 + with: + path: | + ~/.maestro/bin + ~/.maestro/lib + key: ${{ runner.os }}-maestro-${{ env.MAESTRO_VERSION }} - - name: Run Maestro suites + - name: Install Maestro CLI + if: steps.maestro-cache.outputs.cache-hit != 'true' + run: curl -fsSL "https://get.maestro.mobile.dev" | bash + + - name: Add Maestro to PATH + run: | + echo "$HOME/.maestro/bin" >> "$GITHUB_PATH" + "$HOME/.maestro/bin/maestro" --version + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Build and cache deterministic Jellyfin image + uses: docker/build-push-action@v7 + with: + context: . + file: .maestro/jellyfin-demo/Dockerfile + load: true + tags: plezy-jellyfin-demo:local + cache-from: type=gha,scope=maestro-jellyfin + cache-to: type=gha,mode=max,scope=maestro-jellyfin + + - name: Cache Android 15 AVD + id: api35-avd-cache + uses: actions/cache@v4 + with: + path: | + ~/.android/avd/maestro-api35.avd + ~/.android/avd/maestro-api35.ini + key: ${{ runner.os }}-avd-v1-api35-x86_64-pixel_6 + + - name: Generate Android 15 AVD snapshot + if: steps.api35-avd-cache.outputs.cache-hit != 'true' uses: reactivecircus/android-emulator-runner@v2 with: api-level: 35 arch: x86_64 profile: pixel_6 + avd-name: maestro-api35 + force-avd-creation: false + disable-animations: false + emulator-options: >- + -no-window -gpu swiftshader_indirect -noaudio + -no-boot-anim -camera-back none + script: echo "Generated Android 15 AVD snapshot." + + - name: Cache Android 9 AVD + id: api28-avd-cache + uses: actions/cache@v4 + with: + path: | + ~/.android/avd/maestro-api28.avd + ~/.android/avd/maestro-api28.ini + key: ${{ runner.os }}-avd-v1-api28-x86-pixel_2-playstore + + - name: Generate Android 9 AVD snapshot + if: steps.api28-avd-cache.outputs.cache-hit != 'true' + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 28 + target: google_apis_playstore + arch: x86 + profile: pixel_2 + avd-name: maestro-api28 + force-avd-creation: false + disable-animations: false + emulator-options: >- + -no-window -gpu swiftshader_indirect -noaudio + -no-boot-anim -camera-back none + script: echo "Generated Android 9 AVD snapshot." + + - name: Run Android 15 suites + id: api35 + continue-on-error: true + uses: reactivecircus/android-emulator-runner@v2 + env: + MAESTRO_SKIP_BUILD: "1" + MAESTRO_SKIP_JELLYFIN_BUILD: "1" + MAESTRO_DEVICE_ID: emulator-5554 + with: + api-level: 35 + arch: x86_64 + profile: pixel_6 + avd-name: maestro-api35 + force-avd-creation: false disable-animations: true emulator-options: >- - -no-window -gpu swiftshader_indirect -no-snapshot -noaudio + -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none script: | - python3 scripts/run_maestro.py basic --skip-build --skip-jellyfin-build - python3 scripts/run_maestro.py catalog \ - --skip-build --skip-jellyfin-build --device emulator-5554 - python3 scripts/run_maestro.py media \ - --skip-build --skip-jellyfin-build --device emulator-5554 + status=0 + run_flow() { + "$@" || status=1 + } + + run_flow python3 scripts/run_maestro.py basic + run_flow python3 scripts/run_maestro.py catalog + run_flow python3 scripts/run_maestro.py media + run_flow python3 scripts/run_maestro.py basic \ + --flow .maestro/regression_flows/03_tv_library_focus.yaml \ + --jellyfin-log build/maestro-tv/library-focus.log \ + --diagnostics-dir build/maestro-tv/library-focus-diagnostics + run_flow python3 scripts/run_maestro.py basic \ + --flow .maestro/regression_flows/04_tv_player_keys.yaml \ + --jellyfin-log build/maestro-tv/player-keys.log \ + --diagnostics-dir build/maestro-tv/player-keys-diagnostics + run_flow python3 scripts/run_maestro.py basic \ + --flow .maestro/regression_flows/05_tv_next_episode_back.yaml \ + --jellyfin-log build/maestro-tv/next-episode.log \ + --diagnostics-dir build/maestro-tv/next-episode-diagnostics + run_flow python3 scripts/run_maestro.py basic \ + --fault music-failure \ + --flow .maestro/real_flows/02_music_browse.yaml \ + --jellyfin-log build/maestro-recovery/music-jellyfin.log \ + --proxy-journal build/maestro-recovery/music-proxy-journal.jsonl \ + --diagnostics-dir build/maestro-recovery/music-diagnostics + run_flow python3 scripts/run_maestro.py basic \ + --fault recovery \ + --flow .maestro/regression_flows/06_playback_recovery.yaml \ + --jellyfin-log build/maestro-recovery/jellyfin.log \ + --proxy-journal build/maestro-recovery/proxy-journal.jsonl \ + --diagnostics-dir build/maestro-recovery/diagnostics + exit "$status" + + - name: Run Android 9 legacy playback + id: api28 + continue-on-error: true + uses: reactivecircus/android-emulator-runner@v2 + env: + MAESTRO_SKIP_BUILD: "1" + MAESTRO_SKIP_JELLYFIN_BUILD: "1" + MAESTRO_DEVICE_ID: emulator-5554 + with: + api-level: 28 + target: google_apis_playstore + arch: x86 + profile: pixel_2 + avd-name: maestro-api28 + force-avd-creation: false + disable-animations: true + emulator-options: >- + -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio + -no-boot-anim -camera-back none + script: | + python3 scripts/run_maestro.py basic \ + --flow .maestro/flows/05_playback.yaml \ + --jellyfin-log build/maestro-legacy/jellyfin.log \ + --diagnostics-dir build/maestro-legacy/diagnostics - name: Upload Maestro diagnostics if: always() @@ -101,299 +240,16 @@ jobs: build/maestro build/maestro-real-jellyfin build/maestro-media - ~/.maestro/tests - if-no-files-found: ignore - retention-days: 7 - - android-legacy-playback: - name: Android 9 / API 28 Play Store playback - runs-on: ubuntu-latest - timeout-minutes: 35 - permissions: - contents: read - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Enable KVM - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ - | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - - name: Setup Java - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: "17" - - - name: Setup Flutter - uses: subosito/flutter-action@v2 - with: - channel: stable - flutter-version: "3.44.0" - cache: true - pub-cache: false - - - name: Cache Pub dependencies - uses: actions/cache@v4 - with: - path: ~/.pub-cache - key: ${{ runner.os }}-pub-v3-${{ hashFiles('**/pubspec.yaml', '**/pubspec.lock') }} - - - name: Cache Gradle - uses: actions/cache@v4 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-e2e-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle-e2e- - ${{ runner.os }}-gradle- - - - name: Install dependencies - run: flutter pub get - - - name: Install Maestro - run: | - curl -fsSL "https://get.maestro.mobile.dev" | bash - echo "$HOME/.maestro/bin" >> "$GITHUB_PATH" - - - name: Build debug APK - run: flutter build apk --debug - - - name: Build deterministic Jellyfin image - run: python3 scripts/run_maestro.py build-image - - - name: Run legacy playback flow - uses: reactivecircus/android-emulator-runner@v2 - with: - api-level: 28 - target: google_apis_playstore - arch: x86 - profile: pixel_2 - disable-animations: true - emulator-options: >- - -no-window -gpu swiftshader_indirect -no-snapshot -noaudio - -no-boot-anim -camera-back none - script: | - python3 scripts/run_maestro.py basic \ - --skip-build --skip-jellyfin-build \ - --device emulator-5554 \ - --flow .maestro/flows/05_playback.yaml \ - --jellyfin-log build/maestro-legacy/jellyfin.log \ - --diagnostics-dir build/maestro-legacy/diagnostics - - - name: Upload legacy playback diagnostics - if: always() - uses: actions/upload-artifact@v4 - with: - name: maestro-android-legacy-diagnostics - path: | + build/maestro-tv + build/maestro-recovery build/maestro-legacy ~/.maestro/tests if-no-files-found: ignore retention-days: 7 - android-tv-regressions: - name: Android TV and D-pad regressions - runs-on: ubuntu-latest - timeout-minutes: 60 - permissions: - contents: read - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Enable KVM + - name: Report suite failures + if: ${{ always() && (steps.api35.outcome != 'success' || steps.api28.outcome != 'success') }} run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ - | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - - name: Setup Java - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: "17" - - - name: Setup Flutter - uses: subosito/flutter-action@v2 - with: - channel: stable - flutter-version: "3.44.0" - cache: true - pub-cache: false - - - name: Cache Pub dependencies - uses: actions/cache@v4 - with: - path: ~/.pub-cache - key: ${{ runner.os }}-pub-v3-${{ hashFiles('**/pubspec.yaml', '**/pubspec.lock') }} - - - name: Cache Gradle - uses: actions/cache@v4 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-e2e-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle-e2e- - ${{ runner.os }}-gradle- - - - name: Install dependencies - run: flutter pub get - - - name: Install Maestro - run: | - curl -fsSL "https://get.maestro.mobile.dev" | bash - echo "$HOME/.maestro/bin" >> "$GITHUB_PATH" - - - name: Build debug APK - run: flutter build apk --debug - - - name: Build deterministic Jellyfin image - run: python3 scripts/run_maestro.py build-image - - - name: Run TV regression flows - uses: reactivecircus/android-emulator-runner@v2 - with: - api-level: 35 - arch: x86_64 - profile: pixel_6 - disable-animations: true - emulator-options: >- - -no-window -gpu swiftshader_indirect -no-snapshot -noaudio - -no-boot-anim -camera-back none - script: | - python3 scripts/run_maestro.py basic \ - --skip-build --skip-jellyfin-build \ - --device emulator-5554 \ - --flow .maestro/regression_flows/03_tv_library_focus.yaml \ - --jellyfin-log build/maestro-tv/library-focus.log \ - --diagnostics-dir build/maestro-tv/library-focus-diagnostics - python3 scripts/run_maestro.py basic \ - --skip-build --skip-jellyfin-build \ - --device emulator-5554 \ - --flow .maestro/regression_flows/04_tv_player_keys.yaml \ - --jellyfin-log build/maestro-tv/player-keys.log \ - --diagnostics-dir build/maestro-tv/player-keys-diagnostics - python3 scripts/run_maestro.py basic \ - --skip-build --skip-jellyfin-build \ - --device emulator-5554 \ - --flow .maestro/regression_flows/05_tv_next_episode_back.yaml \ - --jellyfin-log build/maestro-tv/next-episode.log \ - --diagnostics-dir build/maestro-tv/next-episode-diagnostics - - - name: Upload TV regression diagnostics - if: always() - uses: actions/upload-artifact@v4 - with: - name: maestro-android-tv-diagnostics - path: | - build/maestro-tv - ~/.maestro/tests - if-no-files-found: ignore - retention-days: 7 - - android-playback-recovery: - name: Android playback recovery - runs-on: ubuntu-latest - timeout-minutes: 40 - permissions: - contents: read - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Enable KVM - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ - | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - - name: Setup Java - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: "17" - - - name: Setup Flutter - uses: subosito/flutter-action@v2 - with: - channel: stable - flutter-version: "3.44.0" - cache: true - pub-cache: false - - - name: Cache Pub dependencies - uses: actions/cache@v4 - with: - path: ~/.pub-cache - key: ${{ runner.os }}-pub-v3-${{ hashFiles('**/pubspec.yaml', '**/pubspec.lock') }} - - - name: Cache Gradle - uses: actions/cache@v4 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-e2e-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle-e2e- - ${{ runner.os }}-gradle- - - - name: Install dependencies - run: flutter pub get - - - name: Install Maestro - run: | - curl -fsSL "https://get.maestro.mobile.dev" | bash - echo "$HOME/.maestro/bin" >> "$GITHUB_PATH" - - - name: Build debug APK - run: flutter build apk --debug - - - name: Build deterministic Jellyfin image - run: python3 scripts/run_maestro.py build-image - - - name: Run media recovery flows - uses: reactivecircus/android-emulator-runner@v2 - with: - api-level: 35 - arch: x86_64 - profile: pixel_6 - disable-animations: true - emulator-options: >- - -no-window -gpu swiftshader_indirect -no-snapshot -noaudio - -no-boot-anim -camera-back none - script: | - python3 scripts/run_maestro.py basic \ - --skip-build --skip-jellyfin-build \ - --device emulator-5554 \ - --fault music-failure \ - --flow .maestro/real_flows/02_music_browse.yaml \ - --jellyfin-log build/maestro-recovery/music-jellyfin.log \ - --proxy-journal build/maestro-recovery/music-proxy-journal.jsonl \ - --diagnostics-dir build/maestro-recovery/music-diagnostics - python3 scripts/run_maestro.py basic \ - --skip-build --skip-jellyfin-build \ - --device emulator-5554 \ - --fault recovery \ - --flow .maestro/regression_flows/06_playback_recovery.yaml \ - --jellyfin-log build/maestro-recovery/jellyfin.log \ - --proxy-journal build/maestro-recovery/proxy-journal.jsonl \ - --diagnostics-dir build/maestro-recovery/diagnostics - - - name: Upload recovery diagnostics - if: always() - uses: actions/upload-artifact@v4 - with: - name: maestro-android-recovery-diagnostics - path: | - build/maestro-recovery - ~/.maestro/tests - if-no-files-found: ignore - retention-days: 7 + echo "Android 15 outcome: ${{ steps.api35.outcome }}" + echo "Android 9 outcome: ${{ steps.api28.outcome }}" + exit 1