import 'dart:async'; import 'dart:convert'; import 'package:flutter_test/flutter_test.dart'; import 'package:http/http.dart' as http; import 'package:http/testing.dart'; import 'package:plezy/exceptions/media_server_exceptions.dart'; import 'package:plezy/utils/app_logger.dart'; import 'package:plezy/utils/failover_http_client.dart'; import 'package:plezy/utils/media_server_http_client.dart'; import 'package:plezy/utils/log_redaction_manager.dart'; /// Pins the shared failover semantics both backends now ride on (see the /// class doc): GET-only single-step cascades, generation stamping, two-phase /// persistence, and exhaustion behavior. Backend-level coverage lives in /// jellyfin_client_failures_test.dart's failover group. void main() { setUp(() { MemoryLogOutput.clearLogs(); LogRedactionManager.clearTrackedValues(); setLoggerLevel(false); }); tearDown(() { MemoryLogOutput.clearLogs(); LogRedactionManager.clearTrackedValues(); setLoggerLevel(true); }); const primary = 'https://primary.example.com'; const fallback = 'https://fallback.example.com'; const tertiary = 'https://tertiary.example.com'; http.Response ok([String id = 'ok']) => http.Response(jsonEncode({'id': id}), 200, headers: {'content-type': 'application/json'}); ({FailoverHttpClient client, List<({String url, bool persist})> switches, List exhausted, List requests}) build({ required Future Function(http.Request request, List seen) handler, List endpoints = const [primary, fallback], Future Function(String candidateBaseUrl, AbortController? abort)? validateCandidate, }) { final switches = <({String url, bool persist})>[]; final exhausted = []; final requests = []; late FailoverHttpClient client; client = FailoverHttpClient( baseUrl: endpoints.isEmpty ? primary : endpoints.first, defaultHeaders: const {}, logLabel: 'Test', prioritizedEndpoints: endpoints, client: MockClient((request) { requests.add(request.url); return handler(request, requests); }), onEndpointSwitch: (newBaseUrl, {required persist}) async { switches.add((url: newBaseUrl, persist: persist)); // Mirror the real adapters: the callback owns applying the switch. client.baseUrl = newBaseUrl; }, onAllEndpointsExhausted: () => exhausted.add('x'), validateCandidate: validateCandidate, ); addTearDown(client.close); return (client: client, switches: switches, exhausted: exhausted, requests: requests); } test('validated transient failover switches once and persists the winner', () async { final validations = []; final h = build( handler: (request, _) async { if (request.url.host == 'primary.example.com') throw TimeoutException('down'); return ok(); }, validateCandidate: (candidateBaseUrl, _) async { validations.add(candidateBaseUrl); return true; }, ); final response = await h.client.get('/path'); expect(response.statusCode, 200); expect(validations, [fallback]); expect(h.requests.map((u) => u.host), ['primary.example.com', 'fallback.example.com']); expect(h.switches, [(url: fallback, persist: false), (url: fallback, persist: true)]); expect(h.exhausted, isEmpty); expect(h.client.baseUrl, fallback); }); test('5xx response (not thrown) also triggers the cascade', () async { final h = build( handler: (request, _) async { if (request.url.host == 'primary.example.com') return http.Response('boom', 503); return ok(); }, ); final response = await h.client.get('/path'); expect(response.statusCode, 200); expect(h.switches.last.persist, isTrue); }); test('rejected candidate surfaces the original response without switching', () async { final h = build( handler: (request, _) async { expect(request.url.host, 'primary.example.com'); return http.Response('primary unavailable', 503); }, validateCandidate: (_, _) async => false, ); final response = await h.client.get('/path'); expect(response.statusCode, 503); expect(h.requests.map((uri) => uri.host), ['primary.example.com']); expect(h.switches, isEmpty); expect(h.exhausted, hasLength(1)); expect(h.client.baseUrl, primary); }); test('rejected candidate is skipped before the single authenticated retry', () async { final validations = []; final h = build( endpoints: const [primary, fallback, tertiary], handler: (request, _) async { if (request.url.host == 'primary.example.com') { return http.Response('primary unavailable', 503); } return ok(request.url.host); }, validateCandidate: (candidateBaseUrl, _) async { validations.add(candidateBaseUrl); return candidateBaseUrl == tertiary; }, ); final response = await h.client.get('/path'); expect(response.statusCode, 200); expect(response.data, {'id': 'tertiary.example.com'}); expect(validations, [fallback, tertiary]); expect(h.requests.map((uri) => uri.host), ['primary.example.com', 'tertiary.example.com']); expect(h.switches, [(url: tertiary, persist: false), (url: tertiary, persist: true)]); expect(h.exhausted, isEmpty); expect(h.client.baseUrl, tertiary); }); test('throwing candidate validator surfaces the original transport failure', () async { final h = build( handler: (request, _) async { expect(request.url.host, 'primary.example.com'); throw TimeoutException('primary unavailable'); }, validateCandidate: (_, _) async => throw StateError('probe failed'), ); await expectLater( h.client.get('/path'), throwsA(isA().having((error) => error.isTransient, 'isTransient', isTrue)), ); expect(h.requests.map((uri) => uri.host), ['primary.example.com']); expect(h.switches, isEmpty); expect(h.exhausted, hasLength(1)); expect(h.client.baseUrl, primary); }); test('switch diagnostics contain no endpoint host or base-path literals', () async { const primaryCanary = 'https://primary-canary.invalid/private-primary-path'; const fallbackCanary = 'https://fallback-canary.invalid/private-fallback-path'; final h = build( endpoints: const [primaryCanary, fallbackCanary], handler: (request, _) async { if (request.url.host == 'primary-canary.invalid') throw TimeoutException('down'); return ok(); }, ); final response = await h.client.get('/resource'); final storedFields = MemoryLogOutput.getLogs().expand( (entry) => [entry.message, if (entry.error != null) entry.error.toString()], ); expect(response.statusCode, 200); expect(h.requests.map((uri) => uri.host), ['primary-canary.invalid', 'fallback-canary.invalid']); expect(h.switches, [(url: fallbackCanary, persist: false), (url: fallbackCanary, persist: true)]); expect(h.client.baseUrl, fallbackCanary); for (final field in storedFields) { expect(field, isNot(contains('primary-canary.invalid'))); expect(field, isNot(contains('private-primary-path'))); expect(field, isNot(contains('fallback-canary.invalid'))); expect(field, isNot(contains('private-fallback-path'))); } }); test('4xx answers never fail over', () async { final h = build(handler: (request, _) async => http.Response('nope', 404)); final response = await h.client.get('/path'); expect(response.statusCode, 404); expect(h.requests, hasLength(1)); expect(h.switches, isEmpty); expect(h.exhausted, isEmpty); }); test('allowEndpointFailover: false rethrows without switching', () async { final h = build(handler: (request, _) async => throw TimeoutException('down')); await expectLater( h.client.get('/path', allowEndpointFailover: false), throwsA(isA().having((e) => e.isTransient, 'isTransient', isTrue)), ); expect(h.requests, hasLength(1)); expect(h.switches, isEmpty); expect(h.exhausted, isEmpty); }); test('exhaustion resets to preferred, fires the callback, rethrows the retry failure', () async { final h = build(handler: (request, _) async => throw TimeoutException('all down')); await expectLater(h.client.get('/path'), throwsA(isA())); expect(h.requests.map((u) => u.host), ['primary.example.com', 'fallback.example.com']); // Switch to fallback for the retry, then reset back to preferred. expect(h.switches, [(url: fallback, persist: false), (url: primary, persist: false)]); expect(h.exhausted, hasLength(1)); expect(h.client.baseUrl, primary); }); test('retry answering 5xx counts as exhaustion and returns the response', () async { final h = build( handler: (request, _) async => request.url.host == 'primary.example.com' ? http.Response('boom', 500) : http.Response('also boom', 502), ); final response = await h.client.get('/path'); expect(response.statusCode, 502); expect(h.switches, [(url: fallback, persist: false), (url: primary, persist: false)]); expect(h.exhausted, hasLength(1)); }); test('single endpoint still arms the exhausted callback', () async { final h = build(endpoints: const [primary], handler: (request, _) async => throw TimeoutException('down')); await expectLater(h.client.get('/path'), throwsA(isA())); expect(h.requests, hasLength(1)); expect(h.switches, isEmpty); // resetToFirst is a no-op at index 0 expect(h.exhausted, hasLength(1)); }); test('no endpoints disables failover and the exhausted callback', () async { final h = build(endpoints: const [], handler: (request, _) async => throw TimeoutException('down')); await expectLater(h.client.get('/path'), throwsA(isA())); expect(h.switches, isEmpty); expect(h.exhausted, isEmpty); }); test('a request raced by a switch does not cascade again', () async { final firstRequestGate = Completer(); var primaryHits = 0; final h = build( handler: (request, _) async { if (request.url.host == 'primary.example.com') { primaryHits++; if (primaryHits == 1) { // Request A: hang until request B's cascade has completed. await firstRequestGate.future; } throw TimeoutException('down'); } return ok(); }, ); final requestA = h.client.get('/a'); final responseB = await h.client.get('/b'); expect(responseB.statusCode, 200); firstRequestGate.complete(); // A fails after B already switched the active endpoint (generation moved): // it must rethrow instead of starting a second cascade. await expectLater(requestA, throwsA(isA())); expect(h.switches, [(url: fallback, persist: false), (url: fallback, persist: true)]); expect(h.exhausted, isEmpty); }); test('rejected later candidate keeps the last accepted endpoint authoritative', () async { final validations = []; final h = build( endpoints: const [primary, fallback, tertiary], handler: (request, _) async { expect(request.url.host, 'fallback.example.com'); return http.Response('fallback unavailable', 503); }, validateCandidate: (candidateBaseUrl, _) async { validations.add(candidateBaseUrl); return false; }, ); h.client.resetEndpoints(const [primary, fallback, tertiary], currentBaseUrl: fallback); h.client.baseUrl = fallback; expect((await h.client.get('/first')).statusCode, 503); expect((await h.client.get('/second')).statusCode, 503); expect(validations, [tertiary, tertiary]); expect(h.requests.map((uri) => uri.host), ['fallback.example.com', 'fallback.example.com']); expect(h.switches, isEmpty); expect(h.client.baseUrl, fallback); expect(h.exhausted, hasLength(2)); }); test('resetEndpoints replaces the cascade list', () async { final h = build( handler: (request, _) async { if (request.url.host == 'tertiary.example.com') return ok(); throw TimeoutException('down'); }, ); h.client.resetEndpoints(const [fallback, tertiary], currentBaseUrl: fallback); h.client.baseUrl = fallback; final response = await h.client.get('/path'); expect(response.statusCode, 200); expect(h.requests.map((u) => u.host), ['fallback.example.com', 'tertiary.example.com']); expect(h.switches, [(url: tertiary, persist: false), (url: tertiary, persist: true)]); }); }