fix(plex): validate a failover candidate before switching the live endpoint
A transient GET failure on a healthy endpoint could park the client on an unreachable fallback (e.g. the server host's Docker bridge gateway, which plex.tv advertises as a local connection) for a full connect timeout, failing every request in flight during that window (log bbr90). The cascade now probes each candidate with an unauthenticated /identity request under the discovery-race budget and only switches when it answers as the expected server, mirroring the Jellyfin trust gate. Unreachable-looking private IPv4 candidates stay in the list — a client on the server host can legitimately reach them, so reachability is probed, not inferred.
This commit is contained in:
@@ -620,8 +620,11 @@ class PlexServer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for (final connection in connections) {
|
for (final connection in connections) {
|
||||||
// Skip endpoints that are never reachable from an external client:
|
// Skip endpoints that are never reachable from any client: IPv6
|
||||||
// Docker bridge addresses and IPv6 link-local / all-zeros addresses.
|
// link-local / all-zeros addresses. Private IPv4 addresses (including
|
||||||
|
// Docker bridge gateways) are deliberately kept — a client running on
|
||||||
|
// the server host itself can reach them, so reachability is probed at
|
||||||
|
// failover time (PlexClient's validateCandidate), not inferred here.
|
||||||
if (_isUnreachableAddress(connection.address)) {
|
if (_isUnreachableAddress(connection.address)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -332,6 +332,12 @@ class PlexClient
|
|||||||
final Future<void> Function(String newBaseUrl)? _onEndpointChanged;
|
final Future<void> Function(String newBaseUrl)? _onEndpointChanged;
|
||||||
final VoidCallback? _onAllEndpointsExhausted;
|
final VoidCallback? _onAllEndpointsExhausted;
|
||||||
|
|
||||||
|
/// Test seam for [_validateFailoverCandidate]'s ephemeral probe client,
|
||||||
|
/// mirroring [JellyfinClient.forTesting]'s `endpointProbeHttpClientFactory`.
|
||||||
|
/// Each validation constructs (and closes) its own client, so this is a
|
||||||
|
/// factory rather than a shared instance.
|
||||||
|
final http.Client Function()? _endpointProbeHttpClientFactory;
|
||||||
|
|
||||||
/// Server identifier - all PlexMetadataDto items created by this client are tagged with this
|
/// Server identifier - all PlexMetadataDto items created by this client are tagged with this
|
||||||
@override
|
@override
|
||||||
final ServerId serverId;
|
final ServerId serverId;
|
||||||
@@ -460,6 +466,7 @@ class PlexClient
|
|||||||
this._onEndpointChanged,
|
this._onEndpointChanged,
|
||||||
this._onAllEndpointsExhausted,
|
this._onAllEndpointsExhausted,
|
||||||
http.Client? httpClient,
|
http.Client? httpClient,
|
||||||
|
this._endpointProbeHttpClientFactory,
|
||||||
}) {
|
}) {
|
||||||
LogRedactionManager.registerServer(config.baseUrl, config.token);
|
LogRedactionManager.registerServer(config.baseUrl, config.token);
|
||||||
|
|
||||||
@@ -474,6 +481,7 @@ class PlexClient
|
|||||||
prioritizedEndpoints: prioritizedEndpoints ?? const [],
|
prioritizedEndpoints: prioritizedEndpoints ?? const [],
|
||||||
onEndpointSwitch: (newBaseUrl, {required persist}) => _handleEndpointSwitch(newBaseUrl, persist: persist),
|
onEndpointSwitch: (newBaseUrl, {required persist}) => _handleEndpointSwitch(newBaseUrl, persist: persist),
|
||||||
onAllEndpointsExhausted: _onAllEndpointsExhausted,
|
onAllEndpointsExhausted: _onAllEndpointsExhausted,
|
||||||
|
validateCandidate: _validateFailoverCandidate,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -490,6 +498,8 @@ class PlexClient
|
|||||||
String? serverName,
|
String? serverName,
|
||||||
required http.Client httpClient,
|
required http.Client httpClient,
|
||||||
List<String>? prioritizedEndpoints,
|
List<String>? prioritizedEndpoints,
|
||||||
|
http.Client Function()? endpointProbeHttpClientFactory,
|
||||||
|
VoidCallback? onAllEndpointsExhausted,
|
||||||
List<({String identifier, String gridEndpoint})> epgProviders = const [],
|
List<({String identifier, String gridEndpoint})> epgProviders = const [],
|
||||||
String? homeHubKey,
|
String? homeHubKey,
|
||||||
String? promotedHubKey,
|
String? promotedHubKey,
|
||||||
@@ -502,6 +512,8 @@ class PlexClient
|
|||||||
serverName: serverName,
|
serverName: serverName,
|
||||||
httpClient: httpClient,
|
httpClient: httpClient,
|
||||||
prioritizedEndpoints: prioritizedEndpoints,
|
prioritizedEndpoints: prioritizedEndpoints,
|
||||||
|
endpointProbeHttpClientFactory: endpointProbeHttpClientFactory,
|
||||||
|
onAllEndpointsExhausted: onAllEndpointsExhausted,
|
||||||
);
|
);
|
||||||
client._providerLibraries = const [];
|
client._providerLibraries = const [];
|
||||||
client._providerEpg = epgProviders;
|
client._providerEpg = epgProviders;
|
||||||
@@ -2920,6 +2932,40 @@ class PlexClient
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Trust gate for endpoint failover: before the cascade may switch to a
|
||||||
|
/// fallback candidate, it must answer the unauthenticated `/identity` probe
|
||||||
|
/// quickly *and* identify as this client's server.
|
||||||
|
///
|
||||||
|
/// plex.tv advertises every interface of the server host as a connection
|
||||||
|
/// candidate, including addresses only that host can reach (e.g. its Docker
|
||||||
|
/// bridge gateway) — whether such an address works is a property of the
|
||||||
|
/// session, not the address, so it can only be probed, not filtered. Without
|
||||||
|
/// this gate one transient error on a healthy endpoint parked the live base
|
||||||
|
/// URL on a dead candidate for a full connect timeout (log bbr90).
|
||||||
|
///
|
||||||
|
/// The probe is deliberately unauthenticated — the token must not be sent to
|
||||||
|
/// an endpoint whose identity is unconfirmed — and uses the discovery race's
|
||||||
|
/// budget: every viable candidate already answered within it at discovery
|
||||||
|
/// time. Cancellations propagate to abort the cascade; other probe failures
|
||||||
|
/// propagate and reject the candidate ([FailoverHttpClient] semantics).
|
||||||
|
Future<bool> _validateFailoverCandidate(String candidateBaseUrl, AbortController? abort) async {
|
||||||
|
LogRedactionManager.registerServerUrl(candidateBaseUrl);
|
||||||
|
final probe = MediaServerHttpClient(
|
||||||
|
client: _endpointProbeHttpClientFactory?.call(),
|
||||||
|
baseUrl: candidateBaseUrl,
|
||||||
|
defaultHeaders: const {'Accept': 'application/json'},
|
||||||
|
connectTimeout: MediaServerTimeouts.connectionRace,
|
||||||
|
receiveTimeout: MediaServerTimeouts.connectionRace,
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
final response = await probe.get('/identity', abort: abort);
|
||||||
|
if (response.statusCode != 200) return false;
|
||||||
|
return _getMediaContainer(response)?['machineIdentifier']?.toString() == serverId;
|
||||||
|
} finally {
|
||||||
|
probe.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Validate and apply a Plex Home identity in place. The candidate token is
|
/// Validate and apply a Plex Home identity in place. The candidate token is
|
||||||
/// first checked against the authenticated root endpoint, whose
|
/// first checked against the authenticated root endpoint, whose
|
||||||
/// `machineIdentifier` must still identify this client’s server. Provider
|
/// `machineIdentifier` must still identify this client’s server. Provider
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import 'dart:async';
|
|||||||
|
|
||||||
import 'package:drift/native.dart';
|
import 'package:drift/native.dart';
|
||||||
import 'package:flutter_test/flutter_test.dart';
|
import 'package:flutter_test/flutter_test.dart';
|
||||||
|
import 'package:http/testing.dart';
|
||||||
import 'package:http/http.dart' as http;
|
import 'package:http/http.dart' as http;
|
||||||
import 'package:plezy/database/app_database.dart';
|
import 'package:plezy/database/app_database.dart';
|
||||||
import 'package:plezy/exceptions/media_server_exceptions.dart';
|
import 'package:plezy/exceptions/media_server_exceptions.dart';
|
||||||
@@ -525,6 +526,119 @@ void main() {
|
|||||||
expect(transport.requestCount, 1);
|
expect(transport.requestCount, 1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
group('Plex endpoint failover candidate validation', () {
|
||||||
|
http.Response identity(String machineIdentifier) => http.Response(
|
||||||
|
jsonEncode({
|
||||||
|
'MediaContainer': {'machineIdentifier': machineIdentifier},
|
||||||
|
}),
|
||||||
|
200,
|
||||||
|
headers: {'content-type': 'application/json'},
|
||||||
|
);
|
||||||
|
|
||||||
|
test('validates the candidate unauthenticated before the authenticated retry and switches', () async {
|
||||||
|
const primary = 'https://plex.example.com';
|
||||||
|
const fallback = 'https://plex-fallback.example.com';
|
||||||
|
final events = <String>[];
|
||||||
|
final probeRequests = <http.Request>[];
|
||||||
|
final client = testPlexClient(
|
||||||
|
serverId: publicServerId,
|
||||||
|
profileScopeId: defaultProfileScopeId,
|
||||||
|
httpClient: MockClient((request) async {
|
||||||
|
events.add('application:${request.url.host}');
|
||||||
|
expect(request.headers['X-Plex-Token'], isNotNull);
|
||||||
|
if (request.url.host == 'plex.example.com') {
|
||||||
|
throw TimeoutException('primary down');
|
||||||
|
}
|
||||||
|
return identity('server-id');
|
||||||
|
}),
|
||||||
|
prioritizedEndpoints: const [primary, fallback],
|
||||||
|
endpointProbeHttpClientFactory: () => MockClient((request) async {
|
||||||
|
probeRequests.add(request);
|
||||||
|
events.add('probe:${request.url.host}');
|
||||||
|
return identity('server-id');
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
addTearDown(client.close);
|
||||||
|
|
||||||
|
expect(await client.getMachineIdentifier(), 'server-id');
|
||||||
|
|
||||||
|
expect(events, [
|
||||||
|
'application:plex.example.com',
|
||||||
|
'probe:plex-fallback.example.com',
|
||||||
|
'application:plex-fallback.example.com',
|
||||||
|
]);
|
||||||
|
expect(probeRequests.single.url.path, '/identity');
|
||||||
|
expect(probeRequests.single.headers.keys.map((name) => name.toLowerCase()), isNot(contains('x-plex-token')));
|
||||||
|
expect(client.config.baseUrl, fallback);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('wrong-machine candidate is skipped before one authenticated retry to a valid candidate', () async {
|
||||||
|
final events = <String>[];
|
||||||
|
var exhausted = 0;
|
||||||
|
final client = testPlexClient(
|
||||||
|
serverId: publicServerId,
|
||||||
|
profileScopeId: defaultProfileScopeId,
|
||||||
|
httpClient: MockClient((request) async {
|
||||||
|
events.add('application:${request.url.host}');
|
||||||
|
if (request.url.host == 'plex.example.com') {
|
||||||
|
throw TimeoutException('primary down');
|
||||||
|
}
|
||||||
|
expect(request.url.host, 'valid.example.com');
|
||||||
|
return identity('server-id');
|
||||||
|
}),
|
||||||
|
prioritizedEndpoints: const [
|
||||||
|
'https://plex.example.com',
|
||||||
|
'https://wrong-machine.example.com',
|
||||||
|
'https://valid.example.com',
|
||||||
|
],
|
||||||
|
endpointProbeHttpClientFactory: () => MockClient((request) async {
|
||||||
|
events.add('probe:${request.url.host}');
|
||||||
|
return identity(request.url.host == 'wrong-machine.example.com' ? 'other-server' : 'server-id');
|
||||||
|
}),
|
||||||
|
onAllEndpointsExhausted: () => exhausted++,
|
||||||
|
);
|
||||||
|
addTearDown(client.close);
|
||||||
|
|
||||||
|
expect(await client.getMachineIdentifier(), 'server-id');
|
||||||
|
|
||||||
|
expect(events, [
|
||||||
|
'application:plex.example.com',
|
||||||
|
'probe:wrong-machine.example.com',
|
||||||
|
'probe:valid.example.com',
|
||||||
|
'application:valid.example.com',
|
||||||
|
]);
|
||||||
|
expect(exhausted, 0);
|
||||||
|
expect(client.config.baseUrl, 'https://valid.example.com');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('unreachable candidate receives no authenticated request and the base URL stays put', () async {
|
||||||
|
final events = <String>[];
|
||||||
|
var exhausted = 0;
|
||||||
|
final client = testPlexClient(
|
||||||
|
serverId: publicServerId,
|
||||||
|
profileScopeId: defaultProfileScopeId,
|
||||||
|
httpClient: MockClient((request) async {
|
||||||
|
events.add('application:${request.url.host}');
|
||||||
|
expect(request.url.host, 'plex.example.com', reason: 'unvalidated candidates must not see the token');
|
||||||
|
throw TimeoutException('primary down');
|
||||||
|
}),
|
||||||
|
prioritizedEndpoints: const ['https://plex.example.com', 'https://unreachable.example.com'],
|
||||||
|
endpointProbeHttpClientFactory: () => MockClient((request) async {
|
||||||
|
events.add('probe:${request.url.host}');
|
||||||
|
throw TimeoutException('probe unavailable');
|
||||||
|
}),
|
||||||
|
onAllEndpointsExhausted: () => exhausted++,
|
||||||
|
);
|
||||||
|
addTearDown(client.close);
|
||||||
|
|
||||||
|
expect(await client.getMachineIdentifier(), isNull);
|
||||||
|
|
||||||
|
expect(events, ['application:plex.example.com', 'probe:unreachable.example.com']);
|
||||||
|
expect(exhausted, 1);
|
||||||
|
expect(client.config.baseUrl, 'https://plex.example.com');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
test('metadata edit preserves locked fields and removed tag wire format', () async {
|
test('metadata edit preserves locked fields and removed tag wire format', () async {
|
||||||
http.Request? captured;
|
http.Request? captured;
|
||||||
final client = makeClient((request) async {
|
final client = makeClient((request) async {
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import 'dart:convert';
|
|||||||
import 'package:drift/native.dart';
|
import 'package:drift/native.dart';
|
||||||
import 'package:flutter_test/flutter_test.dart';
|
import 'package:flutter_test/flutter_test.dart';
|
||||||
import 'package:http/http.dart' as http;
|
import 'package:http/http.dart' as http;
|
||||||
|
import 'package:http/testing.dart';
|
||||||
import 'package:plezy/database/app_database.dart';
|
import 'package:plezy/database/app_database.dart';
|
||||||
import 'package:plezy/models/plex/plex_config.dart';
|
import 'package:plezy/models/plex/plex_config.dart';
|
||||||
import 'package:plezy/services/plex_api_cache.dart';
|
import 'package:plezy/services/plex_api_cache.dart';
|
||||||
@@ -248,6 +249,17 @@ void main() {
|
|||||||
serverName: 'Server',
|
serverName: 'Server',
|
||||||
httpClient: httpClient,
|
httpClient: httpClient,
|
||||||
prioritizedEndpoints: const [primary, fallback],
|
prioritizedEndpoints: const [primary, fallback],
|
||||||
|
// The candidate must validate for the cascade to reach the
|
||||||
|
// authenticated retry whose failure this test pins.
|
||||||
|
endpointProbeHttpClientFactory: () => MockClient(
|
||||||
|
(_) async => http.Response(
|
||||||
|
jsonEncode({
|
||||||
|
'MediaContainer': {'machineIdentifier': 'server-id'},
|
||||||
|
}),
|
||||||
|
200,
|
||||||
|
headers: {'content-type': 'application/json'},
|
||||||
|
),
|
||||||
|
),
|
||||||
);
|
);
|
||||||
addTearDown(client.close);
|
addTearDown(client.close);
|
||||||
|
|
||||||
|
|||||||
@@ -149,6 +149,8 @@ PlexClient testPlexClient({
|
|||||||
http.Client? httpClient,
|
http.Client? httpClient,
|
||||||
Future<http.Response> Function(http.Request request)? handler,
|
Future<http.Response> Function(http.Request request)? handler,
|
||||||
List<String>? prioritizedEndpoints,
|
List<String>? prioritizedEndpoints,
|
||||||
|
http.Client Function()? endpointProbeHttpClientFactory,
|
||||||
|
void Function()? onAllEndpointsExhausted,
|
||||||
List<({String identifier, String gridEndpoint})> epgProviders = const [],
|
List<({String identifier, String gridEndpoint})> epgProviders = const [],
|
||||||
String? homeHubKey,
|
String? homeHubKey,
|
||||||
String? promotedHubKey,
|
String? promotedHubKey,
|
||||||
@@ -163,6 +165,8 @@ PlexClient testPlexClient({
|
|||||||
serverName: serverName,
|
serverName: serverName,
|
||||||
httpClient: httpClient ?? MockClient(handler ?? _defaultResponse),
|
httpClient: httpClient ?? MockClient(handler ?? _defaultResponse),
|
||||||
prioritizedEndpoints: prioritizedEndpoints,
|
prioritizedEndpoints: prioritizedEndpoints,
|
||||||
|
endpointProbeHttpClientFactory: endpointProbeHttpClientFactory,
|
||||||
|
onAllEndpointsExhausted: onAllEndpointsExhausted,
|
||||||
epgProviders: epgProviders,
|
epgProviders: epgProviders,
|
||||||
homeHubKey: homeHubKey,
|
homeHubKey: homeHubKey,
|
||||||
promotedHubKey: promotedHubKey,
|
promotedHubKey: promotedHubKey,
|
||||||
|
|||||||
Reference in New Issue
Block a user