fix: failover storm during network loss
This commit is contained in:
@@ -37,6 +37,12 @@ class PlaybackProgressTracker {
|
||||
/// Update interval (default: 10 seconds)
|
||||
final Duration updateInterval;
|
||||
|
||||
/// Counts consecutive online progress failures for backoff logic.
|
||||
int _consecutiveFailures = 0;
|
||||
|
||||
/// Timer ticks to skip before retrying after failures (exponential backoff).
|
||||
int _ticksToSkip = 0;
|
||||
|
||||
PlaybackProgressTracker({
|
||||
required this.client,
|
||||
required this.metadata,
|
||||
@@ -64,6 +70,12 @@ class PlaybackProgressTracker {
|
||||
|
||||
_progressTimer = Timer.periodic(updateInterval, (timer) {
|
||||
if (player.state.playing) {
|
||||
// Skip ticks when backing off after consecutive failures to avoid
|
||||
// flooding the network with doomed requests during an outage.
|
||||
if (_ticksToSkip > 0) {
|
||||
_ticksToSkip--;
|
||||
return;
|
||||
}
|
||||
_sendProgress('playing');
|
||||
}
|
||||
});
|
||||
@@ -103,6 +115,12 @@ class PlaybackProgressTracker {
|
||||
} else {
|
||||
// Send progress to server immediately
|
||||
await _sendOnlineProgress(state, position, duration);
|
||||
// Success — reset backoff state
|
||||
if (_consecutiveFailures > 0) {
|
||||
appLogger.d('Progress update succeeded after $_consecutiveFailures consecutive failure(s), resetting backoff');
|
||||
_consecutiveFailures = 0;
|
||||
_ticksToSkip = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Emit watch state event on stop for UI updates across screens
|
||||
@@ -114,7 +132,18 @@ class PlaybackProgressTracker {
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
appLogger.d('Failed to send progress update (non-critical)', error: e);
|
||||
if (!isOffline) {
|
||||
_consecutiveFailures++;
|
||||
// Exponential backoff: skip 1, 2, 4, 8... ticks (capped at 6 ≈ 60s)
|
||||
_ticksToSkip = (1 << (_consecutiveFailures - 1)).clamp(1, 6);
|
||||
appLogger.d(
|
||||
'Progress update failed ($_consecutiveFailures consecutive), '
|
||||
'skipping next $_ticksToSkip tick(s)',
|
||||
error: e,
|
||||
);
|
||||
} else {
|
||||
appLogger.d('Failed to send progress update (non-critical)', error: e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -563,6 +563,12 @@ class PlexServer {
|
||||
}
|
||||
|
||||
for (final connection in connections) {
|
||||
// Skip endpoints that are never reachable from an external client:
|
||||
// Docker bridge addresses and IPv6 link-local / all-zeros addresses.
|
||||
if (_isUnreachableAddress(connection.address)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// First, try the actual connection URI (may be HTTPS plex.direct)
|
||||
final isPlexDirect = connection.uri.contains('.plex.direct');
|
||||
final isHttps = connection.protocol == 'https';
|
||||
@@ -738,6 +744,32 @@ class PlexServer {
|
||||
|
||||
return entries.first.key;
|
||||
}
|
||||
|
||||
/// Returns true if the address is known to be unreachable from external
|
||||
/// clients (Docker bridge networks, IPv6 link-local, or all-zeros).
|
||||
static bool _isUnreachableAddress(String address) {
|
||||
// Docker bridge subnets (172.17.0.0/16, 172.18.0.0/16, etc.)
|
||||
// Docker uses 172.17-31.x.x by default.
|
||||
final dockerPattern = RegExp(r'^172\.(1[7-9]|2[0-9]|3[01])\.');
|
||||
if (dockerPattern.hasMatch(address)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// IPv6 all-zeros (::) or link-local (fe80::)
|
||||
final normalized = address.replaceAll('-', ':').toLowerCase();
|
||||
if (normalized == '::' || normalized == '0000:0000:0000:0000:0000:0000:0000:0000') {
|
||||
return true;
|
||||
}
|
||||
// Condensed all-zeros variants
|
||||
if (RegExp(r'^(0+:){7}0+$').hasMatch(normalized)) {
|
||||
return true;
|
||||
}
|
||||
if (normalized.startsWith('fe80:') || normalized.startsWith('fe80::')) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// Represents a connection to a Plex server
|
||||
|
||||
@@ -2,6 +2,9 @@ import 'package:dio/dio.dart';
|
||||
|
||||
import '../utils/app_logger.dart';
|
||||
|
||||
/// Key used to stamp requests with the failover generation they were issued under.
|
||||
const _generationKey = '_failoverGeneration';
|
||||
|
||||
/// Maintains the list of endpoints we can cycle through when one fails.
|
||||
class EndpointFailoverManager {
|
||||
EndpointFailoverManager(List<String> urls) {
|
||||
@@ -11,6 +14,11 @@ class EndpointFailoverManager {
|
||||
late List<String> _endpoints;
|
||||
int _currentIndex = 0;
|
||||
|
||||
/// Incremented every time the active endpoint changes. Requests stamped with
|
||||
/// an older generation should not trigger additional failover cascades.
|
||||
int _generation = 0;
|
||||
int get generation => _generation;
|
||||
|
||||
List<String> get endpoints => List.unmodifiable(_endpoints);
|
||||
|
||||
String get current => _endpoints[_currentIndex];
|
||||
@@ -21,9 +29,20 @@ class EndpointFailoverManager {
|
||||
String? moveToNext() {
|
||||
if (!hasFallback) return null;
|
||||
_currentIndex++;
|
||||
_generation++;
|
||||
return _endpoints[_currentIndex];
|
||||
}
|
||||
|
||||
/// Reset back to the first (preferred) endpoint. Called when all endpoints
|
||||
/// are exhausted so the next failure cycle starts from the best candidate.
|
||||
void resetToFirst() {
|
||||
if (_currentIndex != 0) {
|
||||
_currentIndex = 0;
|
||||
_generation++;
|
||||
appLogger.d('Failover endpoint list reset to first candidate');
|
||||
}
|
||||
}
|
||||
|
||||
/// Replace the endpoint list and optionally set the active endpoint.
|
||||
void reset(List<String> urls, {String? currentBaseUrl}) {
|
||||
_setEndpoints(urls);
|
||||
@@ -33,6 +52,7 @@ class EndpointFailoverManager {
|
||||
} else {
|
||||
_currentIndex = 0;
|
||||
}
|
||||
_generation++;
|
||||
}
|
||||
|
||||
void _setEndpoints(List<String> urls) {
|
||||
@@ -65,9 +85,37 @@ class EndpointFailoverInterceptor extends Interceptor {
|
||||
final Future<void> Function(String newBaseUrl) _onEndpointSwitch;
|
||||
bool _isSwitching = false;
|
||||
|
||||
@override
|
||||
void onRequest(RequestOptions options, RequestInterceptorHandler handler) {
|
||||
// Stamp every outgoing request with the current failover generation so
|
||||
// we can detect stale requests in onError.
|
||||
options.extra[_generationKey] = endpointManager.generation;
|
||||
handler.next(options);
|
||||
}
|
||||
|
||||
@override
|
||||
void onError(DioException err, ErrorInterceptorHandler handler) async {
|
||||
if (_isSwitching || !_shouldAttemptFailover(err) || !endpointManager.hasFallback) {
|
||||
if (_isSwitching || !_shouldAttemptFailover(err)) {
|
||||
handler.next(err);
|
||||
return;
|
||||
}
|
||||
|
||||
// If the endpoint changed since this request was dispatched, the request
|
||||
// timed out on an already-abandoned endpoint. Don't cascade another switch.
|
||||
final requestGeneration = err.requestOptions.extra[_generationKey] as int?;
|
||||
if (requestGeneration != null && requestGeneration != endpointManager.generation) {
|
||||
appLogger.d(
|
||||
'Skipping failover for stale request (generation $requestGeneration != ${endpointManager.generation})',
|
||||
error: {'path': err.requestOptions.path},
|
||||
);
|
||||
handler.next(err);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!endpointManager.hasFallback) {
|
||||
// All endpoints exhausted — reset to first so the next failure cycle
|
||||
// starts from the preferred endpoint (handles transient network outages).
|
||||
endpointManager.resetToFirst();
|
||||
handler.next(err);
|
||||
return;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user