fix: failover storm during network loss

This commit is contained in:
edde746
2026-01-28 04:03:43 +01:00
parent 33e977872e
commit 3c039ad69c
3 changed files with 111 additions and 2 deletions
+30 -1
View File
@@ -37,6 +37,12 @@ class PlaybackProgressTracker {
/// Update interval (default: 10 seconds)
final Duration updateInterval;
/// Counts consecutive online progress failures for backoff logic.
int _consecutiveFailures = 0;
/// Timer ticks to skip before retrying after failures (exponential backoff).
int _ticksToSkip = 0;
PlaybackProgressTracker({
required this.client,
required this.metadata,
@@ -64,6 +70,12 @@ class PlaybackProgressTracker {
_progressTimer = Timer.periodic(updateInterval, (timer) {
if (player.state.playing) {
// Skip ticks when backing off after consecutive failures to avoid
// flooding the network with doomed requests during an outage.
if (_ticksToSkip > 0) {
_ticksToSkip--;
return;
}
_sendProgress('playing');
}
});
@@ -103,6 +115,12 @@ class PlaybackProgressTracker {
} else {
// Send progress to server immediately
await _sendOnlineProgress(state, position, duration);
// Success — reset backoff state
if (_consecutiveFailures > 0) {
appLogger.d('Progress update succeeded after $_consecutiveFailures consecutive failure(s), resetting backoff');
_consecutiveFailures = 0;
_ticksToSkip = 0;
}
}
// Emit watch state event on stop for UI updates across screens
@@ -114,7 +132,18 @@ class PlaybackProgressTracker {
);
}
} catch (e) {
appLogger.d('Failed to send progress update (non-critical)', error: e);
if (!isOffline) {
_consecutiveFailures++;
// Exponential backoff: skip 1, 2, 4, 8... ticks (capped at 6 ≈ 60s)
_ticksToSkip = (1 << (_consecutiveFailures - 1)).clamp(1, 6);
appLogger.d(
'Progress update failed ($_consecutiveFailures consecutive), '
'skipping next $_ticksToSkip tick(s)',
error: e,
);
} else {
appLogger.d('Failed to send progress update (non-critical)', error: e);
}
}
}
+32
View File
@@ -563,6 +563,12 @@ class PlexServer {
}
for (final connection in connections) {
// Skip endpoints that are never reachable from an external client:
// Docker bridge addresses and IPv6 link-local / all-zeros addresses.
if (_isUnreachableAddress(connection.address)) {
continue;
}
// First, try the actual connection URI (may be HTTPS plex.direct)
final isPlexDirect = connection.uri.contains('.plex.direct');
final isHttps = connection.protocol == 'https';
@@ -738,6 +744,32 @@ class PlexServer {
return entries.first.key;
}
/// Returns true if the address is known to be unreachable from external
/// clients (Docker bridge networks, IPv6 link-local, or all-zeros).
static bool _isUnreachableAddress(String address) {
// Docker bridge subnets (172.17.0.0/16, 172.18.0.0/16, etc.)
// Docker uses 172.17-31.x.x by default.
final dockerPattern = RegExp(r'^172\.(1[7-9]|2[0-9]|3[01])\.');
if (dockerPattern.hasMatch(address)) {
return true;
}
// IPv6 all-zeros (::) or link-local (fe80::)
final normalized = address.replaceAll('-', ':').toLowerCase();
if (normalized == '::' || normalized == '0000:0000:0000:0000:0000:0000:0000:0000') {
return true;
}
// Condensed all-zeros variants
if (RegExp(r'^(0+:){7}0+$').hasMatch(normalized)) {
return true;
}
if (normalized.startsWith('fe80:') || normalized.startsWith('fe80::')) {
return true;
}
return false;
}
}
/// Represents a connection to a Plex server
+49 -1
View File
@@ -2,6 +2,9 @@ import 'package:dio/dio.dart';
import '../utils/app_logger.dart';
/// Key used to stamp requests with the failover generation they were issued under.
const _generationKey = '_failoverGeneration';
/// Maintains the list of endpoints we can cycle through when one fails.
class EndpointFailoverManager {
EndpointFailoverManager(List<String> urls) {
@@ -11,6 +14,11 @@ class EndpointFailoverManager {
late List<String> _endpoints;
int _currentIndex = 0;
/// Incremented every time the active endpoint changes. Requests stamped with
/// an older generation should not trigger additional failover cascades.
int _generation = 0;
int get generation => _generation;
List<String> get endpoints => List.unmodifiable(_endpoints);
String get current => _endpoints[_currentIndex];
@@ -21,9 +29,20 @@ class EndpointFailoverManager {
String? moveToNext() {
if (!hasFallback) return null;
_currentIndex++;
_generation++;
return _endpoints[_currentIndex];
}
/// Reset back to the first (preferred) endpoint. Called when all endpoints
/// are exhausted so the next failure cycle starts from the best candidate.
void resetToFirst() {
if (_currentIndex != 0) {
_currentIndex = 0;
_generation++;
appLogger.d('Failover endpoint list reset to first candidate');
}
}
/// Replace the endpoint list and optionally set the active endpoint.
void reset(List<String> urls, {String? currentBaseUrl}) {
_setEndpoints(urls);
@@ -33,6 +52,7 @@ class EndpointFailoverManager {
} else {
_currentIndex = 0;
}
_generation++;
}
void _setEndpoints(List<String> urls) {
@@ -65,9 +85,37 @@ class EndpointFailoverInterceptor extends Interceptor {
final Future<void> Function(String newBaseUrl) _onEndpointSwitch;
bool _isSwitching = false;
@override
void onRequest(RequestOptions options, RequestInterceptorHandler handler) {
// Stamp every outgoing request with the current failover generation so
// we can detect stale requests in onError.
options.extra[_generationKey] = endpointManager.generation;
handler.next(options);
}
@override
void onError(DioException err, ErrorInterceptorHandler handler) async {
if (_isSwitching || !_shouldAttemptFailover(err) || !endpointManager.hasFallback) {
if (_isSwitching || !_shouldAttemptFailover(err)) {
handler.next(err);
return;
}
// If the endpoint changed since this request was dispatched, the request
// timed out on an already-abandoned endpoint. Don't cascade another switch.
final requestGeneration = err.requestOptions.extra[_generationKey] as int?;
if (requestGeneration != null && requestGeneration != endpointManager.generation) {
appLogger.d(
'Skipping failover for stale request (generation $requestGeneration != ${endpointManager.generation})',
error: {'path': err.requestOptions.path},
);
handler.next(err);
return;
}
if (!endpointManager.hasFallback) {
// All endpoints exhausted — reset to first so the next failure cycle
// starts from the preferred endpoint (handles transient network outages).
endpointManager.resetToFirst();
handler.next(err);
return;
}