diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index 3b882dec..7a62f884 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -152,26 +152,17 @@ android { packaging { jniLibs { - // Three copies of libc++_shared.so reach the merge: the libmpv AAR's - // (NDK r29 — exports std::from_chars that libmpv.so needs), the - // :libass module's CMake-contributed copy (NDK 28.2 — lacks it), and - // peerless2012:ass's bundled copy (also old). pickFirst keeps the merge - // from erroring on the duplicates; WHICH copy wins is pinned by the - // sourceSets block below: extractMpvLibcxx unpacks the libmpv AAR's copy - // into an app jniLibs dir, and PROJECT-scope sources beat sub-projects - // and external AARs. libc++ is backward ABI-compatible, so the older-NDK - // consumers (libass.so, libasskt.so, ffmpeg decoder, cronet) run fine - // against the newer copy. + // pickFirst only suppresses the duplicate libc++ merge error; the + // sourceSets rule below makes libmpv's newer runtime win for + // std::from_chars, while older native consumers remain ABI-compatible. pickFirsts.add("lib/*/libc++_shared.so") } } sourceSets { getByName("main") { - // libc++_shared.so extracted from the libmpv AAR by extractMpvLibcxx. - // App source-set jniLibs sit in the PROJECT scope, merged ahead of - // subprojects (:libass) and external AARs, so with the pickFirst rule - // above this copy deterministically wins regardless of dependency order. + // PROJECT-scope jniLibs merge ahead of subprojects/AARs, so dependency + // order cannot accidentally select the older libc++ copy. jniLibs.srcDir(File(mpvDir, "libcxx/jni")) } } @@ -181,17 +172,15 @@ flutter { source = "../.." } -// Download libdovi before any CMake/native build task tasks.matching { it.name.contains("CMake") || it.name.contains("externalNative") }.configureEach { dependsOn(downloadLibdovi) } -// Download the libmpv AAR before compilation tasks.matching { it.name.startsWith("pre") && it.name.endsWith("Build") }.configureEach { dependsOn(downloadLibmpv, extractMpvLibcxx) } -// merge{Debug,Profile,Release}JniLibFolders snapshot jniLibs source dirs as inputs; -// Gradle 8 requires an explicit dependency on the producing task. +// Gradle snapshots jniLibs source dirs before task execution; this keeps the +// extracted libmpv libc++ directory present during input discovery. tasks.matching { it.name.startsWith("merge") && it.name.endsWith("JniLibFolders") }.configureEach { dependsOn(extractMpvLibcxx) } @@ -216,12 +205,8 @@ dependencies { // FFmpeg audio decoder for unsupported codecs (ALAC, DTS, TrueHD, etc.) implementation("org.jellyfin.media3:media3-ffmpeg-decoder:1.9.0+1") - // libass ASS/SSA subtitle rendering: optimized native core (libass.so + - // prefab headers) from the edde746/libass-android fork's releases; Kotlin/JNI - // bindings + Media3 glue live in the android/libass module. -PlocalAssCore - // swaps in a mavenLocal()-published core (0.4.0-local) for native A/B tests. - val assCoreVersion = if (project.hasProperty("localAssCore")) "0.4.0-local" else "0.4.1-plezy.1" - implementation("io.github.peerless2012:ass:$assCoreVersion@aar") + // Keeping libass in-project lets its static core share the app's native + // packaging rules. implementation(project(":libass")) testImplementation("junit:junit:4.13.2") diff --git a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/AssLatencyCalibrator.kt b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/AssLatencyCalibrator.kt new file mode 100644 index 00000000..c524cb1c --- /dev/null +++ b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/AssLatencyCalibrator.kt @@ -0,0 +1,196 @@ +package com.edde746.plezy.exoplayer + +import android.os.Build +import android.view.SurfaceControl +import android.view.SurfaceView +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger +import kotlin.math.abs +import kotlin.math.roundToInt + +/** + * Auto-calibrates [ExoPlayerCore]'s subtitle/video layer offset by measuring, per device, how much + * later the codec VIDEO plane reaches the display than the GL subtitle OVERLAY plane — the + * inter-plane latency (a deep video VPP on TV boxes) that otherwise makes subtitles a frame ahead. + * + * Both planes are pinned to the same video-frame target (releaseTimeNs). Measuring BOTH with the + * same mechanism — [SurfaceTxProbe] + the API-34 previous-release fence — the per-SoC buffer-recycle + * bias cancels in the difference, leaving the pure offset: + * + * offsetFrames = round( (median(videoReleaseVsTarget) − median(overlayReleaseVsTarget)) / frameMs ) + * + * This is a MEDIAN-of-each-plane model, NOT a per-frame paired difference: every sample is + * self-paired to its own frame's target, and the per-plane release-vs-target is a steady hardware + * constant, so the two medians need not come from the same frames — only from the same (stable) + * device. Both medians use a recent-sample window so they stay roughly contemporaneous, and a + * confidence gate refuses to converge on an ambiguous (~half-frame) result. + * + * Video samples arrive every few frames from the metadata listener (codec thread); overlay samples + * from the libass GL thread's pre-swap hook (sparse — only when subtitles swap). Once both planes + * have enough samples and the result is confident it converges ONCE, applies + persists, and stops + * all probing (zero steady-state overhead). Re-runs each play to re-confirm. + * + * Threads: probeVideo (codec), probeOverlay (libass GL), onResult (probe reader). The lock guards + * only counter/sample state and is NEVER held across the binder/JNI `applyTransactionToFrame` call + * (so the GL swap path is never stalled by it). The native slot ring is allocated atomically. + */ +internal class AssLatencyCalibrator( + private val videoSurface: SurfaceView, + private val overlaySurface: SurfaceView, + private val onCalibrated: (Int) -> Unit, + private val onDone: () -> Unit, + private val log: (String) -> Unit, +) { + private val lock = Any() + private val finished = AtomicBoolean(false) + private val applyFail = AtomicInteger(0) + + @Volatile private var frameIntervalNs: Long = 0L + @Volatile private var converged = false + @Volatile private var stopped = false + + private var videoFrames = 0 + private var videoAttempts = 0 + private var overlaySwaps = 0 + private var overlayAttempts = 0 + private val videoRel = ArrayDeque() + private val overlayRel = ArrayDeque() + + fun start() { + SurfaceTxProbe.sink = { tag, latch, release, count, state, source, cb -> + onResult(tag, latch, release, count, state, source, cb) + } + log("calibration started (API ${Build.VERSION.SDK_INT})") + } + + fun probeVideo(releaseTimeNs: Long, fps: Float) { + if (stopped || converged || Build.VERSION.SDK_INT < 34) return + if (fps > 1f) frameIntervalNs = (1_000_000_000.0 / fps).toLong() + var doAttach = false + var giveUp = false + synchronized(lock) { + videoFrames++ + if (videoFrames > GIVEUP_FRAMES) { + giveUp = true + } else if (videoAttempts < VIDEO_ATTEMPT_CAP && videoFrames % VIDEO_EVERY == 0) { + videoAttempts++ + doAttach = true + } + } + if (giveUp) finishIncomplete() else if (doAttach) attach(videoSurface, releaseTimeNs, SurfaceTxProbe.SOURCE_VIDEO) + } + + fun probeOverlay(releaseTimeNs: Long) { + if (stopped || converged || Build.VERSION.SDK_INT < 34) return + var doAttach = false + synchronized(lock) { + overlaySwaps++ + if (overlayAttempts < OVERLAY_ATTEMPT_CAP && overlaySwaps % OVERLAY_EVERY == 0) { + overlayAttempts++ + doAttach = true + } + } + if (doAttach) attach(overlaySurface, releaseTimeNs, SurfaceTxProbe.SOURCE_OVERLAY) + } + + /** Never called while holding [lock]: applyTransactionToFrame is a binder call and runs on the + * GL swap path. The native slot allocation is atomic, so concurrent callers are safe. */ + private fun attach(surface: SurfaceView, releaseTimeNs: Long, source: Int) { + try { + val tx = SurfaceControl.Transaction() + SurfaceTxProbe.nativeAttach(tx, releaseTimeNs, source) + surface.applyTransactionToFrame(tx) + } catch (t: Throwable) { + if (applyFail.incrementAndGet() == 1) { + log("applyTransactionToFrame failed: ${t.javaClass.simpleName}: ${t.message}") + } + } + } + + private fun onResult( + tag: Long, + latchNs: Long, + releaseNs: Long, + surfaceCount: Int, + fenceState: Int, + source: Int, + callbackNs: Long, + ) { + if (surfaceCount <= 0 || fenceState != FENCE_OK || releaseNs <= 0) return + val relMs = (releaseNs - tag) / 1_000_000.0 + var result: Int? = null + synchronized(lock) { + if (converged || stopped) return + val list = if (source == SurfaceTxProbe.SOURCE_OVERLAY) overlayRel else videoRel + val window = if (source == SurfaceTxProbe.SOURCE_OVERLAY) OVERLAY_WINDOW else VIDEO_WINDOW + list.addLast(relMs) + while (list.size > window) list.removeFirst() // keep a recent window so medians stay fresh + result = tryComputeOffset() + } + result?.let { + onCalibrated(it) + finish() + } + } + + private fun tryComputeOffset(): Int? { + val frameMs = frameIntervalNs / 1_000_000.0 + if (frameMs <= 0.5) return null + if (videoRel.size < VIDEO_MIN || overlayRel.size < OVERLAY_MIN) return null + val medV = median(videoRel) + val medO = median(overlayRel) + val offsetMs = medV - medO + val raw = offsetMs / frameMs + val frames = raw.roundToInt() + // Confidence gate: only trust a measurement that points clearly at an integer frame count. + // An ambiguous ~half-frame result (corruption / drift / wrong fps) keeps collecting instead. + if (abs(raw - frames) > CONFIDENCE_TOL) return null + val clamped = frames.coerceIn(-2, 2) + converged = true + log( + "CALIBRATED offsetFrames=$clamped raw=${"%.2f".format(raw)} " + + "(video=${"%.1f".format(medV)}ms − overlay=${"%.1f".format(medO)}ms = ${"%.1f".format(offsetMs)}ms " + + "/ frameMs=${"%.2f".format(frameMs)}) samples video=${videoRel.size} overlay=${overlayRel.size} " + + "applyFail=${applyFail.get()}" + ) + return clamped + } + + private fun finishIncomplete() { + if (converged || stopped) return + log( + "calibration incomplete after $videoFrames frames " + + "(video=${videoRel.size} overlay=${overlayRel.size}); keeping seed" + ) + finish() + } + + fun stop() = finish() + + private fun finish() { + if (!finished.compareAndSet(false, true)) return + stopped = true + SurfaceTxProbe.sink = null + onDone() + } + + private fun median(xs: Collection): Double { + val s = xs.sorted() + val n = s.size + return if (n % 2 == 1) s[n / 2] else (s[n / 2 - 1] + s[n / 2]) / 2.0 + } + + companion object { + private const val FENCE_OK = 3 + private const val VIDEO_EVERY = 2 + private const val OVERLAY_EVERY = 1 // overlay swaps are already sparse + private const val VIDEO_MIN = 40 + private const val OVERLAY_MIN = 12 + private const val VIDEO_WINDOW = 60 // recent-sample window for the median + private const val OVERLAY_WINDOW = 40 + private const val VIDEO_ATTEMPT_CAP = 200 // ~16s @24fps/2 of video probing, then the window freezes + private const val OVERLAY_ATTEMPT_CAP = 60 + private const val CONFIDENCE_TOL = 0.33 // |raw − round(raw)| must be within ⅓ frame to converge + private const val GIVEUP_FRAMES = 1800 // ~75s @24fps without enough confident overlay samples + } +} diff --git a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerCore.kt b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerCore.kt index 894cf096..a12e3cda 100644 --- a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerCore.kt +++ b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerCore.kt @@ -103,6 +103,16 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { /** Per-frame "video is at X" logcat stream (tag AssFrameCb) for diagnosing * ASS subtitle lag against the libass pipeline's render/swap lines. */ private const val ASS_FRAME_LOGS = false + private const val ASS_SYNC_LOG_INTERVAL_FRAMES = 120L + + /** Auto-calibrate the subtitle/video layer offset per device (API 34+) by measuring the + * video vs overlay plane present timing. See [AssLatencyCalibrator]. Falls back to the + * seeded value (persisted calibration or the Dart perf-tier proxy) when off/unsupported. */ + private const val ASS_LATENCY_AUTOCAL = true + + /** SharedPreferences store for the per-device subtitle/video latency calibration. */ + private const val ASS_CAL_PREFS = "plezy_ass_calibration" + private const val ASS_CAL_KEY_FRAMES = "video_latency_frames" private const val TS_TIMESTAMP_SEARCH_PACKETS = 1800 private val DV_CODEC_PROFILE_REGEX = Regex("""(?:^|,)\s*dvh[1e]\.(\d{2})""") @@ -158,6 +168,9 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { private var videoZoomScale: Float = 1.0f private var assHandler: AssHandler? = null private var assSubtitleView: AssSubtitleSurfaceView? = null + // Touched from the codec metadata listener, the GL-thread overlay hook, and the main-thread + // media-item-transition/dispose paths — keep visibility across threads. + @Volatile private var latencyCalibrator: AssLatencyCalibrator? = null private var assForceMargins = false private var lastAssMargins: IntArray? = null private var overlayLayoutListener: ViewTreeObserver.OnGlobalLayoutListener? = null @@ -165,6 +178,17 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { private var exoPlayer: ExoPlayer? = null private var renderersFactory: PlezyRenderersFactory? = null private val subtitleDelayUs = AtomicLong(0L) + + /** + * Frames the hardware codec→display path lags a GL subtitle overlay pinned to the + * same release time (the overlay otherwise shows a frame ahead of the picture). + * The subtitle is rendered this many frames earlier to match the later video — a + * content-time shift, so it never delays the overlay's present (delaying the + * present freezes the single-slot latest-wins pipeline). Device-specific: ~1 on + * low-end TV boxes (longer video pipeline), 0 on phones. Set from Dart at init + * from the device performance tier ([com.plezy/device] auto low-end signal). + */ + @Volatile private var assVideoLatencyFrames = 0 private var subtitlePositionPercent: Int = 100 private var subtitleFontSize: Float = 55f private var lastSubtitleCues: List = emptyList() @@ -233,6 +257,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { private val fpsTimestamps = LongArray(FPS_SAMPLE_COUNT) @Volatile private var fpsTimestampCount = 0 + private var assSyncFrameCount = 0L // Audio focus private var audioFocusManager: AudioFocusManager? = null @@ -693,7 +718,61 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { exoPlayer!!.addAnalyticsListener(decoderHangListener) exoPlayer!!.setVideoFrameMetadataListener { presentationTimeUs, releaseTimeNs, _, _ -> // ASS bypasses Media3's text renderer, so apply sub-delay before libass renders. - assSubtitleView?.requestRender(presentationTimeUs - subtitleDelayUs.get(), releaseTimeNs) + // Also render the subtitle one video-frame earlier than the picture: the + // hardware codec→display path lags a GL overlay pinned to the same release + // time, so without this the overlay shows a frame ahead of the video. Done + // as a content-time shift (not a present delay, which would freeze the + // single-slot latest-wins pipeline by superseding every frame). + val assView = assSubtitleView + val latencyFrames = assVideoLatencyFrames + val fps = detectedFrameRate.takeIf { it > 1f } ?: currentVideoFormat?.frameRate?.takeIf { it > 1f } ?: 0f + val videoLatencyUs = if (latencyFrames != 0 && fps > 1f) { + (latencyFrames * 1_000_000.0 / fps).toLong() + } else { + 0L + } + assView?.requestRender(presentationTimeUs - subtitleDelayUs.get() - videoLatencyUs, releaseTimeNs) + if (ASS_LATENCY_AUTOCAL && assView != null && Build.VERSION.SDK_INT >= 34) { + val calibrator = latencyCalibrator ?: surfaceView?.let { sv -> + AssLatencyCalibrator( + videoSurface = sv, + overlaySurface = assView, + onCalibrated = { frames -> onAssLatencyCalibrated(frames) }, + onDone = { assSubtitleView?.setPreSwapProbe(null) }, + log = { msg -> emitLog("info", "ass-latency-cal", msg) }, + ).also { + it.start() + // Bind the hook to THIS instance, not the volatile field, so a concurrent transition + // reset can't redirect it to a different/null calibrator mid-swap. + assView.setPreSwapProbe { rt -> it.probeOverlay(rt) } + latencyCalibrator = it + } + } + calibrator?.probeVideo(releaseTimeNs, fps) + } + assSyncFrameCount++ + if (assView != null && assSyncFrameCount % ASS_SYNC_LOG_INTERVAL_FRAMES == 0L) { + emitLog( + "info", + "ass-sync", + "frames=$assSyncFrameCount swaps=${assView.swapCount} late=${assView.lateSwapCount} " + + "phaseLeadMs=${assView.phaseLeadMs} swapLeadMs=${assView.swapLeadMs} frameOffMs=${videoLatencyUs / 1000} sleepMs=${assView.lastScheduledSleepMs} " + + "headroomMs=${assView.lastSwapHeadroomMs} leadMs=${assView.lastSwapLeadMs} " + + "minLeadMs=${assView.minLeadChangedMs?.toString() ?: "n/a"} " + + "present=${assView.presentSource} " + + "presentErrMs=${assView.lastPresentErrorMs?.toString() ?: "n/a"} " + + "worstPresentMs=${assView.worstPresentErrorMs?.toString() ?: "n/a"} " + + "presentHist=${assView.presentErrorHistogram.joinToString(",")} " + + "presentMeasured=${assView.presentMeasuredCount} " + + "presentInvalid=${assView.presentInvalidCount} presentDropped=${assView.presentDroppedCount} " + + "render=${assView.changedRenderCount}/${assView.renderCount} " + + "libassMs=${assView.lastLibassMs}/${assView.maxLibassMs} libassHist=${assView.libassMsHistogram.joinToString(",")} " + + "spec=${assView.specHits}/${assView.specMisses}/${assView.specSkips} " + + "prefetch=${assView.prefetchCount} blankClears=${assView.blankClearCount} " + + "coalesced=${assView.coalescedRequestCount} stale=${assView.staleGenerationCount}/${assView.staleBeforeSwapCount} " + + "superseded=${assView.supersededBeforeSwapCount}" + ) + } if (ASS_FRAME_LOGS) { // Reference stream for subtitle-lag diagnosis: the video frame ExoPlayer // is releasing right now and how far ahead of its vsync we are. Subtitle @@ -1313,6 +1392,12 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { delegate?.onEvent("file-loaded", null) delegate?.onPropertyChange("eof-reached", false) emitCurrentSeekable(force = true) + // Re-calibrate the subtitle/video layer offset each play: tear down the converged calibrator + // so the metadata listener lazily spins up a fresh one. The seeded (persisted) value stays + // applied meanwhile, so subtitles are right from the first frame. + latencyCalibrator?.stop() + latencyCalibrator = null + assSubtitleView?.setPreSwapProbe(null) } override fun onVideoSizeChanged(videoSize: VideoSize) { @@ -2579,6 +2664,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { // Reset FPS detection for new content detectedFrameRate = -1f fpsTimestampCount = 0 + assSyncFrameCount = 0 // Reset DV7 retry flag when opening a different file if (uri != currentMediaUri) { @@ -2779,6 +2865,35 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { subtitleDelayUs.set((seconds * 1_000_000).toLong()) } + fun setAssVideoLatencyFrames(frames: Int) { + assVideoLatencyFrames = frames.coerceIn(-2, 2) + } + + /** + * Seed the subtitle/video layer offset at init: prefer this device's persisted measured + * calibration, falling back to [proxyDefault] (the Dart perf-tier guess) on first-ever play. + * The live [AssLatencyCalibrator] re-confirms and updates it each play. + */ + fun seedAssVideoLatencyFrames(proxyDefault: Int) { + val stored = activity.getSharedPreferences(ASS_CAL_PREFS, Context.MODE_PRIVATE) + .getInt(ASS_CAL_KEY_FRAMES, Int.MIN_VALUE) + val seed = if (stored != Int.MIN_VALUE) stored else proxyDefault + setAssVideoLatencyFrames(seed) + Log.d(TAG, "ass latency seed=$seed (stored=${if (stored == Int.MIN_VALUE) "none" else stored}, proxy=$proxyDefault)") + } + + /** Apply + persist a freshly measured calibration (called by [AssLatencyCalibrator]). */ + private fun onAssLatencyCalibrated(frames: Int) { + val clamped = frames.coerceIn(-2, 2) + val previous = assVideoLatencyFrames + setAssVideoLatencyFrames(clamped) + activity.getSharedPreferences(ASS_CAL_PREFS, Context.MODE_PRIVATE) + .edit().putInt(ASS_CAL_KEY_FRAMES, clamped).apply() + if (clamped != previous) { + emitLog("info", "ass-latency-cal", "applied offsetFrames=$clamped (was $previous), persisted") + } + } + fun setDebugDvConversionMode(mode: String): Boolean { val override = when (mode.trim().lowercase()) { "auto" -> null @@ -2820,6 +2935,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { audioDecoderInitName = null detectedFrameRate = -1f fpsTimestampCount = 0 + assSyncFrameCount = 0 firstFrameRendered = false currentVideoFormat = null loggedNativeDvSelectionKey = null @@ -3227,7 +3343,26 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { "subSpecMisses" to assSubtitleView?.specMisses, "subSpecSkips" to assSubtitleView?.specSkips, "subPrefetches" to assSubtitleView?.prefetchCount, + "subBlankClears" to assSubtitleView?.blankClearCount, + "subCoalesced" to assSubtitleView?.coalescedRequestCount, + "subStaleGeneration" to assSubtitleView?.staleGenerationCount, + "subSupersededBeforeSwap" to assSubtitleView?.supersededBeforeSwapCount, + "subStaleBeforeSwap" to assSubtitleView?.staleBeforeSwapCount, "subMinLeadMs" to assSubtitleView?.minLeadChangedMs, + "subPhaseLeadMs" to assSubtitleView?.phaseLeadMs, + "subLastLeadMs" to assSubtitleView?.lastSwapLeadMs, + "subLastHeadroomMs" to assSubtitleView?.lastSwapHeadroomMs, + "subLastSleepMs" to assSubtitleView?.lastScheduledSleepMs, + // Actual on-screen present time vs the video frame's release target (ground + // truth for frame-perfection; null/false on emulator + pre-29 devices). + "subPresentTimingEnabled" to assSubtitleView?.presentTimingEnabled, + "subPresentSource" to assSubtitleView?.presentSource, + "subPresentErrMs" to assSubtitleView?.lastPresentErrorMs, + "subWorstPresentErrMs" to assSubtitleView?.worstPresentErrorMs, + "subPresentMeasured" to assSubtitleView?.presentMeasuredCount, + "subPresentInvalid" to assSubtitleView?.presentInvalidCount, + "subPresentDropped" to assSubtitleView?.presentDroppedCount, + "subPresentErrHist" to assSubtitleView?.presentErrorHistogram, // Color info "colorSpace" to videoFormat?.colorInfo?.colorSpace, "colorRange" to videoFormat?.colorInfo?.colorRange, @@ -3409,6 +3544,9 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { // Synchronous ownership invalidation — stale code can no longer // reach surface state through instance fields. + latencyCalibrator?.stop() + latencyCalibrator = null + assSubtitleView?.setPreSwapProbe(null) surfaceContainer = null videoAspectContainer = null surfaceView = null diff --git a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerPlugin.kt b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerPlugin.kt index c35c19c8..2ed13cfc 100644 --- a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerPlugin.kt +++ b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerPlugin.kt @@ -8,6 +8,7 @@ import android.net.Uri import android.os.Handler import android.os.Looper import android.util.Log +import com.edde746.plezy.libass.media.AssHandler import com.edde746.plezy.mpv.MpvPlayerCore import io.flutter.embedding.engine.plugins.FlutterPlugin import io.flutter.embedding.engine.plugins.activity.ActivityAware @@ -201,7 +202,12 @@ class ExoPlayerPlugin : val tunnelingEnabled = call.argument("tunnelingEnabled") ?: true val dvConversionMode = call.argument("dvConversionMode") ?: "auto" val audioPassthroughEnabled = call.argument("audioPassthroughEnabled") ?: false + val assVideoLatencyFrames = call.argument("assVideoLatencyFrames") ?: 0 + val subtitleRenderScale = call.argument("subtitleRenderScale")?.toFloat() ?: 1.0f configuredBufferSizeBytes = bufferSizeBytes + // Global libass overlay render scale — set before the player/handler is built below so the + // first frame-size apply already uses it. + AssHandler.setRenderScale(subtitleRenderScale) currentActivity.runOnUiThread { sessionGeneration++ @@ -229,6 +235,8 @@ class ExoPlayerPlugin : if (success && playerCore?.setDebugDvConversionMode(dvConversionMode) != true) { Log.w(TAG, "Invalid DV conversion mode during initialize: $dvConversionMode") } + // Seed from this device's persisted calibration, falling back to the Dart perf-tier proxy. + playerCore?.seedAssVideoLatencyFrames(assVideoLatencyFrames) // Start hidden playerCore?.setVisible(false) diff --git a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/SurfaceTxProbe.kt b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/SurfaceTxProbe.kt new file mode 100644 index 00000000..0dee396a --- /dev/null +++ b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/SurfaceTxProbe.kt @@ -0,0 +1,44 @@ +package com.edde746.plezy.exoplayer + +import android.view.SurfaceControl +import androidx.annotation.Keep + +/** + * EXPERIMENTAL SPIKE: the API-34 transaction callback path is the only in-app signal we have + * for comparing the codec video plane and the libass overlay plane with the same clock. + */ +// R8 cannot see the JNI-by-name callback edge, and class-level @Keep is not enough for members. +@Keep +object SurfaceTxProbe { + init { + System.loadLibrary("asskt") + } + + const val SOURCE_VIDEO = 0 + const val SOURCE_OVERLAY = 1 + + // CLOCK_MONOTONIC keeps native callback times comparable with ExoPlayer release targets. + @Volatile + @JvmStatic + var sink: ((Long, Long, Long, Int, Int, Int, Long) -> Unit)? = null + + // The same transaction must be applied to the frame so the callback follows that buffer. + @JvmStatic + external fun nativeAttach(transaction: SurfaceControl.Transaction, tag: Long, source: Int) + + // JNI calls this by name after shrink. + @Keep + @JvmStatic + fun onResult( + tag: Long, + latchNs: Long, + releaseNs: Long, + surfaceCount: Int, + fenceState: Int, + source: Int, + callbackNs: Long, + ) { + val s = sink ?: return + s.invoke(tag, latchNs, releaseNs, surfaceCount, fenceState, source, callbackNs) + } +} diff --git a/android/build.gradle.kts b/android/build.gradle.kts index 3fa01898..89176ef4 100644 --- a/android/build.gradle.kts +++ b/android/build.gradle.kts @@ -2,28 +2,6 @@ allprojects { repositories { google() mavenCentral() - if (providers.gradleProperty("localAssCore").isPresent) { - // Opt-in A/B of a locally built libass native core: build it in the - // libass-android fork with `./gradlew :lib_ass:publishToMavenLocal - // -PVERSION_NAME=0.4.0-local`, then build this app with -PlocalAssCore. - mavenLocal() - } else { - // Production libass native core: the -O3/NEON/asm AAR published on - // the edde746/libass-android fork's releases (the upstream Maven - // artifact io.github.peerless2012:ass ships un-optimized natives — - // see the fork's pinned libass-cmake fix). Resolved as - // /ass-.aar with no metadata probing. - exclusiveContent { - forRepository { - ivy { - url = uri("https://github.com/edde746/libass-android/releases/download") - patternLayout { artifact("[revision]/[artifact]-[revision].[ext]") } - metadataSources { artifact() } - } - } - filter { includeModule("io.github.peerless2012", "ass") } - } - } } } diff --git a/android/libass/build.gradle.kts b/android/libass/build.gradle.kts index 33415cf8..e6b5debf 100644 --- a/android/libass/build.gradle.kts +++ b/android/libass/build.gradle.kts @@ -1,7 +1,5 @@ -// libass ASS subtitle rendering: Kotlin/JNI bindings + Media3 integration -// (extractor, parsers, AssHandler, GL atlas overlay). The native libass core -// (libass.so + prefab headers) comes from the Maven artifact -// io.github.peerless2012:ass; this module compiles its JNI against it. +// Static-linking the native core here avoids shipping a separate libass.so with +// different merge rules from the app. plugins { id("com.android.library") id("org.jetbrains.kotlin.android") @@ -20,34 +18,19 @@ android { defaultConfig { minSdk = 21 consumerProguardFiles("consumer-rules.pro") - if (project.hasProperty("localAssCore")) { - // The locally published A/B core only ships device ABIs (x86 would need - // nasm on the host); match it so prefab resolution doesn't fail. - ndk { - abiFilters += listOf("armeabi-v7a", "arm64-v8a") - } - } externalNativeBuild { cmake { - // libass.so in the prefab AAR is built against c++_shared (abi.json: stl=c++_shared); - // prefab validates consumer STL compatibility. - arguments += listOf("-DANDROID_STL=c++_shared") + // HarfBuzz pulls in C++, so the JNI library must use the shared STL that + // the app already pins through libmpv. + // A shared cache keeps per-ABI CMake runs from redownloading libass. + arguments += listOf( + "-DANDROID_STL=c++_shared", + "-DLIBASS_CACHE_DIR=${layout.buildDirectory.get().asFile}/libass-prebuilt", + ) } } } - buildFeatures { - prefab = true - } - - packaging { - jniLibs { - // libass.so is a prefab IMPORTED target (linked, not owned) — the app packages - // it from the io.github.peerless2012:ass AAR; don't duplicate it here. - excludes.add("**/libass.so") - } - } - compileOptions { sourceCompatibility = JavaVersion.VERSION_11 targetCompatibility = JavaVersion.VERSION_11 @@ -66,13 +49,6 @@ android { } dependencies { - // compileOnly: prefab headers + link-time libass.so come from the AAR; runtime - // packaging of libass.so is the app's implementation dependency. - // -PlocalAssCore swaps in a mavenLocal()-published core for A/B tests (must - // match the app module's version so one libass.so is linked and packaged). - val assCoreVersion = if (project.hasProperty("localAssCore")) "0.4.0-local" else "0.4.1-plezy.1" - compileOnly("io.github.peerless2012:ass:$assCoreVersion@aar") - implementation("androidx.annotation:annotation:1.9.1") implementation("androidx.annotation:annotation-experimental:1.5.1") implementation("androidx.media3:media3-exoplayer:1.9.2") diff --git a/android/libass/src/main/cpp/AssKt.c b/android/libass/src/main/cpp/AssKt.c index e7558465..b3a9a9c6 100644 --- a/android/libass/src/main/cpp/AssKt.c +++ b/android/libass/src/main/cpp/AssKt.c @@ -1,10 +1,15 @@ // JNI bindings for libass. Exports use standard Java___ // naming so no RegisterNatives/JNI_OnLoad registration is needed. +#include +#include #include #include +#include #include +#include #include #include +#include #include static inline long long nowMs(void) { @@ -61,7 +66,13 @@ JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_Ass_nativeAssDeinit(JNIEnv* JNIEXPORT jlong JNICALL Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackInit(JNIEnv* env, jclass clazz, jlong ass) { - return (jlong)ass_new_track((ASS_Library*)ass); + ASS_Track* track = ass_new_track((ASS_Library*)ass); + if (track != NULL) { + if (ass_track_set_feature(track, ASS_FEATURE_FAST_BLUR, 1) != 0) { + __android_log_print(ANDROID_LOG_WARN, LOG_TAG, "ASS_FEATURE_FAST_BLUR unavailable in libass build"); + } + } + return (jlong)track; } // Shared body of readBuffer/readChunk: pins the byte array and feeds libass. @@ -132,10 +143,54 @@ JNIEXPORT jlong JNICALL Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackNex // --- AssRender --- +// The fork's fontconfig build has no Android font search defaults. A tiny +// process-local config lets /system fonts resolve without adding Context/JNI plumbing. +static char* ensureFontsConf(void) { + const char* tmp = getenv("TMPDIR"); + if (tmp == NULL || tmp[0] == '\0') tmp = "/data/local/tmp"; + + char cacheDir[PATH_MAX]; + snprintf(cacheDir, sizeof(cacheDir), "%s/fontconfig", tmp); + mkdir(cacheDir, 0700); + + char* confPath = (char*)malloc(PATH_MAX); + if (confPath == NULL) return NULL; + snprintf(confPath, PATH_MAX, "%s/fonts.conf", tmp); + + FILE* f = fopen(confPath, "w"); + if (f == NULL) { + free(confPath); + return NULL; + } + fprintf( + f, + "\n" + "\n" + "\n" + " /system/fonts\n" + " /system/font\n" + " /product/fonts\n" + " /data/fonts\n" + " %s\n" + "\n", + cacheDir); + fclose(f); + return confPath; +} + JNIEXPORT jlong JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderInit(JNIEnv* env, jclass clazz, jlong ass) { ASS_Renderer* assRenderer = ass_renderer_init((ASS_Library*)ass); - ass_set_fonts(assRenderer, NULL, "sans-serif", ASS_FONTPROVIDER_FONTCONFIG, NULL, 1); + if (assRenderer == NULL) return 0; + unsigned threads = ass_set_threads(assRenderer, 0); + if (threads == 0) { + __android_log_print(ANDROID_LOG_WARN, LOG_TAG, "libass threading unavailable in native build"); + } else { + __android_log_print(ANDROID_LOG_INFO, LOG_TAG, "libass rendering threads enabled: %u", threads); + } + char* fontsConf = ensureFontsConf(); + ass_set_fonts(assRenderer, NULL, "sans-serif", ASS_FONTPROVIDER_FONTCONFIG, fontsConf, 1); + free(fontsConf); return (jlong)assRenderer; } @@ -195,6 +250,13 @@ static int comparePackItemsByHeightDesc(const void* a, const void* b) { return ib->img->h - ia->img->h; } +static int imageListHasOutput(ASS_Image* image) { + for (ASS_Image* img = image; img != NULL; img = img->next) { + if (img->w > 0 && img->h > 0) return 1; + } + return 0; +} + // Throttle for truncation warnings (shared across renderers; logging only). static int truncationLogCounter = 0; @@ -214,8 +276,9 @@ static int truncationLogCounter = 0; // Never fails on content size: images that don't fit the remaining atlas/vertex // capacity are dropped and counted in AssAtlasFrame.truncated, so a heavy frame // degrades instead of going stale. Returns NULL only for missing buffers/handles. -// On changed == 0, returns (0, 0, 0, changed, 0) without touching the buffers — -// caller reuses the atlas texture already on the GPU. +// On changed == 0, returns (0, 0, 0, changed, 0, hasOutput) without touching the +// buffers. hasOutput lets Kotlin distinguish "reuse the previous atlas" from +// "the current frame is blank and the GL surface must be cleared." JNIEXPORT jobject JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderFrameAtlas( JNIEnv* env, jclass clazz, jlong render, jlong track, jlong time, jobject atlasBuf, jint atlasMaxW, jint atlasMaxH, jobject vertexBuf) { @@ -223,7 +286,7 @@ JNIEXPORT jobject JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRende jclass atlasFrameClass = (*env)->FindClass(env, "com/edde746/plezy/libass/AssAtlasFrame"); if (!atlasFrameClass) return NULL; - jmethodID ctor = (*env)->GetMethodID(env, atlasFrameClass, "", "(IIIII)V"); + jmethodID ctor = (*env)->GetMethodID(env, atlasFrameClass, "", "(IIIIIZ)V"); if (!ctor) return NULL; const long long t0 = nowMs(); @@ -231,13 +294,23 @@ JNIEXPORT jobject JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRende ASS_Image* image = ass_render_frame((ASS_Renderer*)render, (ASS_Track*)track, time, &changed); const long long tAss = nowMs(); - if (changed == 0 || image == NULL) { + if (changed == 0) { + const jboolean hasOutput = imageListHasOutput(image) ? JNI_TRUE : JNI_FALSE; + if (tAss - t0 > 40) { + __android_log_print( + ANDROID_LOG_WARN, LOG_TAG, "slow render t=%lldms: ass=%lldms (changed=%d, hasOutput=%d)", + (long long)time, tAss - t0, changed, hasOutput == JNI_TRUE); + } + return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, 0, hasOutput); + } + + if (image == NULL) { if (tAss - t0 > 40) { __android_log_print( ANDROID_LOG_WARN, LOG_TAG, "slow render t=%lldms: ass=%lldms (changed=%d, no output)", (long long)time, tAss - t0, changed); } - return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, 0); + return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, 0, JNI_FALSE); } uint8_t* atlasPixels = (uint8_t*)(*env)->GetDirectBufferAddress(env, atlasBuf); @@ -259,7 +332,7 @@ JNIEXPORT jobject JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRende if (img->w > 0 && img->h > 0) total++; } if (total == 0) { - return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, 0); + return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, 0, JNI_FALSE); } // Pass 1: assign packing slots in height-sorted order so mixed-size frames pack @@ -324,7 +397,7 @@ JNIEXPORT jobject JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRende free(items); free(slotX); free(slotY); - return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, truncated); + return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, truncated, JNI_TRUE); } memset(atlasPixels, 0, (size_t)atlasMaxW * packedH); @@ -432,5 +505,141 @@ JNIEXPORT jobject JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRende // atlasWidth is the full row stride (GLES2 can't upload with stride ≠ width); // atlasHeight is the packed height — the only rows worth uploading. - return (*env)->NewObject(env, atlasFrameClass, ctor, atlasMaxW, packedH, qi, changed, truncated); + return (*env)->NewObject(env, atlasFrameClass, ctor, atlasMaxW, packedH, qi, changed, truncated, JNI_TRUE); +} + +// --- AssFrameTimestamps (EGL_ANDROID_get_frame_timestamps) --- +// +// Measures the overlay buffer's ACTUAL on-screen present time so subtitle +// frame-perfection can be checked against the video frame's release time as +// ground truth, instead of the queue time (eglSwapBuffers return) the swap loop +// otherwise sees. The Java EGLExt only exposes eglPresentationTimeANDROID, so the +// frame-timestamp entry points are resolved here via eglGetProcAddress. +// +// All functions run on the GL thread with the pipeline's EGL context current. + +// Older NDK eglext.h may predate the extension; fall back to the spec values. +#ifndef EGL_TIMESTAMPS_ANDROID +#define EGL_TIMESTAMPS_ANDROID 0x3430 +#endif +#ifndef EGL_COMPOSITION_LATCH_TIME_ANDROID +#define EGL_COMPOSITION_LATCH_TIME_ANDROID 0x3436 +#endif +#ifndef EGL_FIRST_COMPOSITION_START_TIME_ANDROID +#define EGL_FIRST_COMPOSITION_START_TIME_ANDROID 0x3437 +#endif +#ifndef EGL_DISPLAY_PRESENT_TIME_ANDROID +#define EGL_DISPLAY_PRESENT_TIME_ANDROID 0x343A +#endif +#ifndef EGL_TIMESTAMP_INVALID_ANDROID +#define EGL_TIMESTAMP_INVALID_ANDROID (-1) +#endif +#ifndef EGL_TIMESTAMP_PENDING_ANDROID +#define EGL_TIMESTAMP_PENDING_ANDROID (-2) +#endif +#ifndef EGL_ANDROID_get_frame_timestamps +typedef khronos_stime_nanoseconds_t EGLnsecsANDROID; +typedef EGLBoolean(EGLAPIENTRYP PFNEGLGETNEXTFRAMEIDANDROIDPROC)(EGLDisplay, EGLSurface, EGLuint64KHR*); +typedef EGLBoolean(EGLAPIENTRYP PFNEGLGETFRAMETIMESTAMPSANDROIDPROC)( + EGLDisplay, EGLSurface, EGLuint64KHR, EGLint, const EGLint*, EGLnsecsANDROID*); +typedef EGLBoolean(EGLAPIENTRYP PFNEGLGETFRAMETIMESTAMPSUPPORTEDANDROIDPROC)(EGLDisplay, EGLSurface, EGLint); +#endif + +// nativeInit status: success codes are the chosen timestamp source (≥ 0); the +// actual display present time on code 0, and SurfaceFlinger composition timestamps +// (a near-constant ~1-vsync earlier than scanout) on codes 1/2 — a constant bias +// that doesn't hide the inter-layer jitter / multi-vsync outliers we look for. +// Negative codes are failure reasons surfaced to the stats path for diagnosis. +#define FT_SRC_PRESENT 0 +#define FT_SRC_COMPOSITION_START 1 +#define FT_SRC_COMPOSITION_LATCH 2 +#define FT_ERR_NO_SURFACE (-1) +#define FT_ERR_NO_EXTENSION (-2) +#define FT_ERR_NO_PROC (-3) +#define FT_ERR_UNSUPPORTED (-4) +#define FT_ERR_ENABLE_FAILED (-5) + +static PFNEGLGETNEXTFRAMEIDANDROIDPROC pEglGetNextFrameId = NULL; +static PFNEGLGETFRAMETIMESTAMPSANDROIDPROC pEglGetFrameTimestamps = NULL; +static PFNEGLGETFRAMETIMESTAMPSUPPORTEDANDROIDPROC pEglGetFrameTimestampSupported = NULL; +static EGLDisplay gFtDisplay = EGL_NO_DISPLAY; +static EGLSurface gFtSurface = EGL_NO_SURFACE; +static EGLint gFtPresentName = EGL_DISPLAY_PRESENT_TIME_ANDROID; + +// Probes the extension on the currently-current draw surface and enables capture. +// Re-resolves the display/surface each call so surface recreation is handled. +// Returns one of the FT_* codes above. +JNIEXPORT jint JNICALL +Java_com_edde746_plezy_libass_AssFrameTimestamps_nativeInit(JNIEnv* env, jclass clazz) { + gFtSurface = EGL_NO_SURFACE; + EGLDisplay dpy = eglGetCurrentDisplay(); + EGLSurface surf = eglGetCurrentSurface(EGL_DRAW); + if (dpy == EGL_NO_DISPLAY || surf == EGL_NO_SURFACE) return FT_ERR_NO_SURFACE; + + const char* exts = eglQueryString(dpy, EGL_EXTENSIONS); + if (exts == NULL || strstr(exts, "EGL_ANDROID_get_frame_timestamps") == NULL) { + __android_log_print(ANDROID_LOG_INFO, LOG_TAG, "frame-timestamps: extension not present"); + return FT_ERR_NO_EXTENSION; + } + if (pEglGetNextFrameId == NULL) { + pEglGetNextFrameId = (PFNEGLGETNEXTFRAMEIDANDROIDPROC)eglGetProcAddress("eglGetNextFrameIdANDROID"); + pEglGetFrameTimestamps = (PFNEGLGETFRAMETIMESTAMPSANDROIDPROC)eglGetProcAddress("eglGetFrameTimestampsANDROID"); + pEglGetFrameTimestampSupported = + (PFNEGLGETFRAMETIMESTAMPSUPPORTEDANDROIDPROC)eglGetProcAddress("eglGetFrameTimestampSupportedANDROID"); + } + if (pEglGetNextFrameId == NULL || pEglGetFrameTimestamps == NULL || pEglGetFrameTimestampSupported == NULL) { + __android_log_print(ANDROID_LOG_WARN, LOG_TAG, "frame-timestamps: entry points unresolved"); + return FT_ERR_NO_PROC; + } + + // Prefer true display present; many TV HWCs (e.g. Amlogic) don't report present + // fences but do report SurfaceFlinger composition timestamps. Take the first + // supported — composition timing still pins the swap to a vsync for A-vs-B. + jint status; + if (pEglGetFrameTimestampSupported(dpy, surf, EGL_DISPLAY_PRESENT_TIME_ANDROID)) { + gFtPresentName = EGL_DISPLAY_PRESENT_TIME_ANDROID; + status = FT_SRC_PRESENT; + } else if (pEglGetFrameTimestampSupported(dpy, surf, EGL_FIRST_COMPOSITION_START_TIME_ANDROID)) { + gFtPresentName = EGL_FIRST_COMPOSITION_START_TIME_ANDROID; + status = FT_SRC_COMPOSITION_START; + } else if (pEglGetFrameTimestampSupported(dpy, surf, EGL_COMPOSITION_LATCH_TIME_ANDROID)) { + gFtPresentName = EGL_COMPOSITION_LATCH_TIME_ANDROID; + status = FT_SRC_COMPOSITION_LATCH; + } else { + __android_log_print(ANDROID_LOG_INFO, LOG_TAG, "frame-timestamps: no supported timestamp name"); + return FT_ERR_UNSUPPORTED; + } + + if (!eglSurfaceAttrib(dpy, surf, EGL_TIMESTAMPS_ANDROID, EGL_TRUE)) { + __android_log_print(ANDROID_LOG_WARN, LOG_TAG, "frame-timestamps: enable failed 0x%x", eglGetError()); + return FT_ERR_ENABLE_FAILED; + } + gFtDisplay = dpy; + gFtSurface = surf; + __android_log_print(ANDROID_LOG_INFO, LOG_TAG, "frame-timestamps: enabled (source=%d)", status); + return status; +} + +// Frame id the next eglSwapBuffers will produce; call immediately before it. +JNIEXPORT jlong JNICALL +Java_com_edde746_plezy_libass_AssFrameTimestamps_nativeGetNextFrameId(JNIEnv* env, jclass clazz) { + if (pEglGetNextFrameId == NULL || gFtSurface == EGL_NO_SURFACE) return -1; + EGLuint64KHR id = 0; + if (!pEglGetNextFrameId(gFtDisplay, gFtSurface, &id)) return -1; + return (jlong)id; +} + +// Present (or composition) time for frameId (System.nanoTime() domain), or the +// PENDING(-2)/INVALID(-1) sentinels. Reported a few frames after the swap. +JNIEXPORT jlong JNICALL +Java_com_edde746_plezy_libass_AssFrameTimestamps_nativeGetDisplayPresentTime( + JNIEnv* env, jclass clazz, jlong frameId) { + if (pEglGetFrameTimestamps == NULL || gFtSurface == EGL_NO_SURFACE) return EGL_TIMESTAMP_INVALID_ANDROID; + const EGLint names[1] = {gFtPresentName}; + EGLnsecsANDROID values[1] = {0}; + if (!pEglGetFrameTimestamps(gFtDisplay, gFtSurface, (EGLuint64KHR)frameId, 1, names, values)) { + // Frame id evicted from SurfaceFlinger's history, or bad surface. + return EGL_TIMESTAMP_INVALID_ANDROID; + } + return (jlong)values[0]; } diff --git a/android/libass/src/main/cpp/CMakeLists.txt b/android/libass/src/main/cpp/CMakeLists.txt index 05cbbd75..261a5c45 100644 --- a/android/libass/src/main/cpp/CMakeLists.txt +++ b/android/libass/src/main/cpp/CMakeLists.txt @@ -2,9 +2,43 @@ cmake_minimum_required(VERSION 3.22.1) project("asskt") -# libass headers + libass.so come from the io.github.peerless2012:ass AAR via prefab. -add_library(${CMAKE_PROJECT_NAME} SHARED AssKt.c) -find_package(lib_ass REQUIRED CONFIG) +# Fetching the fork release at configure time keeps per-ABI static archives out +# of git; the pinned hash makes that network step reproducible. +set(LIBASS_VERSION "0.18.1") +set(LIBASS_SHA256 "56088619d3907fabbede978d053eca842c1acdda2fea8f11a1e4e1c5722f0624") +set(LIBASS_ZIP_URL "https://github.com/edde746/libass/releases/download/${LIBASS_VERSION}/libass-android-${LIBASS_VERSION}.zip") + +# Gradle passes one cache root so per-ABI CMake builds share the download. +if(NOT DEFINED LIBASS_CACHE_DIR) + set(LIBASS_CACHE_DIR "${CMAKE_BINARY_DIR}/libass-prebuilt") +endif() +set(LIBASS_ROOT "${LIBASS_CACHE_DIR}/libass-android-${LIBASS_VERSION}") + +if(NOT EXISTS "${LIBASS_ROOT}/lib/${ANDROID_ABI}/libass.a") + set(_libass_zip "${LIBASS_CACHE_DIR}/libass-android-${LIBASS_VERSION}.zip") + message(STATUS "Fetching fork libass ${LIBASS_VERSION} from ${LIBASS_ZIP_URL}") + file(DOWNLOAD "${LIBASS_ZIP_URL}" "${_libass_zip}" + EXPECTED_HASH "SHA256=${LIBASS_SHA256}" + TLS_VERIFY ON + STATUS _libass_dl) + list(GET _libass_dl 0 _libass_dl_code) + if(NOT _libass_dl_code EQUAL 0) + file(REMOVE "${_libass_zip}") + message(FATAL_ERROR "Failed to download libass: ${_libass_dl}") + endif() + file(ARCHIVE_EXTRACT INPUT "${_libass_zip}" DESTINATION "${LIBASS_CACHE_DIR}") +endif() + +add_library(ass STATIC IMPORTED) +set_target_properties(ass PROPERTIES + IMPORTED_LOCATION "${LIBASS_ROOT}/lib/${ANDROID_ABI}/libass.a") +target_include_directories(ass INTERFACE "${LIBASS_ROOT}/include") + +add_library(${CMAKE_PROJECT_NAME} SHARED AssKt.c SurfaceTxProbe.c) +# HarfBuzz brings C++; link the shared STL that the app already packages. +# (SurfaceTxProbe resolves its libandroid/libsync entry points via dlsym, so no extra link.) target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE - lib_ass::ass - log) + ass + log + EGL + c++_shared) diff --git a/android/libass/src/main/cpp/SurfaceTxProbe.c b/android/libass/src/main/cpp/SurfaceTxProbe.c new file mode 100644 index 00000000..b0cc882d --- /dev/null +++ b/android/libass/src/main/cpp/SurfaceTxProbe.c @@ -0,0 +1,321 @@ +// EXPERIMENTAL SPIKE — does the codec VIDEO plane's actual present time differ from a +// GL OSD overlay's, and is that difference measurable in-app? (See the memory note +// project_libass_present_time_instrumentation.) +// +// Android exposes no present time for a codec-owned video layer. The one runtime lead is +// API 34 `SurfaceView.applyTransactionToFrame()`: it merges an otherwise-empty, +// callback-bearing transaction with the codec's *next buffer* transaction. The completion +// stats then carry, per surface, the PREVIOUS-RELEASE fence — the moment the buffer the new +// frame replaced stopped being read by the display ≈ when the new codec frame actually +// reached the video plane. Compared in Kotlin to that frame's `releaseTimeNs` target (the +// overlay's own present error is ~0), the delta is the inter-plane lag. +// +// The SurfaceControl transaction-stats / fromJava APIs are API 29/34 while this module's +// minSdk is 21; clang's __builtin_available doesn't guard the Android availability domain in +// this toolchain. So the libandroid entry points are resolved by dlsym (no +// availability-attributed declarations get compiled). Every call site is additionally +// hard-gated to API >= 34 in Kotlin (VideoLayerLatencyProbe). +// +// Fence timestamps are read with the raw SYNC_IOC_FILE_INFO ioctl rather than libsync: +// libsync.so is NOT a public NDK library, so an app's linker namespace blocks dlopen of it. +// +// Compiled into the libass "asskt" .so; called from app Kotlin SurfaceTxProbe. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LOG_TAG "SurfaceTxProbe" +#define LOGW(...) __android_log_print(ANDROID_LOG_WARN, LOG_TAG, __VA_ARGS__) + +#ifndef SYNC_IOC_MAGIC +#define SYNC_IOC_MAGIC '>' +#endif +struct stp_fence_info { + char obj_name[32]; + char driver_name[32]; + int32_t status; + uint32_t flags; + uint64_t timestamp_ns; +}; +struct stp_file_info { + char name[32]; + int32_t status; + uint32_t flags; + uint32_t num_fences; + uint32_t pad; + uint64_t sync_fence_info; +}; +#ifndef SYNC_IOC_FILE_INFO +#define SYNC_IOC_FILE_INFO _IOWR(SYNC_IOC_MAGIC, 4, struct stp_file_info) +#endif + +// libsync is hidden from app linker namespaces, so use the kernel ioctl directly. +static int64_t readFenceTimeNs(int fd, int* outStatus) { + *outStatus = -1; + if (fd < 0) return -1; + struct stp_file_info probe; + memset(&probe, 0, sizeof(probe)); + if (ioctl(fd, SYNC_IOC_FILE_INFO, &probe) < 0) return -1; + *outStatus = probe.status; + if (probe.status != 1 || probe.num_fences == 0) return -1; + struct stp_fence_info* arr = + (struct stp_fence_info*)calloc(probe.num_fences, sizeof(struct stp_fence_info)); + if (!arr) return -1; + struct stp_file_info req; + memset(&req, 0, sizeof(req)); + req.num_fences = probe.num_fences; + req.sync_fence_info = (uint64_t)(uintptr_t)arr; + int64_t t = -1; + if (ioctl(fd, SYNC_IOC_FILE_INFO, &req) == 0) { + for (uint32_t i = 0; i < req.num_fences; i++) { + if ((int64_t)arr[i].timestamp_ns > t) t = (int64_t)arr[i].timestamp_ns; + } + } + free(arr); + return t; +} + +typedef struct ASurfaceTransaction ASurfaceTransaction; +typedef struct ASurfaceControl ASurfaceControl; +typedef struct ASurfaceTransactionStats ASurfaceTransactionStats; +typedef void (*OnCompleteFn)(void* context, ASurfaceTransactionStats* stats); + +typedef ASurfaceTransaction* (*pf_fromJava)(JNIEnv*, jobject); +typedef void (*pf_setOnComplete)(ASurfaceTransaction*, void*, OnCompleteFn); +typedef int64_t (*pf_latchTime)(ASurfaceTransactionStats*); +typedef void (*pf_getControls)(ASurfaceTransactionStats*, ASurfaceControl***, size_t*); +typedef int (*pf_prevRelease)(ASurfaceTransactionStats*, ASurfaceControl*); +typedef void (*pf_releaseControls)(ASurfaceControl**); + +static pf_fromJava p_fromJava = NULL; +static pf_setOnComplete p_setOnComplete = NULL; +static pf_latchTime p_latchTime = NULL; +static pf_getControls p_getControls = NULL; +static pf_prevRelease p_prevRelease = NULL; +static pf_releaseControls p_releaseControls = NULL; +static int gScResolved = 0; +static int gScOk = 0; +static int gScWarningLogged = 0; + +// Runtime lookup keeps this API-34 probe buildable with the module's minSdk 21. +static int resolveSc(void) { + if (gScResolved) return gScOk; + gScResolved = 1; + void* h = dlopen("libandroid.so", RTLD_NOW | RTLD_GLOBAL); + if (!h) return 0; + p_fromJava = (pf_fromJava)dlsym(h, "ASurfaceTransaction_fromJava"); + p_setOnComplete = (pf_setOnComplete)dlsym(h, "ASurfaceTransaction_setOnComplete"); + p_latchTime = (pf_latchTime)dlsym(h, "ASurfaceTransactionStats_getLatchTime"); + p_getControls = (pf_getControls)dlsym(h, "ASurfaceTransactionStats_getASurfaceControls"); + p_prevRelease = (pf_prevRelease)dlsym(h, "ASurfaceTransactionStats_getPreviousReleaseFenceFd"); + p_releaseControls = (pf_releaseControls)dlsym(h, "ASurfaceTransactionStats_releaseASurfaceControls"); + gScOk = (p_fromJava && p_setOnComplete && p_latchTime && p_getControls && p_prevRelease && + p_releaseControls) + ? 1 + : 0; + return gScOk; +} + +// The completion context is pointer-width, so 32-bit devices need an index side-channel. +#define TAG_RING_SIZE 256 +static int64_t gTagRing[TAG_RING_SIZE]; +static int gSrcRing[TAG_RING_SIZE]; +static int gTagSeq = 0; + +static JavaVM* gVm = NULL; +static jclass gProbeClass = NULL; +static jmethodID gOnResult = NULL; +static int gCallbackLookupAttempted = 0; +static pthread_mutex_t gInitLock = PTHREAD_MUTEX_INITIALIZER; + +static int64_t nowMonoNs(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (int64_t)ts.tv_sec * 1000000000LL + ts.tv_nsec; +} + +static void reportResult(JNIEnv* env, int64_t tag, int64_t latchNs, int64_t releaseNs, int count, + int fenceState, int source, int64_t cbNs) { + if (!env || !gProbeClass || !gOnResult) return; + (*env)->CallStaticVoidMethod(env, gProbeClass, gOnResult, (jlong)tag, (jlong)latchNs, + (jlong)releaseNs, (jint)count, (jint)fenceState, (jint)source, + (jlong)cbNs); + if ((*env)->ExceptionCheck(env)) (*env)->ExceptionClear(env); +} + +// Previous-release fences can still be pending in the binder callback; a worker +// thread keeps SurfaceFlinger's callback dispatch unblocked. +struct fenceJob { + int fd; + int64_t tag; + int64_t latch; + int count; + int source; +}; +#define JOB_RING 64 +static struct fenceJob gJobs[JOB_RING]; +static int gJobHead = 0, gJobTail = 0; +static int gJobDrops = 0; +static pthread_mutex_t gJobLock = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t gJobCond = PTHREAD_COND_INITIALIZER; +static pthread_t gReader; +static int gReaderStarted = 0; + +static void* readerMain(void* arg); + +static int ensureProbeInit(JNIEnv* env, jclass clazz) { + int scOk = 0; + pthread_mutex_lock(&gInitLock); + if (!gVm && (*env)->GetJavaVM(env, &gVm) != JNI_OK) { + LOGW("GetJavaVM failed — transaction probe disabled"); + pthread_mutex_unlock(&gInitLock); + return 0; + } + if (!gProbeClass) { + gProbeClass = (*env)->NewGlobalRef(env, clazz); + if (!gProbeClass) { + LOGW("NewGlobalRef(SurfaceTxProbe) failed — transaction probe disabled"); + pthread_mutex_unlock(&gInitLock); + return 0; + } + } + if (!gCallbackLookupAttempted) { + gCallbackLookupAttempted = 1; + gOnResult = (*env)->GetStaticMethodID(env, clazz, "onResult", "(JJJIIIJ)V"); + if (!gOnResult) { + (*env)->ExceptionClear(env); + LOGW("onResult(JJJIIIJ)V not found — JNI callback disabled (R8 stripped it?)"); + } + } + if (!gOnResult) { + pthread_mutex_unlock(&gInitLock); + return 0; + } + scOk = resolveSc(); + if (!scOk) { + if (!gScWarningLogged) { + gScWarningLogged = 1; + LOGW("libandroid SurfaceControl transaction-stats API unavailable"); + } + pthread_mutex_unlock(&gInitLock); + return 0; + } + if (!gReaderStarted) { + const int rc = pthread_create(&gReader, NULL, readerMain, NULL); + if (rc != 0) { + LOGW("fence reader thread start failed: %d", rc); + pthread_mutex_unlock(&gInitLock); + return 0; + } + gReaderStarted = 1; + } + pthread_mutex_unlock(&gInitLock); + return scOk; +} + +static void* readerMain(void* arg) { + (void)arg; + JNIEnv* env = NULL; + if ((*gVm)->AttachCurrentThread(gVm, &env, NULL) != JNI_OK) return NULL; + for (;;) { + pthread_mutex_lock(&gJobLock); + while (gJobHead == gJobTail) pthread_cond_wait(&gJobCond, &gJobLock); + struct fenceJob job = gJobs[gJobTail]; + gJobTail = (gJobTail + 1) % JOB_RING; + pthread_mutex_unlock(&gJobLock); + + int64_t releaseNs = -1; + int fenceState; + struct pollfd pfd = {.fd = job.fd, .events = POLLIN, .revents = 0}; + int pr = poll(&pfd, 1, 250); + if (pr > 0 && (pfd.revents & POLLIN)) { + int st = -1; + releaseNs = readFenceTimeNs(job.fd, &st); + fenceState = (releaseNs > 0) ? 3 : 1; + } else { + fenceState = 2; + } + close(job.fd); + reportResult(env, job.tag, job.latch, releaseNs, job.count, fenceState, job.source, nowMonoNs()); + } + return NULL; +} + +static void onComplete(void* context, ASurfaceTransactionStats* stats) { + int slot = (int)(intptr_t)context; + __atomic_thread_fence(__ATOMIC_ACQUIRE); + int64_t tag = gTagRing[slot & (TAG_RING_SIZE - 1)]; + int source = gSrcRing[slot & (TAG_RING_SIZE - 1)]; + int64_t latchNs = p_latchTime ? p_latchTime(stats) : -1; + + // Display-global present fences are abort-prone; per-surface release fences are stable here. + int chosenFd = -1; + int count = 0; + ASurfaceControl** controls = NULL; + size_t n = 0; + if (p_getControls) p_getControls(stats, &controls, &n); + count = (int)n; + for (size_t i = 0; i < n; i++) { + int rfd = p_prevRelease ? p_prevRelease(stats, controls[i]) : -1; + if (rfd < 0) continue; + if (chosenFd < 0) + chosenFd = rfd; + else + close(rfd); + } + if (controls && p_releaseControls) p_releaseControls(controls); + + if (chosenFd < 0) { + JNIEnv* env = NULL; + int didAttach = 0; + if ((*gVm)->GetEnv(gVm, (void**)&env, JNI_VERSION_1_6) != JNI_OK) { + if ((*gVm)->AttachCurrentThread(gVm, &env, NULL) != JNI_OK) return; + didAttach = 1; + } + reportResult(env, tag, latchNs, -1, count, 0, source, nowMonoNs()); + if (didAttach) (*gVm)->DetachCurrentThread(gVm); + return; + } + + pthread_mutex_lock(&gJobLock); + int next = (gJobHead + 1) % JOB_RING; + if (next == gJobTail) { + close(chosenFd); + if (gJobDrops++ == 0) LOGW("fence job ring full — dropping samples (slow fence reader?)"); + } else { + gJobs[gJobHead].fd = chosenFd; + gJobs[gJobHead].tag = tag; + gJobs[gJobHead].latch = latchNs; + gJobs[gJobHead].count = count; + gJobs[gJobHead].source = source; + gJobHead = next; + pthread_cond_signal(&gJobCond); + } + pthread_mutex_unlock(&gJobLock); +} + +JNIEXPORT void JNICALL Java_com_edde746_plezy_exoplayer_SurfaceTxProbe_nativeAttach( + JNIEnv* env, jclass clazz, jobject jtransaction, jlong tag, jint source) { + if (!ensureProbeInit(env, clazz)) return; + int slot = __atomic_fetch_add(&gTagSeq, 1, __ATOMIC_RELAXED) & (TAG_RING_SIZE - 1); + gTagRing[slot] = (int64_t)tag; + gSrcRing[slot] = (int)source; + // Binder carries only the slot index, so the ring write has to win publication. + __atomic_thread_fence(__ATOMIC_RELEASE); + + ASurfaceTransaction* tx = p_fromJava(env, jtransaction); + if (!tx) { + LOGW("ASurfaceTransaction_fromJava returned null"); + return; + } + p_setOnComplete(tx, (void*)(intptr_t)slot, onComplete); +} diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/AssAtlasFrame.kt b/android/libass/src/main/java/com/edde746/plezy/libass/AssAtlasFrame.kt index af628d94..8d259636 100644 --- a/android/libass/src/main/java/com/edde746/plezy/libass/AssAtlasFrame.kt +++ b/android/libass/src/main/java/com/edde746/plezy/libass/AssAtlasFrame.kt @@ -11,11 +11,23 @@ package com.edde746.plezy.libass * @param truncated images dropped because they exceeded the atlas/vertex capacity; * the frame is incomplete but never stale (> 0 should be rare — * it means even the GL-max-sized atlas couldn't fit the frame) + * @param hasOutput true when libass reported at least one visible image for this + * timestamp, even when [changed] is 0 and the buffers were not + * rewritten. false means this timestamp should be blank. */ class AssAtlasFrame( val atlasWidth: Int, val atlasHeight: Int, val quadCount: Int, val changed: Int, - val truncated: Int -) + val truncated: Int, + val hasOutput: Boolean +) { + constructor( + atlasWidth: Int, + atlasHeight: Int, + quadCount: Int, + changed: Int, + truncated: Int + ) : this(atlasWidth, atlasHeight, quadCount, changed, truncated, quadCount > 0) +} diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/AssFrameTimestamps.kt b/android/libass/src/main/java/com/edde746/plezy/libass/AssFrameTimestamps.kt new file mode 100644 index 00000000..ed197410 --- /dev/null +++ b/android/libass/src/main/java/com/edde746/plezy/libass/AssFrameTimestamps.kt @@ -0,0 +1,71 @@ +package com.edde746.plezy.libass + +/** + * Thin binding over the `EGL_ANDROID_get_frame_timestamps` extension. + * + * The atlas pipeline pins each overlay swap to the video frame's `releaseTimeNs`, + * but the swap loop can only measure when [android.opengl.EGL14.eglSwapBuffers] + * *returns* — i.e. when the buffer was queued, not when SurfaceFlinger actually + * presented it. That can't reveal a one-vsync compositor latch offset, which is + * exactly the error frame-perfection work chases. This binding reads the buffer's + * *actual on-screen present time* so it can be compared against `releaseTimeNs` + * (the video frame's vsync target) as ground truth. + * + * `android.opengl.EGLExt` exposes only `eglPresentationTimeANDROID`; the + * frame-timestamp entry points are resolved natively via `eglGetProcAddress`. + * + * Requires Android 10 (API 29)+ and the extension; probe with [nativeInit]. + * All calls must run on the GL thread with the pipeline's EGL context current. + */ +internal object AssFrameTimestamps { + /** Present time not yet reported by SurfaceFlinger; query the frame again later. */ + const val PENDING = -2L + + /** The frame was dropped/never presented, or its id was evicted from history. */ + const val INVALID = -1L + + // [nativeInit] status. Success codes are the timestamp source in use (≥ 0): + // true scanout present (SRC_PRESENT) or — where the HWC reports no present fence, + // common on TV SoCs — a SurfaceFlinger composition timestamp ~1 vsync earlier + // (SRC_COMPOSITION_*; a constant bias, harmless to the jitter/outlier analysis). + // Negative codes are failure reasons, surfaced to the stats path for diagnosis. + const val SRC_PRESENT = 0 + const val SRC_COMPOSITION_START = 1 + const val SRC_COMPOSITION_LATCH = 2 + const val ERR_NO_SURFACE = -1 + const val ERR_NO_EXTENSION = -2 + const val ERR_NO_PROC = -3 + const val ERR_UNSUPPORTED = -4 + const val ERR_ENABLE_FAILED = -5 + + /** Short label for a [nativeInit] status code, for logs/getStats. */ + fun sourceLabel(status: Int): String = when (status) { + SRC_PRESENT -> "present" + SRC_COMPOSITION_START -> "comp-start" + SRC_COMPOSITION_LATCH -> "comp-latch" + ERR_NO_SURFACE -> "off:no-surface" + ERR_NO_EXTENSION -> "off:no-ext" + ERR_NO_PROC -> "off:no-proc" + ERR_UNSUPPORTED -> "off:unsupported" + ERR_ENABLE_FAILED -> "off:enable-failed" + else -> "off:$status" + } + + /** + * Probes the extension on the currently-current draw surface and enables + * timestamp capture, preferring true present then composition timestamps. + * Returns an `SRC_*` code (≥ 0) on success or an `ERR_*` code (< 0). Re-resolves + * the surface each call, so it is safe to re-invoke after surface recreation. + */ + @JvmStatic external fun nativeInit(): Int + + /** Frame id the next `eglSwapBuffers` will produce; call immediately before it. */ + @JvmStatic external fun nativeGetNextFrameId(): Long + + /** + * Present (or composition) time for [frameId] in the `System.nanoTime()` domain, + * or the [PENDING]/[INVALID] sentinels. Results are deferred a few frames past + * the swap, so a caller drains pending ids lazily. + */ + @JvmStatic external fun nativeGetDisplayPresentTime(frameId: Long): Long +} diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/media/AssHandler.kt b/android/libass/src/main/java/com/edde746/plezy/libass/media/AssHandler.kt index 6ed0c365..1a0cf98c 100644 --- a/android/libass/src/main/java/com/edde746/plezy/libass/media/AssHandler.kt +++ b/android/libass/src/main/java/com/edde746/plezy/libass/media/AssHandler.kt @@ -17,6 +17,7 @@ import com.edde746.plezy.libass.Ass import com.edde746.plezy.libass.AssRender import com.edde746.plezy.libass.AssTrack import com.edde746.plezy.libass.media.parser.AssHeaderParser +import com.edde746.plezy.libass.media.widget.AssAtlasPipelineConfig /** * Handles ASS subtitle rendering and integration with ExoPlayer. @@ -185,13 +186,15 @@ class AssHandler( * (re)created or the selected track changes, so renderer state survives both. */ private fun applyRenderState(render: AssRender) { + // Storage (script/video authoring size) stays full-res; only the frame and + // margins follow RENDER_SCALE so the raster shrinks while layout is unchanged. if (videoSize.isValid) render.setStorageSize(videoSize.width, videoSize.height) when { - surfaceSize.isValid -> render.setFrameSize(surfaceSize.width, surfaceSize.height) + surfaceSize.isValid -> render.setFrameSize(scaledForRender(surfaceSize.width), scaledForRender(surfaceSize.height)) // Fallback frame until the overlay surface reports its size. - videoSize.isValid -> render.setFrameSize(videoSize.width, videoSize.height) + videoSize.isValid -> render.setFrameSize(scaledForRender(videoSize.width), scaledForRender(videoSize.height)) } - margins?.let { m -> render.setMargins(m[0], m[1], m[2], m[3]) } + margins?.let { m -> render.setMargins(scaledForRender(m[0]), scaledForRender(m[1]), scaledForRender(m[2]), scaledForRender(m[3])) } render.setUseMargins(useMargins) } @@ -220,7 +223,7 @@ class AssHandler( if (surfaceSize.width == width && surfaceSize.height == height) return Log.i("AssHandler", "setOverlaySurfaceSize: width = $width, height = $height") surfaceSize = Size(width, height) - render?.setFrameSize(width, height) + render?.setFrameSize(scaledForRender(width), scaledForRender(height)) } /** @@ -230,9 +233,13 @@ class AssHandler( */ fun setMargins(top: Int, bottom: Int, left: Int, right: Int) { margins = intArrayOf(top, bottom, left, right) - render?.setMargins(top, bottom, left, right) + render?.setMargins(scaledForRender(top), scaledForRender(bottom), scaledForRender(left), scaledForRender(right)) } + /** Frame/margin extents go to libass at the overlay's render resolution; the GL + * side scales the result back up. Identity unless RENDER_SCALE < 1. */ + private fun scaledForRender(px: Int): Int = AssAtlasPipelineConfig.scaledForRender(px) + /** mpv's sub-ass-force-margins: anchor non-positioned events to the visible frame. */ fun setUseMargins(use: Boolean) { useMargins = use @@ -383,4 +390,16 @@ class AssHandler( */ private val Size.isValid get() = width > 0 && height > 0 + + companion object { + /** + * Sets the global libass overlay render scale (fraction of the surface resolution; 1.0 = no + * downscale). Cheaper raster for the render-bound tail on weak GPUs, at some sharpness. Set + * before/at playback init — [AssAtlasPipelineConfig.scaledForRender] reads it on the next + * frame-size apply (overlay surface create / size change), so it takes effect from playback. + */ + fun setRenderScale(scale: Float) { + AssAtlasPipelineConfig.renderScale = scale.coerceIn(0.2f, 1.0f) + } + } } diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/AssSubtitleAtlasPipeline.kt b/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/AssSubtitleAtlasPipeline.kt index b3a74011..3d771a4e 100644 --- a/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/AssSubtitleAtlasPipeline.kt +++ b/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/AssSubtitleAtlasPipeline.kt @@ -6,6 +6,7 @@ import android.opengl.EGLDisplay import android.opengl.EGLExt import android.opengl.EGLSurface import android.opengl.GLES20 +import android.os.Build import android.os.Handler import android.os.HandlerThread import android.os.Process @@ -17,18 +18,20 @@ import androidx.media3.common.util.GlUtil import androidx.media3.common.util.Size import androidx.media3.common.util.UnstableApi import com.edde746.plezy.libass.AssAtlasFrame +import com.edde746.plezy.libass.AssFrameTimestamps import com.edde746.plezy.libass.media.AssHandler import java.nio.ByteBuffer import java.nio.ByteOrder +import java.util.concurrent.atomic.AtomicLong import java.util.concurrent.atomic.AtomicReference +import java.util.concurrent.locks.LockSupport /** * Atlas-rendering pipeline behind [AssSubtitleSurfaceView]. * Runs libass on its own [HandlerThread] into a packed * ALPHA_8 texture atlas plus a single vertex stream, and a GL thread that uploads - * both and issues one `glDrawArrays` per frame. Each swap is pinned to the video's - * target release time via [EGLExt.eglPresentationTimeANDROID]: SurfaceFlinger holds - * the buffer and composes it on the same vsync as the corresponding video frame. + * both and issues one `glDrawArrays` per frame. Each timed swap is pinned to the + * corresponding video frame via [EGLExt.eglPresentationTimeANDROID]. */ @UnstableApi internal object AssAtlasPipelineConfig { @@ -67,9 +70,32 @@ internal object AssAtlasPipelineConfig { * core's ~35-50ms renders it converts near-misses into on-time latches. */ internal const val SPECULATION_ENABLED = true + + /** + * Disabled by default: queueing subtitles one predicted video frame ahead made + * Android playback often look one frame early. Keep the estimator behind a flag + * in case a device-specific compositor path later proves it needs this again. + */ + internal const val COMPOSITOR_PHASE_LEAD_ENABLED = false + + /** + * Internal raster resolution for the libass overlay, as a fraction of the + * physical surface (1.0 = full, off). Lowering it shrinks the libass frame so + * heavy/animated signs rasterize over fewer pixels (raster cost ∝ area; layout + * cost unchanged) and the GL upscales the result to the full surface — a + * quality-for-throughput trade for render-bound low-end TVs. Applied + * consistently to the libass frame size, the mpv-style margins ([AssHandler]) + * and the GL `u_SurfaceSize` ([AtlasRenderer]); at 1.0 it is an exact no-op. + * Runtime-settable from the user's "Render Resolution" subtitle setting (Android exposes + * Full / ¾ / ½ / ⅓ / ¼) via [AssHandler.setRenderScale]; 1.0 = exact no-op (the default). + */ + @Volatile + internal var renderScale = 1.0f + + /** Scales a physical-pixel extent to the libass render resolution; identity at 1.0. */ + internal fun scaledForRender(px: Int): Int = if (renderScale == 1.0f) px else Math.round(px * renderScale) } -/** Payload handed from the libass worker to the GL thread. */ internal class AtlasPayload( val slotIndex: Int, val atlasBuf: ByteBuffer, @@ -77,9 +103,38 @@ internal class AtlasPayload( var frame: AssAtlasFrame, var presentationTimeUs: Long, var releaseTimeNs: Long, - /** Bumped on every content-changing render; lets the GL thread tell "same slot, - * new content" apart from "same slot, same content" when deciding to re-upload. */ - var contentSeq: Long = 0L + var sourcePresentationTimeUs: Long = presentationTimeUs, + var phaseLeadUs: Long = 0L, + /** Slot identity alone is not enough because libass rewrites buffers in place. */ + var contentSeq: Long = 0L, + var requestSeq: Long = 0L, + var stateGeneration: Long = 0L +) + +private class AtlasDrawSnapshot( + val atlasBuf: ByteBuffer, + val vertexBuf: ByteBuffer, + val frame: AssAtlasFrame, + val sourcePresentationTimeUs: Long, + val presentationTimeUs: Long, + val releaseTimeNs: Long, + val phaseLeadUs: Long, + val contentSeq: Long, + val requestSeq: Long, + val stateGeneration: Long +) + +private fun AtlasPayload.snapshot(): AtlasDrawSnapshot = AtlasDrawSnapshot( + atlasBuf = atlasBuf, + vertexBuf = vertexBuf, + frame = frame, + sourcePresentationTimeUs = sourcePresentationTimeUs, + presentationTimeUs = presentationTimeUs, + releaseTimeNs = releaseTimeNs, + phaseLeadUs = phaseLeadUs, + contentSeq = contentSeq, + requestSeq = requestSeq, + stateGeneration = stateGeneration ) /** Payload slots plus the atlas dims their buffers were sized for. */ @@ -99,12 +154,23 @@ internal class AssAtlasPipeline( surface: Surface, width: Int, height: Int, - assHandler: AssHandler, - lowRamDevice: Boolean = false + private val assHandler: AssHandler, + lowRamDevice: Boolean = false, + refreshRateHz: Float = 60f ) { private val surfaceWidth = width private val surfaceHeight = height + /** The calibrator needs the final pre-swap point, and this must survive surface recreation. */ + var preSwapProbe: ((releaseTimeNs: Long) -> Unit)? + get() = glThread.preSwapProbe + set(value) { + glThread.preSwapProbe = value + } + + // Present-error buckets need to stay meaningful across display modes. + private val vsyncNs = (1_000_000_000.0 / (if (refreshRateHz >= 1f) refreshRateHz else 60f)).toLong() + // 3 slots give the render-ahead engine a writable target while one slot is // posted and another is in GL's hands; low-RAM devices stay at 2 slots with // speculation off (the legacy on-demand behavior, ~19 MB less in buffers). @@ -147,6 +213,11 @@ internal class AssAtlasPipeline( // first actual render. Confined to the libass thread after creation. private var slots: AtlasSlots? = null + private fun rendererStateGeneration(): Long = + assHandler.render?.let { + (System.identityHashCode(it).toLong() shl 32) or (it.stateGeneration.toLong() and 0xffffffffL) + } ?: -1L + private fun acquireSlots(): AtlasSlots { slots?.let { return it } if (!dimsLatch.await(1, java.util.concurrent.TimeUnit.SECONDS)) { @@ -174,14 +245,18 @@ internal class AssAtlasPipeline( @Volatile private var glLastTakenSlot = -1 private val pendingPayload = AtomicReference(null) + private val latestReadyRequestSeq = AtomicLong(0L) private val glThread = AtlasGlThread( surface, width, height, assHandler, + vsyncNs = vsyncNs, takePending = { pendingPayload.getAndSet(null)?.also { glLastTakenSlot = it.slotIndex } }, + latestReadyRequestSeq = { latestReadyRequestSeq.get() }, + currentStateGeneration = ::rendererStateGeneration, resolveAtlasDims = ::resolveAtlasDims ) private val libassThread = AtlasLibassThread( @@ -189,7 +264,9 @@ internal class AssAtlasPipeline( acquireSlots = ::acquireSlots, speculationEnabled = speculationEnabled, glTakenSlot = { glLastTakenSlot }, + stateGeneration = ::rendererStateGeneration, onFrameReady = { payload -> + latestReadyRequestSeq.set(payload.requestSeq) pendingPayload.set(payload) glThread.triggerDraw() } @@ -259,14 +336,72 @@ internal class AssAtlasPipeline( /** Speculation rounds skipped (paused, pending request, no confident cadence). */ val specSkips: Long get() = libassThread.specSkips + /** changed==0/no-output renders forced into explicit transparent swaps. */ + val blankClearCount: Long get() = libassThread.blankClearCount + /** Cache-warming prefetch renders of upcoming events. */ val prefetchCount: Long get() = libassThread.prefetchCount + /** Frame requests replaced before the libass worker serviced them. */ + val coalescedRequestCount: Long get() = libassThread.coalescedRequestCount + + /** Completed libass results discarded because renderer state changed before handoff. */ + val staleGenerationCount: Long get() = libassThread.staleGenerationCount + + /** Completed overlay snapshots skipped because a newer completed request superseded them before swap. */ + val supersededBeforeSwapCount: Long get() = glThread.supersededBeforeSwapCount + + /** Completed overlay snapshots skipped because renderer state changed before swap. */ + val staleBeforeSwapCount: Long get() = glThread.staleBeforeSwapCount + /** Worst (minimum) lead of a changed-content pinned swap vs its target release * time, in ms; negative = the new content was queued after the video frame's * vsync. Long.MAX_VALUE until a changed pinned swap happened. */ val minLeadChangedMs: Long get() = glThread.minLeadChangedMs + /** Current steady-state compositor phase lead, in ms. */ + val phaseLeadMs: Long get() = libassThread.phaseLeadUs / 1000 + + /** Most recent pinned swap lead vs its target release time, in ms. */ + val lastSwapLeadMs: Long get() = glThread.lastSwapLeadMs + + /** Most recent pinned swap headroom when GL work started, in ms. */ + val lastSwapHeadroomMs: Long get() = glThread.lastSwapHeadroomMs + + /** Most recent phase-led wait before swap, in ms. */ + val lastScheduledSleepMs: Long get() = glThread.lastScheduledSleepMs + + /** Adaptive swap lead actually in effect (half the measured refresh), in ms. */ + val swapLeadMs: Long get() = glThread.swapLeadNs / 1_000_000 + + /** True once the EGL frame-timestamp extension is probed and capturing. */ + val presentTimingEnabled: Boolean get() = glThread.presentTimingEnabled + + /** Active present-time source, or why it's off (present/comp-start/comp-latch/off:…). */ + val presentSource: String get() = glThread.presentSource + + /** Actual on-screen present time of the most recent measured swap minus its + * target release time, in ms (negative = presented before the video frame's + * vsync, positive = after). The frame-perfection ground truth. */ + val lastPresentErrorMs: Long get() = glThread.lastPresentErrorMs + + /** Largest-magnitude present error observed, in ms. */ + val worstPresentErrorMs: Long get() = glThread.worstPresentErrorMs + + /** Pinned swaps whose actual present time was read back from SurfaceFlinger. */ + val presentMeasuredCount: Long get() = glThread.presentMeasuredCount + + /** Swaps SurfaceFlinger reported as dropped/never-presented. */ + val presentInvalidCount: Long get() = glThread.presentInvalidCount + + /** Pending present-time reads evicted unread because the ring filled. */ + val presentDroppedCount: Long get() = glThread.presentDroppedCount + + /** Present-error distribution in vsync-interval units: + * [≤−1.5, (−1.5,−0.5), (−0.5,+0.5), [+0.5,+1.5), ≥+1.5]. A clean spike in the + * middle bucket = frame-perfect; a spike at [+0.5,+1.5) = a 1-vsync late latch. */ + val presentErrorHistogram: List get() = glThread.presentErrorHistogram + fun releaseAndWait() { libassThread.releaseAndWait() glThread.releaseAndWait() @@ -296,6 +431,91 @@ private fun postShutdownAndWait( /** Transport for [postShutdownAndWait] — the handler callback calls [release] then notifies. */ private class Ack(val latch: Any, val release: () -> Unit) +private class PhaseAdjustedFrame( + val sourcePtsUs: Long, + val renderPtsUs: Long, + val releaseNs: Long, + val phaseLeadUs: Long, + val releaseLeadNs: Long +) + +/** + * Predicts the next video frame from the callback cadence. Media3 gives us the + * frame currently being released, but a separate subtitle SurfaceView may not latch + * in the same composition phase. Once cadence is stable, callback N renders and + * timestamps the ASS layer for predicted frame N+1. + */ +private class SubtitleFramePhaseEstimator { + private val ptsDeltasUs = LongArray(DELTA_SAMPLES) + private val releaseDeltasNs = LongArray(DELTA_SAMPLES) + private var deltaCount = 0 + private var deltaIndex = 0 + private var lastPtsUs = UNSET + private var lastReleaseNs = C.TIME_UNSET + + @Synchronized + fun adjust(ptsUs: Long, releaseNs: Long): PhaseAdjustedFrame { + if (!AssAtlasPipelineConfig.COMPOSITOR_PHASE_LEAD_ENABLED || releaseNs == C.TIME_UNSET) { + return PhaseAdjustedFrame(ptsUs, ptsUs, releaseNs, 0L, 0L) + } + + observe(ptsUs, releaseNs) + if (deltaCount < MIN_DELTA_SAMPLES) { + return PhaseAdjustedFrame(ptsUs, ptsUs, releaseNs, 0L, 0L) + } + + val ptsLeadUs = median(ptsDeltasUs, deltaCount) + val releaseLeadNs = median(releaseDeltasNs, deltaCount) + return PhaseAdjustedFrame( + sourcePtsUs = ptsUs, + renderPtsUs = saturatedAdd(ptsUs, ptsLeadUs), + releaseNs = saturatedAdd(releaseNs, releaseLeadNs), + phaseLeadUs = ptsLeadUs, + releaseLeadNs = releaseLeadNs + ) + } + + private fun observe(ptsUs: Long, releaseNs: Long) { + val prevPtsUs = lastPtsUs + val prevReleaseNs = lastReleaseNs + lastPtsUs = ptsUs + lastReleaseNs = releaseNs + if (prevPtsUs == UNSET || prevReleaseNs == C.TIME_UNSET) return + + val ptsDeltaUs = ptsUs - prevPtsUs + val releaseDeltaNs = releaseNs - prevReleaseNs + if (ptsDeltaUs <= 0 || ptsDeltaUs > MAX_DELTA_US || + releaseDeltaNs <= 0 || releaseDeltaNs > MAX_DELTA_NS + ) { + deltaCount = 0 + deltaIndex = 0 + return + } + + ptsDeltasUs[deltaIndex] = ptsDeltaUs + releaseDeltasNs[deltaIndex] = releaseDeltaNs + deltaIndex = (deltaIndex + 1) % DELTA_SAMPLES + if (deltaCount < DELTA_SAMPLES) deltaCount++ + } + + private fun median(values: LongArray, count: Int): Long { + val copy = values.copyOfRange(0, count) + copy.sort() + return copy[count / 2] + } + + private fun saturatedAdd(value: Long, delta: Long): Long = + if (delta > 0 && value > Long.MAX_VALUE - delta) Long.MAX_VALUE else value + delta + + private companion object { + const val UNSET = Long.MIN_VALUE + const val DELTA_SAMPLES = 8 + const val MIN_DELTA_SAMPLES = 4 + const val MAX_DELTA_US = 250_000L + const val MAX_DELTA_NS = 250_000_000L + } +} + /** * Runs libass off the GL thread into a packed atlas + vertex stream. Latest-wins: * older pending renders are dropped when a newer one arrives. Slot choice, the @@ -308,30 +528,54 @@ private class AtlasLibassThread( private val acquireSlots: () -> AtlasSlots, private val speculationEnabled: Boolean, private val glTakenSlot: () -> Int, + private val stateGeneration: () -> Long, private val onFrameReady: (AtlasPayload) -> Unit ) : HandlerThread(TAG, Process.THREAD_PRIORITY_DISPLAY) { - /** Immutable (pts, release) request — handed off through a single atomic so a - * concurrent enqueue can neither be lost by drain's consume nor torn in half. - * [enqueueNs] timestamps the handoff so drain can report how long the request - * sat behind an in-flight render (the queue-wait component of subtitle lag). */ - private class PendingFrame(val ptsUs: Long, val releaseNs: Long, val enqueueNs: Long = System.nanoTime()) + /** Immutable frame request — handed off through a single atomic so a concurrent + * enqueue can neither be lost by drain's consume nor torn in half. [ptsUs] is + * the libass render timestamp after any phase lead; [sourcePtsUs] is the video + * callback PTS after user subtitle delay but before the compositor lead. + * [sequence] is the selected-video-frame request identity, and [enqueueNs] + * timestamps the handoff so drain can report queue wait. */ + private class PendingFrame( + val sourcePtsUs: Long, + val ptsUs: Long, + val releaseNs: Long, + val phaseLeadUs: Long, + val releaseLeadNs: Long, + val sequence: Long, + val enqueueNs: Long = System.nanoTime() + ) private lateinit var handler: Handler private val pending = AtomicReference(null) + private val phaseEstimator = SubtitleFramePhaseEstimator() + private val requestSeqCounter = AtomicLong(0L) @Volatile private var lastRequestedPtsUs = UNSET private var contentSeqCounter = 0L // Thread-confined; created on first render so non-ASS playback never allocates. private var engine: SpecRenderEngine? = null + private var engineStateGeneration = Long.MIN_VALUE val specHits: Long get() = engine?.specHits ?: 0L val specMisses: Long get() = engine?.specMisses ?: 0L val specSkips: Long get() = engine?.specSkips ?: 0L + val blankClearCount: Long get() = engine?.blankClearCount ?: 0L val prefetchCount: Long get() = engine?.prefetchCount ?: 0L + @Volatile var coalescedRequestCount = 0L + private set + + @Volatile var staleGenerationCount = 0L + private set + + @Volatile var phaseLeadUs = 0L + private set + // Telemetry; single-writer (this thread), read from the stats path. @Volatile var renderCount = 0L private set @@ -382,15 +626,31 @@ private class AtlasLibassThread( fun enqueue(presentationTimeUs: Long, releaseTimeNs: Long) { if (!::handler.isInitialized) return - val dropped = pending.getAndSet(PendingFrame(presentationTimeUs, releaseTimeNs)) - if (dropped != null && AssAtlasPipelineConfig.TIMING_LOGS) { - // A request was coalesced away — the renderer is behind by at least one - // frame. agedMs = how long the dropped request had been waiting. - Log.d( - TAG, - "drop pts=${dropped.ptsUs / 1000}ms agedMs=${(System.nanoTime() - dropped.enqueueNs) / 1_000_000} " + - "replacedBy=${presentationTimeUs / 1000}ms" + val adjusted = phaseEstimator.adjust(presentationTimeUs, releaseTimeNs) + phaseLeadUs = adjusted.phaseLeadUs + val sequence = requestSeqCounter.incrementAndGet() + val dropped = pending.getAndSet( + PendingFrame( + sourcePtsUs = adjusted.sourcePtsUs, + ptsUs = adjusted.renderPtsUs, + releaseNs = adjusted.releaseNs, + phaseLeadUs = adjusted.phaseLeadUs, + releaseLeadNs = adjusted.releaseLeadNs, + sequence = sequence ) + ) + if (dropped != null) { + coalescedRequestCount++ + if (AssAtlasPipelineConfig.TIMING_LOGS) { + // A request was coalesced away — the renderer is behind by at least one + // frame. agedMs = how long the dropped request had been waiting. + Log.d( + TAG, + "drop seq=${dropped.sequence} src=${dropped.sourcePtsUs / 1000}ms pts=${dropped.ptsUs / 1000}ms " + + "agedMs=${(System.nanoTime() - dropped.enqueueNs) / 1_000_000} " + + "replacedBySeq=$sequence replacedBySrc=${presentationTimeUs / 1000}ms replacedByPts=${adjusted.renderPtsUs / 1000}ms" + ) + } } handler.removeMessages(MSG_RENDER) handler.sendEmptyMessage(MSG_RENDER) @@ -405,8 +665,10 @@ private class AtlasLibassThread( enqueue(pts, C.TIME_UNSET) } - private fun ensureEngine(slots: AtlasSlots): SpecRenderEngine { - engine?.let { return it } + private fun ensureEngine(slots: AtlasSlots, generation: Long): SpecRenderEngine { + engine?.takeIf { engineStateGeneration == generation }?.let { return it } + engineStateGeneration = generation + unchangedStreak = 0 return SpecRenderEngine( slotCount = slots.payloads.size, speculationEnabled = speculationEnabled, @@ -414,11 +676,7 @@ private class AtlasLibassThread( // Renderer identity in the high bits + its state generation in the low bits: // a recreated renderer (media item transition) can never alias a stale // speculation, even if the new generation counter happens to match. - stateGeneration = { - assHandler.render?.let { - (System.identityHashCode(it).toLong() shl 32) or (it.stateGeneration.toLong() and 0xffffffffL) - } ?: -1L - }, + stateGeneration = stateGeneration, glTakenSlot = glTakenSlot, debugLog = if (AssAtlasPipelineConfig.TIMING_LOGS) ({ msg -> Log.d(TAG, msg) }) else null ).also { engine = it } @@ -453,9 +711,10 @@ private class AtlasLibassThread( private fun drainAndRender() { val request = pending.getAndSet(null) ?: return + val sourcePts = request.sourcePtsUs val pts = request.ptsUs val releaseNs = request.releaseNs - lastRequestedPtsUs = pts + lastRequestedPtsUs = sourcePts val tDrain = System.nanoTime() // How long the request sat in the handoff (behind an in-flight on-demand or // speculative render) — the queue-wait component of any subtitle lag. @@ -464,27 +723,41 @@ private class AtlasLibassThread( // keeps the slot buffers unallocated for non-ASS playback. if (assHandler.render == null) return val slots = acquireSlots() - val engine = ensureEngine(slots) + val generation = stateGeneration() + val engine = ensureEngine(slots, generation) val pinned = releaseNs != C.TIME_UNSET // Budget left until the video frame's vsync when we START servicing. val budgetMs = if (pinned) (releaseNs - tDrain) / 1_000_000 else -1L when (val outcome = engine.service(pts, pinned)) { is SpecRenderEngine.Outcome.Post -> { + if (stateGeneration() != generation) { + staleGenerationCount++ + engineStateGeneration = Long.MIN_VALUE + if (AssAtlasPipelineConfig.TIMING_LOGS) { + Log.d(TAG, "stale-render req=${request.sequence} pts=${pts / 1000}ms") + } + return + } val payload = slots.payloads[outcome.slot] if (outcome.newContent) { payload.frame = outcome.frame payload.contentSeq = ++contentSeqCounter } + payload.sourcePresentationTimeUs = sourcePts payload.presentationTimeUs = pts payload.releaseTimeNs = releaseNs + payload.phaseLeadUs = request.phaseLeadUs + payload.requestSeq = request.sequence + payload.stateGeneration = generation onFrameReady(payload) if (AssAtlasPipelineConfig.TIMING_LOGS) { Log.d( TAG, - "render pts=${pts / 1000}ms seq=${payload.contentSeq} waitMs=$waitMs budgetMs=$budgetMs " + + "render req=${request.sequence} src=${sourcePts / 1000}ms pts=${pts / 1000}ms phaseLeadMs=${request.phaseLeadUs / 1000} " + + "releaseLeadMs=${request.releaseLeadNs / 1_000_000} seq=${payload.contentSeq} waitMs=$waitMs budgetMs=$budgetMs " + "libassMs=$lastLibassMs lockWaitMs=${assHandler.render?.lastLockWaitMs} " + - "specHit=${outcome.specHit} changed=${payload.frame.changed} quads=${payload.frame.quadCount} " + + "specHit=${outcome.specHit} changed=${payload.frame.changed} output=${payload.frame.hasOutput} quads=${payload.frame.quadCount} " + "atlas=${payload.frame.atlasWidth}x${payload.frame.atlasHeight} truncated=${payload.frame.truncated}" ) } @@ -498,6 +771,7 @@ private class AtlasLibassThread( // Pre-render the predicted next frame in the dead time between requests so the // next service is (usually) a GL-only hit. Never delays a waiting request. + if (stateGeneration() != generation) return engine.speculateAfter(pts, pinned, hasPending = pending.get() != null)?.let { write -> val payload = slots.payloads[write.slot] payload.frame = write.frame @@ -506,7 +780,7 @@ private class AtlasLibassThread( Log.d( TAG, "spec after=${pts / 1000}ms seq=${payload.contentSeq} libassMs=$lastLibassMs " + - "lockWaitMs=${assHandler.render?.lastLockWaitMs} slot=${write.slot} quads=${write.frame.quadCount}" + "lockWaitMs=${assHandler.render?.lastLockWaitMs} slot=${write.slot} output=${write.frame.hasOutput} quads=${write.frame.quadCount}" ) } } @@ -545,7 +819,7 @@ private class AtlasLibassThread( val now = System.nanoTime() if (now - lastPrefetchNs < PREFETCH_COOLDOWN_NS) return val track = assHandler.track ?: return - val nowMs = ptsUs / 1000 + val nowMs = Math.floorDiv(ptsUs, 1_000L) // Events closer than MIN_AHEAD are the regular per-frame path's business; // beyond HORIZON the warmed bitmaps may be evicted before they're needed. val targetMs = track.nextEventStartMs(nowMs + PREFETCH_MIN_AHEAD_MS) @@ -619,9 +893,8 @@ private class AtlasLibassThread( /** * Owns the EGL surface, uploads the atlas + vertex stream and issues a single - * `glDrawArrays` per frame. Swaps immediately with the swap pinned to the video's - * target release time via [EGLExt.eglPresentationTimeANDROID]; SurfaceFlinger - * holds the buffer until then, so the thread is never blocked waiting for a vsync. + * `glDrawArrays` per frame. Timed swaps are queued close to the target video + * release time and stamped via [EGLExt.eglPresentationTimeANDROID]. */ @UnstableApi private class AtlasGlThread( @@ -629,10 +902,57 @@ private class AtlasGlThread( @Volatile private var width: Int, @Volatile private var height: Int, private val assHandler: AssHandler, + private val vsyncNs: Long, private val takePending: () -> AtlasPayload?, + private val latestReadyRequestSeq: () -> Long, + private val currentStateGeneration: () -> Long, private val resolveAtlasDims: (maxTextureSize: Int) -> Pair ) : HandlerThread(TAG, Process.THREAD_PRIORITY_DISPLAY) { + // Calibration hook invoked just before each pinned eglSwapBuffers (see AssAtlasPipeline). + @Volatile var preSwapProbe: ((releaseTimeNs: Long) -> Unit)? = null + + // Swap lead = half the refresh interval, clamped. Measured live from the actual + // gap between video-frame release targets rather than display.refreshRate, which + // can still read the pre-switch rate when a pre-open mode change hasn't settled + // (the bug that pinned the lead at ~8 ms while the panel was really at 24 Hz). + private val releaseDeltasNs = LongArray(8) + private var releaseDeltaCount = 0 + private var releaseDeltaIndex = 0 + private var lastCadenceReleaseNs = C.TIME_UNSET + + // Median gap between successive video-frame release targets (≈ one refresh at + // matched cadence). Drives both the swap lead and the present offset. + @Volatile private var measuredIntervalNs: Long = vsyncNs + + @Volatile var swapLeadNs: Long = + (vsyncNs / 2).coerceIn(SCHEDULED_SWAP_LEAD_MIN_NS, SCHEDULED_SWAP_LEAD_MAX_NS) + private set + + private fun updateSwapLead(releaseNs: Long) { + val prev = lastCadenceReleaseNs + lastCadenceReleaseNs = releaseNs + if (prev == C.TIME_UNSET) return + val d = releaseNs - prev + if (d <= 0 || d > MAX_RELEASE_DELTA_NS) { + releaseDeltaCount = 0 + releaseDeltaIndex = 0 + return + } + releaseDeltasNs[releaseDeltaIndex] = d + releaseDeltaIndex = (releaseDeltaIndex + 1) % releaseDeltasNs.size + if (releaseDeltaCount < releaseDeltasNs.size) releaseDeltaCount++ + if (releaseDeltaCount >= 4) { + val copy = releaseDeltasNs.copyOf(releaseDeltaCount) + copy.sort() + measuredIntervalNs = copy[releaseDeltaCount / 2] + swapLeadNs = (measuredIntervalNs / 2).coerceIn( + SCHEDULED_SWAP_LEAD_MIN_NS, SCHEDULED_SWAP_LEAD_MAX_NS + ) + } + } + + private lateinit var handler: Handler private var eglDisplay: EGLDisplay = EGL14.EGL_NO_DISPLAY private var eglContext: EGLContext = EGL14.EGL_NO_CONTEXT @@ -658,6 +978,59 @@ private class AtlasGlThread( @Volatile var minLeadChangedMs = Long.MAX_VALUE private set + @Volatile var lastSwapLeadMs = 0L + private set + + @Volatile var lastSwapHeadroomMs = 0L + private set + + @Volatile var lastScheduledSleepMs = 0L + private set + + @Volatile var supersededBeforeSwapCount = 0L + private set + + @Volatile var staleBeforeSwapCount = 0L + private set + + // --- Present-time ground truth (EGL_ANDROID_get_frame_timestamps) --- + // The actual on-screen present time is reported a few swaps after eglSwapBuffers, + // so swapped frame ids are recorded here and resolved lazily. Confined to this + // (single) GL thread; telemetry fields are @Volatile for the stats reader. + @Volatile var presentTimingEnabled = false + private set + + /** Which timestamp source the probe settled on, or why it's off (for diagnosis). */ + @Volatile var presentSource = "off:uninit" + private set + + private val ptFrameId = LongArray(PRESENT_RING) + private val ptReleaseNs = LongArray(PRESENT_RING) + private var ptHead = 0 + private var ptCount = 0 + + @Volatile var lastPresentErrorMs = 0L + private set + + @Volatile var worstPresentErrorMs = 0L + private set + + @Volatile var presentMeasuredCount = 0L + private set + + @Volatile var presentInvalidCount = 0L + private set + + @Volatile var presentDroppedCount = 0L + private set + + private val ptBuckets = LongArray(PRESENT_BUCKETS) + val presentErrorHistogram: List get() = ptBuckets.toList() + + // Tracks renderer-state generation so transient seek swaps don't permanently + // corrupt the worst-case lead/present signals. + private var lastGenerationSeen = Long.MIN_VALUE + override fun start() { super.start() handler = Handler(looper) { msg -> @@ -717,11 +1090,34 @@ private class AtlasGlThread( val (atlasW, atlasH) = resolveAtlasDims(maxTexture[0]) renderer.allocateAtlasTexture(atlasW, atlasH) sizeChanged(width, height) + initPresentTiming() } catch (e: GlUtil.GlException) { Log.e(TAG, "Failed to initialize EGL", e) } } + /** Probes the EGL frame-timestamp extension on the freshly-current surface. + * EGL_ANDROID_get_frame_timestamps exists from Android 8.0 (entry points are + * resolved at runtime via eglGetProcAddress), so gate at O and let the native + * probe disable itself where the driver/emulator reports it unsupported — that + * "unsupported" result is itself the signal that present time can't be measured. */ + private fun initPresentTiming() { + ptHead = 0 + ptCount = 0 + val status = if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) { + AssFrameTimestamps.ERR_UNSUPPORTED + } else { + try { + AssFrameTimestamps.nativeInit() + } catch (t: Throwable) { + Log.w(TAG, "frame-timestamp init failed; present timing disabled", t) + AssFrameTimestamps.ERR_NO_PROC + } + } + presentTimingEnabled = status >= 0 + presentSource = AssFrameTimestamps.sourceLabel(status) + } + private fun sizeChanged(width: Int, height: Int) { renderer.onSurfaceChanged(width, height) if (eglDisplay != EGL14.EGL_NO_DISPLAY) { @@ -732,62 +1128,177 @@ private class AtlasGlThread( private fun drawAndSwap() { if (eglDisplay == EGL14.EGL_NO_DISPLAY) return + drainPresentTimestamps() val payload = takePending() ?: return - // Render immediately (GL commands queue on the GPU). Re-upload only when the - // slot's content actually changed — identity alone is not enough because the - // libass side rewrites slot buffers in place (contentSeq tracks the rewrites). val t0 = System.nanoTime() - val reuse = payload === lastUploadedPayload && payload.contentSeq == lastUploadedSeq - renderer.onDrawFrame(payload, reuseUploads = reuse) + val snapshot = payload.snapshot() + val reuse = payload === lastUploadedPayload && snapshot.contentSeq == lastUploadedSeq + renderer.onDrawFrame(snapshot, reuseUploads = reuse) lastUploadedPayload = payload - lastUploadedSeq = payload.contentSeq + lastUploadedSeq = snapshot.contentSeq val t1 = System.nanoTime() - // Swap immediately with the presentation time set: SurfaceFlinger holds the - // queued buffer until the video frame's target release time, so the subtitle - // can never appear early, and the GL thread is free again within a couple of - // milliseconds. Sleeping here until near the target vsync (as a removed - // TextureView path once required) made this thread blind for almost a whole - // frame interval — the single-slot latest-wins handoff would then drop an - // intermediate subtitle state (e.g. the transition to blank), showing the - // previous state one frame too long. - if (payload.releaseTimeNs != C.TIME_UNSET) { - EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, payload.releaseTimeNs) + // The overlay needs latch margin, but a drawn pinned payload still belongs to + // its video frame even if the next frame becomes ready while we wait. + val pinned = snapshot.releaseTimeNs != C.TIME_UNSET + val contentChanged = snapshot.contentSeq != lastSwappedSeq + if (pinned) updateSwapLead(snapshot.releaseTimeNs) + val presentTarget = snapshot.releaseTimeNs + val scheduledSleepMs = if (pinned) sleepUntilScheduledSwap(presentTarget) else 0L + val latestReadySeq = latestReadyRequestSeq() + val currentGeneration = currentStateGeneration() + if (currentGeneration != lastGenerationSeen) { + // Seek/track/size churn should not poison the worst-case lead signal. + lastGenerationSeen = currentGeneration + minLeadChangedMs = Long.MAX_VALUE + } + val stale = snapshot.stateGeneration != currentGeneration + // A newer ready request can be for the next video frame, while this buffer is + // still the only correct content for its own target frame. + val superseded = !pinned && snapshot.requestSeq < latestReadySeq + if (stale || superseded) { + if (stale) staleBeforeSwapCount++ + if (superseded) supersededBeforeSwapCount++ + lastScheduledSleepMs = scheduledSleepMs + if (AssAtlasPipelineConfig.TIMING_LOGS) { + Log.d( + TAG, + "skip-before-swap req=${snapshot.requestSeq} latest=$latestReadySeq " + + "stale=$stale gen=${snapshot.stateGeneration}/$currentGeneration " + + "src=${snapshot.sourcePresentationTimeUs / 1000}ms pts=${snapshot.presentationTimeUs / 1000}ms " + + "sleepMs=$scheduledSleepMs pinned=$pinned" + ) + } + return + } + if (pinned) { + // Keep the empty probe transaction out from between the presentation + // timestamp and the swap it is meant to measure. + preSwapProbe?.invoke(presentTarget) + EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentTarget) + } + // SurfaceFlinger only knows the frame id before swap and the present time later. + val frameId = if (pinned && presentTimingEnabled) { + try { + AssFrameTimestamps.nativeGetNextFrameId() + } catch (t: Throwable) { + presentTimingEnabled = false + -1L + } + } else { + -1L } EGL14.eglSwapBuffers(eglDisplay, eglSurface) val t2 = System.nanoTime() - if (payload.releaseTimeNs != C.TIME_UNSET) { + if (frameId >= 0) recordSwappedFrame(frameId, presentTarget) + var headroomMs = -1L + var leadMs = -1L + if (pinned) { swapCount++ - val lateNs = t2 - payload.releaseTimeNs + headroomMs = (presentTarget - t0) / 1_000_000 + val lateNs = t2 - presentTarget + leadMs = -lateNs / 1_000_000 + lastSwapHeadroomMs = headroomMs + lastSwapLeadMs = leadMs + lastScheduledSleepMs = scheduledSleepMs if (lateNs > LATE_THRESHOLD_NS) { lateSwapCount++ val lateMs = lateNs / 1_000_000 if (lateMs > maxLateMs) maxLateMs = lateMs } - // Lead of changed-content swaps is the frame-perfection signal: ≥ 0 means - // the new subtitle content reached the queue before the video frame's vsync. - if (payload.contentSeq != lastSwappedSeq) { - val leadMs = -lateNs / 1_000_000 + if (contentChanged) { if (leadMs < minLeadChangedMs) minLeadChangedMs = leadMs } + if (swapCount % SYNC_LOG_INTERVAL_SWAPS == 0L) { + Log.i( + TAG, + "[ASS-sync] swaps=$swapCount late=$lateSwapCount maxLateMs=$maxLateMs " + + "minLeadChangedMs=${if (minLeadChangedMs == Long.MAX_VALUE) "n/a" else minLeadChangedMs} " + + "present=$presentSource presentErrMs=$lastPresentErrorMs worstPresentMs=$worstPresentErrorMs " + + "presentHist=${ptBuckets.joinToString(",")} measured=$presentMeasuredCount " + + "presentInvalid=$presentInvalidCount presentDropped=$presentDroppedCount " + + "src=${snapshot.sourcePresentationTimeUs / 1000}ms pts=${snapshot.presentationTimeUs / 1000}ms " + + "phaseLeadMs=${snapshot.phaseLeadUs / 1000} sleepMs=$scheduledSleepMs headroomMs=$headroomMs leadMs=$leadMs " + + "req=${snapshot.requestSeq} seq=${snapshot.contentSeq} superseded=$supersededBeforeSwapCount stale=$staleBeforeSwapCount " + + "changed=$contentChanged reused=$reuse" + ) + } } - lastSwappedSeq = payload.contentSeq + lastSwappedSeq = snapshot.contentSeq if (AssAtlasPipelineConfig.TIMING_LOGS) { - val pinned = payload.releaseTimeNs != C.TIME_UNSET // headroomMs: slack before the target vsync when GL STARTED; leadMs: slack // when the buffer was actually queued (negative = queued after the vsync). - val headroomMs = if (pinned) (payload.releaseTimeNs - t0) / 1_000_000 else -1L - val leadMs = if (pinned) (payload.releaseTimeNs - t2) / 1_000_000 else -1L Log.d( TAG, - "swap pts=${payload.presentationTimeUs / 1000}ms seq=${payload.contentSeq} " + - "quads=${payload.frame.quadCount} reused=$reuse drawMs=${(t1 - t0) / 1_000_000} " + - "swapMs=${(t2 - t1) / 1_000_000} headroomMs=$headroomMs leadMs=$leadMs pinned=$pinned" + "swap src=${snapshot.sourcePresentationTimeUs / 1000}ms pts=${snapshot.presentationTimeUs / 1000}ms " + + "phaseLeadMs=${snapshot.phaseLeadUs / 1000} req=${snapshot.requestSeq} seq=${snapshot.contentSeq} " + + "quads=${snapshot.frame.quadCount} reused=$reuse drawMs=${(t1 - t0) / 1_000_000} " + + "sleepMs=$scheduledSleepMs swapMs=${(t2 - t1) / 1_000_000} headroomMs=$headroomMs leadMs=$leadMs pinned=$pinned" ) } } + private fun sleepUntilScheduledSwap(releaseTimeNs: Long): Long { + val startNs = System.nanoTime() + val wakeNs = releaseTimeNs - swapLeadNs + var remainingNs = wakeNs - startNs + while (remainingNs > SCHEDULED_SWAP_SPIN_NS && !Thread.currentThread().isInterrupted) { + LockSupport.parkNanos(remainingNs) + remainingNs = wakeNs - System.nanoTime() + } + return ((System.nanoTime() - startNs).coerceAtLeast(0L)) / 1_000_000 + } + + private fun recordSwappedFrame(frameId: Long, releaseNs: Long) { + if (ptCount == PRESENT_RING) { + ptHead = (ptHead + 1) % PRESENT_RING + ptCount-- + presentDroppedCount++ + } + val tail = (ptHead + ptCount) % PRESENT_RING + ptFrameId[tail] = frameId + ptReleaseNs[tail] = releaseNs + ptCount++ + } + + private fun drainPresentTimestamps() { + if (!presentTimingEnabled) return + while (ptCount > 0) { + val idx = ptHead + val present = try { + AssFrameTimestamps.nativeGetDisplayPresentTime(ptFrameId[idx]) + } catch (t: Throwable) { + presentTimingEnabled = false + return + } + if (present == AssFrameTimestamps.PENDING) return + ptHead = (ptHead + 1) % PRESENT_RING + ptCount-- + if (present <= 0L) { + presentInvalidCount++ + continue + } + recordPresentError(present - ptReleaseNs[idx]) + } + } + + private fun recordPresentError(errorNs: Long) { + presentMeasuredCount++ + val errorMs = errorNs / 1_000_000 + lastPresentErrorMs = errorMs + if (Math.abs(errorMs) > Math.abs(worstPresentErrorMs)) worstPresentErrorMs = errorMs + val frac = errorNs.toDouble() / vsyncNs.toDouble() + val bucket = when { + frac < -1.5 -> 0 + frac < -0.5 -> 1 + frac < 0.5 -> 2 + frac < 1.5 -> 3 + else -> 4 + } + ptBuckets[bucket]++ + } + private fun releaseEgl() { if (eglDisplay != EGL14.EGL_NO_DISPLAY) { try { @@ -810,6 +1321,15 @@ private class AtlasGlThread( private const val MSG_DRAW = 2 private const val MSG_SIZE_CHANGED = 3 private const val MSG_RELEASE = 4 + private const val SYNC_LOG_INTERVAL_SWAPS = 120L + // Missing SurfaceFlinger's latch deadline costs a full refresh, so keep the + // margin proportional to the active cadence. + private const val SCHEDULED_SWAP_LEAD_MIN_NS = 6_000_000L + private const val SCHEDULED_SWAP_LEAD_MAX_NS = 18_000_000L + private const val SCHEDULED_SWAP_SPIN_NS = 200_000L + private const val MAX_RELEASE_DELTA_NS = 250_000_000L + private const val PRESENT_RING = 16 + private const val PRESENT_BUCKETS = 5 /** * Swaps finishing this far past the target release time arrived after the @@ -850,8 +1370,9 @@ private class AtlasRenderer(private val assHandler: AssHandler) { varying vec4 v_Color; uniform sampler2D u_Texture; void main() { - float alpha = texture2D(u_Texture, v_TexCoord).a; - gl_FragColor = v_Color * alpha; + float mask = texture2D(u_Texture, v_TexCoord).a; + float alpha = v_Color.a * mask; + gl_FragColor = vec4(v_Color.rgb * alpha, alpha); } """.trimIndent() @@ -915,17 +1436,30 @@ private class AtlasRenderer(private val assHandler: AssHandler) { GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1) GLES20.glEnable(GLES20.GL_BLEND) - GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA) + // Store the translucent SurfaceView buffer premultiplied, matching Android + // layer composition and avoiding a second alpha multiply on libass masks. + GLES20.glBlendFuncSeparate( + GLES20.GL_ONE, + GLES20.GL_ONE_MINUS_SRC_ALPHA, + GLES20.GL_ONE, + GLES20.GL_ONE_MINUS_SRC_ALPHA + ) } fun onSurfaceChanged(width: Int, height: Int) { surfaceSize = Size(width, height) - assHandler.render?.setFrameSize(width, height) + // Render libass at RENDER_SCALE of the physical surface; the viewport stays + // full-size so the GL upscales the lower-res atlas to fill the surface. The + // u_SurfaceSize denominator must match the (scaled) libass frame so vertices, + // baked in frame-space, still map across the whole surface. + val frameW = AssAtlasPipelineConfig.scaledForRender(width) + val frameH = AssAtlasPipelineConfig.scaledForRender(height) + assHandler.render?.setFrameSize(frameW, frameH) GLES20.glViewport(0, 0, width, height) - GLES20.glUniform2f(uSurfaceSize, width.toFloat(), height.toFloat()) + GLES20.glUniform2f(uSurfaceSize, frameW.toFloat(), frameH.toFloat()) } - fun onDrawFrame(payload: AtlasPayload, reuseUploads: Boolean) { + fun onDrawFrame(payload: AtlasDrawSnapshot, reuseUploads: Boolean) { GlUtil.clearFocusedBuffers() val frame = payload.frame diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/AssSubtitleSurfaceView.kt b/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/AssSubtitleSurfaceView.kt index b1ed14f9..4e6420e6 100644 --- a/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/AssSubtitleSurfaceView.kt +++ b/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/AssSubtitleSurfaceView.kt @@ -23,6 +23,7 @@ class AssSubtitleSurfaceView( SurfaceHolder.Callback { private var pipeline: AssAtlasPipeline? = null + private var preSwapProbe: ((Long) -> Unit)? = null init { setZOrderMediaOverlay(true) @@ -30,6 +31,16 @@ class AssSubtitleSurfaceView( holder.addCallback(this) } + /** + * Hook invoked on the GL thread just before each pinned overlay swap, with that swap's target + * releaseTimeNs. Set by the app's latency calibrator; cleared (null) once it converges. + * Survives pipeline recreation (re-applied in [surfaceCreated]). + */ + fun setPreSwapProbe(hook: ((Long) -> Unit)?) { + preSwapProbe = hook + pipeline?.preSwapProbe = hook + } + fun requestRender(presentationTimeUs: Long, releaseTimeNs: Long) { pipeline?.requestRender(presentationTimeUs, releaseTimeNs) } @@ -75,20 +86,83 @@ class AssSubtitleSurfaceView( /** Speculation rounds skipped (paused, pending request, no confident cadence). */ val specSkips: Long get() = pipeline?.specSkips ?: 0L + /** changed==0/no-output renders forced into explicit transparent swaps. */ + val blankClearCount: Long get() = pipeline?.blankClearCount ?: 0L + /** Cache-warming prefetch renders of upcoming events. */ val prefetchCount: Long get() = pipeline?.prefetchCount ?: 0L + /** Frame requests replaced before the libass worker serviced them. */ + val coalescedRequestCount: Long get() = pipeline?.coalescedRequestCount ?: 0L + + /** Completed libass results discarded because renderer state changed before handoff. */ + val staleGenerationCount: Long get() = pipeline?.staleGenerationCount ?: 0L + + /** Completed overlay snapshots skipped because newer completed content superseded them. */ + val supersededBeforeSwapCount: Long get() = pipeline?.supersededBeforeSwapCount ?: 0L + + /** Completed overlay snapshots skipped because renderer state changed before swap. */ + val staleBeforeSwapCount: Long get() = pipeline?.staleBeforeSwapCount ?: 0L + /** Minimum lead of changed-content pinned swaps vs the video frame's release * time, in ms (negative = late); null until one happened. */ val minLeadChangedMs: Long? get() = pipeline?.minLeadChangedMs?.takeIf { it != Long.MAX_VALUE } + /** Current compositor phase lead applied by the atlas pipeline, in ms. */ + val phaseLeadMs: Long get() = pipeline?.phaseLeadMs ?: 0L + + /** Most recent pinned swap lead vs its target release time, in ms. */ + val lastSwapLeadMs: Long get() = pipeline?.lastSwapLeadMs ?: 0L + + /** Most recent pinned swap headroom when GL work started, in ms. */ + val lastSwapHeadroomMs: Long get() = pipeline?.lastSwapHeadroomMs ?: 0L + + /** Most recent phase-led wait before swap, in ms. */ + val lastScheduledSleepMs: Long get() = pipeline?.lastScheduledSleepMs ?: 0L + + /** Adaptive swap lead actually in effect (half the measured refresh interval), in ms. */ + val swapLeadMs: Long get() = pipeline?.swapLeadMs ?: 0L + + /** True once the EGL frame-timestamp extension is probed and capturing present + * times (API 26+, real device). False on the emulator / pre-26 / no driver support. */ + val presentTimingEnabled: Boolean get() = pipeline?.presentTimingEnabled ?: false + + /** Active present-time source, or why it's off (present/comp-start/comp-latch/off:…). */ + val presentSource: String get() = pipeline?.presentSource ?: "off:no-pipeline" + + /** Actual on-screen present time of the most recent measured swap minus its + * target release time, in ms (negative = before the video frame's vsync). The + * frame-perfection ground truth; null until a swap has been measured. */ + val lastPresentErrorMs: Long? get() = pipeline?.lastPresentErrorMs.takeIf { presentMeasuredCount > 0 } + + /** Largest-magnitude present error observed, in ms; null until measured. */ + val worstPresentErrorMs: Long? get() = pipeline?.worstPresentErrorMs.takeIf { presentMeasuredCount > 0 } + + /** Pinned swaps whose actual present time was read back from SurfaceFlinger. */ + val presentMeasuredCount: Long get() = pipeline?.presentMeasuredCount ?: 0L + + /** Swaps SurfaceFlinger reported as dropped/never-presented. */ + val presentInvalidCount: Long get() = pipeline?.presentInvalidCount ?: 0L + + /** Pending present-time reads evicted unread because the ring filled. */ + val presentDroppedCount: Long get() = pipeline?.presentDroppedCount ?: 0L + + /** Present-error distribution in vsync-interval units: + * [≤−1.5, (−1.5,−0.5), (−0.5,+0.5), [+0.5,+1.5), ≥+1.5]. Middle = frame-perfect. */ + val presentErrorHistogram: List get() = pipeline?.presentErrorHistogram ?: emptyList() + override fun surfaceCreated(holder: SurfaceHolder) { val rect = holder.surfaceFrame assHandler.setOverlaySurfaceSize(rect.width(), rect.height()) val lowRam = (context.getSystemService(Context.ACTIVITY_SERVICE) as? ActivityManager) ?.isLowRamDevice ?: false - pipeline = AssAtlasPipeline(holder.surface, rect.width(), rect.height(), assHandler, lowRam) - .also { it.start() } + // Display refresh drives the present-error histogram's vsync-relative buckets. + val refreshRate = display?.refreshRate?.takeIf { it >= 1f } ?: 60f + pipeline = AssAtlasPipeline(holder.surface, rect.width(), rect.height(), assHandler, lowRam, refreshRate) + .also { + it.preSwapProbe = preSwapProbe + it.start() + } } override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) { diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/SpecRenderEngine.kt b/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/SpecRenderEngine.kt index abaf5cef..f9a84ab5 100644 --- a/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/SpecRenderEngine.kt +++ b/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/SpecRenderEngine.kt @@ -1,7 +1,6 @@ package com.edde746.plezy.libass.media.widget import com.edde746.plezy.libass.AssAtlasFrame -import kotlin.math.abs /** * Decision core of the speculative render-ahead pipeline. @@ -99,6 +98,10 @@ internal class SpecRenderEngine( var specSkips = 0L private set + @Volatile + var blankClearCount = 0L + private set + /** * Services a render request for [ptsUs]. [pinned] is false for invalidate * repaints (paused margin changes etc.), which never feed the cadence @@ -107,10 +110,11 @@ internal class SpecRenderEngine( fun service(ptsUs: Long, pinned: Boolean): Outcome { if (pinned) updateDeltaEstimator(ptsUs) + val ptsMs = toLibassMs(ptsUs) if (specPtsUs != UNSET) { - val eps = epsilonUs() + val specPtsMs = toLibassMs(specPtsUs) val genNow = stateGeneration() - val hit = genNow == specGen && eps > 0 && abs(ptsUs - specPtsUs) <= eps + val hit = genNow == specGen && ptsMs == specPtsMs val slot = if (specIsLibassLast) libassLastSlot else specSlot val frame = libassLastFrame val specPts = specPtsUs @@ -118,23 +122,32 @@ internal class SpecRenderEngine( if (hit && slot >= 0 && frame != null) { specHits++ lastPostedSlot = slot - debugLog?.invoke("hit pts=${ptsUs / 1000}ms spec=${specPts / 1000}ms d=${(ptsUs - specPts) / 1000}ms slot=$slot") + debugLog?.invoke( + "hit pts=${ptsMs}ms spec=${specPtsMs}ms dUs=${ptsUs - specPts} slot=$slot" + ) return Outcome.Post(slot, frame, newContent = false, specHit = true) } specMisses++ debugLog?.invoke( - "miss pts=${ptsUs / 1000}ms spec=${specPts / 1000}ms d=${(ptsUs - specPts) / 1000}ms eps=${eps / 1000}ms " + + "miss pts=${ptsMs}ms spec=${specPtsMs}ms dUs=${ptsUs - specPts} " + "gen=${if (genNow == specGen) "ok" else "CHANGED"} slot=$slot frame=${frame != null}" ) } else { - debugLog?.invoke("no-spec pts=${ptsUs / 1000}ms") + debugLog?.invoke("no-spec pts=${ptsMs}ms") } - // On-demand render. Preferring libassLastSlot as the target makes changed == 0 - // unambiguous: the buffers were untouched and already hold the right content. + // On-demand render. For changed == 0 with visible output, the buffers were + // untouched and libassLastSlot already holds the right content. val target = renderTargetSlot() ?: return Outcome.Skip - val frame = renderAt(ptsUs / 1000, target) ?: return Outcome.Skip + val frame = renderAt(ptsMs, target) ?: return Outcome.Skip if (frame.changed == 0) { + if (isImplicitBlank(frame)) { + blankClearCount++ + libassLastSlot = target + libassLastFrame = frame + lastPostedSlot = target + return Outcome.Post(target, frame, newContent = true, specHit = false) + } val lastSlot = libassLastSlot val lastFrame = libassLastFrame ?: return Outcome.Skip if (lastSlot < 0) return Outcome.Skip @@ -169,13 +182,21 @@ internal class SpecRenderEngine( } val gen = stateGeneration() val specPts = servicedPtsUs + medianDeltaUs() - val frame = renderAt(specPts / 1000, target) ?: run { + val frame = renderAt(toLibassMs(specPts), target) ?: run { specSkips++ return null } specGen = gen specPtsUs = specPts if (frame.changed == 0) { + if (isImplicitBlank(frame)) { + blankClearCount++ + libassLastSlot = target + libassLastFrame = frame + specIsLibassLast = false + specSlot = target + return SpecWrite(target, frame) + } // Content at specPts is identical to libass's last render — nothing was // written; a hit will repost libassLastSlot (and GL will skip the upload). specIsLibassLast = true @@ -189,6 +210,9 @@ internal class SpecRenderEngine( return SpecWrite(target, frame) } + private fun isImplicitBlank(frame: AssAtlasFrame): Boolean = + !frame.hasOutput && libassLastFrame?.hasOutput == true + /** * Pre-renders [ptsUs] (an upcoming event's start) purely to warm the * renderer's glyph/bitmap caches before that content is actually needed — @@ -204,7 +228,7 @@ internal class SpecRenderEngine( fun prefetch(ptsUs: Long): SpecWrite? { specPtsUs = UNSET val target = renderTargetSlot() ?: return null - val frame = renderAt(ptsUs / 1000, target) ?: return null + val frame = renderAt(toLibassMs(ptsUs), target) ?: return null prefetchCount++ if (frame.changed == 0) return null libassLastSlot = target @@ -253,13 +277,12 @@ internal class SpecRenderEngine( return copy[deltaCount / 2] } - private fun epsilonUs(): Long = if (deltaValid()) minOf(medianDeltaUs() / 2, EPSILON_CAP_US) else 0L + private fun toLibassMs(ptsUs: Long): Long = Math.floorDiv(ptsUs, 1_000L) private companion object { const val UNSET = Long.MIN_VALUE const val DELTA_SAMPLES = 8 const val MIN_DELTA_SAMPLES = 4 const val MAX_DELTA_US = 250_000L - const val EPSILON_CAP_US = 8_000L } } diff --git a/android/libass/src/test/java/com/edde746/plezy/libass/media/widget/SpecRenderEngineTest.kt b/android/libass/src/test/java/com/edde746/plezy/libass/media/widget/SpecRenderEngineTest.kt index 4c8956c9..283cf902 100644 --- a/android/libass/src/test/java/com/edde746/plezy/libass/media/widget/SpecRenderEngineTest.kt +++ b/android/libass/src/test/java/com/edde746/plezy/libass/media/widget/SpecRenderEngineTest.kt @@ -53,7 +53,9 @@ class SpecRenderEngineTest { fun changed(quads: Int = 5) = AssAtlasFrame(2048, 100, quads, 2, 0) - fun unchanged() = AssAtlasFrame(0, 0, 0, 0, 0) + fun unchanged() = AssAtlasFrame(0, 0, 0, 0, 0, hasOutput = true) + + fun blankUnchanged() = AssAtlasFrame(0, 0, 0, 0, 0, hasOutput = false) } @Test @@ -74,15 +76,40 @@ class SpecRenderEngineTest { } @Test - fun `hit tolerates jitter within epsilon`() { + fun `hit tolerates jitter only within the same libass millisecond`() { val h = Harness() val last = h.prime() - val outcome = h.engine.service(last + DELTA + 3_000, pinned = true) + val outcome = h.engine.service(last + DELTA + 999, pinned = true) assertTrue((outcome as SpecRenderEngine.Outcome.Post).specHit) } + @Test + fun `jitter crossing a libass millisecond boundary misses`() { + val h = Harness() + val last = h.prime() + val before = h.calls.size + + h.script.add(changed()) + val outcome = h.engine.service(last + DELTA + 1_000, pinned = true) + + assertTrue(outcome is SpecRenderEngine.Outcome.Post) + assertFalse((outcome as SpecRenderEngine.Outcome.Post).specHit) + assertEquals(before + 1, h.calls.size) + assertEquals(1L, h.engine.specMisses) + } + + @Test + fun `negative pts floors to the preceding libass millisecond`() { + val h = Harness() + + h.script.add(changed()) + h.engine.service(-1, pinned = true) + + assertEquals(-1L, h.calls.last().timeMs) + } + @Test fun `seek misses and renders on demand into the spec slot`() { val h = Harness() @@ -201,6 +228,23 @@ class SpecRenderEngineTest { assertEquals(SpecRenderEngine.Outcome.Skip, h.engine.service(0, pinned = true)) } + @Test + fun `changed 0 without output clears previous content`() { + val h = Harness() + h.script.add(changed()) + val first = h.engine.service(0, pinned = true) as SpecRenderEngine.Outcome.Post + + h.script.add(blankUnchanged()) + val outcome = h.engine.service(DELTA, pinned = true) as SpecRenderEngine.Outcome.Post + + assertTrue(outcome.newContent) + assertFalse(outcome.specHit) + assertFalse(outcome.frame.hasOutput) + assertEquals(0, outcome.frame.quadCount) + assertTrue("blank frame should use a fresh metadata slot", outcome.slot != first.slot) + assertEquals(1L, h.engine.blankClearCount) + } + @Test fun `renderer gone skips`() { val h = Harness() @@ -246,6 +290,34 @@ class SpecRenderEngineTest { assertNotNull(first) // first slot existed; hit reposts whichever slot was last rendered } + @Test + fun `speculative changed 0 without output clears on hit`() { + val h = Harness() + h.script.add(changed()) + h.engine.service(0, pinned = true) + var pts = 0L + repeat(4) { + pts += DELTA + h.script.add(changed()) + h.engine.service(pts, pinned = true) + } + + h.script.add(blankUnchanged()) + val write = h.engine.speculateAfter(pts, pinned = true, hasPending = false) + assertNotNull(write) + assertFalse(write!!.frame.hasOutput) + + val before = h.calls.size + val outcome = h.engine.service(pts + DELTA, pinned = true) as SpecRenderEngine.Outcome.Post + + assertTrue(outcome.specHit) + assertFalse(outcome.newContent) + assertFalse(outcome.frame.hasOutput) + assertEquals(0, outcome.frame.quadCount) + assertEquals(before, h.calls.size) + assertEquals(1L, h.engine.blankClearCount) + } + @Test fun `speculative write reports slot for seq bump`() { val h = Harness() diff --git a/android/settings.gradle.kts b/android/settings.gradle.kts index 48747da0..ea7051f3 100644 --- a/android/settings.gradle.kts +++ b/android/settings.gradle.kts @@ -24,6 +24,4 @@ plugins { } include(":app") -// libass subtitle module (Kotlin/JNI bindings + Media3 glue); the native libass -// core stays the upstream Maven artifact io.github.peerless2012:ass. include(":libass")