diff --git a/.gitignore b/.gitignore index 8fc4708e..931d27de 100644 --- a/.gitignore +++ b/.gitignore @@ -70,4 +70,4 @@ duplication-report/ **/fastlane/report.xml maestro/**/*.png -server/deploy.sh \ No newline at end of file +server/deploy.shscripts/framesync/ diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index 787a5f85..73df8fa2 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -23,24 +23,6 @@ val downloadLibmpv by tasks.registering { } } -val assVersion = "fp-3" -val assDir = layout.buildDirectory.dir("libass").get().asFile -val assAars = listOf("lib_ass-release.aar", "lib_ass_kt-release.aar", "lib_ass_media-release.aar") - -val downloadLibass by tasks.registering { - val stamp = File(assDir, ".version") - outputs.upToDateWhen { stamp.exists() && stamp.readText().trim() == assVersion } - doLast { - assDir.mkdirs() - val baseUrl = "https://github.com/edde746/libass-android/releases/download/$assVersion" - assAars.forEach { name -> - val dest = File(assDir, name) - exec { commandLine("curl", "-sfL", "$baseUrl/$name", "-o", dest.absolutePath) } - } - stamp.writeText(assVersion) - } -} - val doviVersion = "2.3.1" val doviDir = layout.buildDirectory.dir("libdovi").get().asFile val doviAbis = mapOf( @@ -160,10 +142,9 @@ tasks.matching { it.name.contains("CMake") || it.name.contains("externalNative") dependsOn(downloadLibdovi) } -// Download libmpv and libass AARs before compilation +// Download the libmpv AAR before compilation tasks.matching { it.name.startsWith("pre") && it.name.endsWith("Build") }.configureEach { dependsOn(downloadLibmpv) - dependsOn(downloadLibass) } dependencies { @@ -186,8 +167,13 @@ dependencies { // FFmpeg audio decoder for unsupported codecs (ALAC, DTS, TrueHD, etc.) implementation("org.jellyfin.media3:media3-ffmpeg-decoder:1.9.0+1") - // libass-android for ASS/SSA subtitle rendering - assAars.forEach { implementation(files(File(assDir, it))) } + // libass ASS/SSA subtitle rendering: optimized native core (libass.so + + // prefab headers) from the edde746/libass-android fork's releases; Kotlin/JNI + // bindings + Media3 glue live in the android/libass module. -PlocalAssCore + // swaps in a mavenLocal()-published core (0.4.0-local) for native A/B tests. + val assCoreVersion = if (project.hasProperty("localAssCore")) "0.4.0-local" else "0.4.1-plezy.1" + implementation("io.github.peerless2012:ass:$assCoreVersion@aar") + implementation(project(":libass")) testImplementation("junit:junit:4.13.2") } diff --git a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerCore.kt b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerCore.kt index 56b6e849..194c0e5a 100644 --- a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerCore.kt +++ b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerCore.kt @@ -59,14 +59,13 @@ import androidx.media3.extractor.ts.TsExtractor import androidx.media3.ui.AspectRatioFrameLayout import androidx.media3.ui.CaptionStyleCompat import androidx.media3.ui.SubtitleView +import com.edde746.plezy.libass.media.AssHandler +import com.edde746.plezy.libass.media.parser.AssSubtitleParserFactory +import com.edde746.plezy.libass.media.widget.AssSubtitleSurfaceView import com.edde746.plezy.shared.AudioFocusManager import com.edde746.plezy.shared.DeviceQuirks import com.edde746.plezy.shared.FlutterOverlayHelper import com.edde746.plezy.shared.FrameRateManager -import io.github.peerless2012.ass.media.AssHandler -import io.github.peerless2012.ass.media.parser.AssSubtitleParserFactory -import io.github.peerless2012.ass.media.type.AssRenderType -import io.github.peerless2012.ass.media.widget.AssSubtitleSurfaceView import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong import org.chromium.net.CronetEngine @@ -97,6 +96,10 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { private const val DECODER_HANG_TIMEOUT_MS = 5000L private const val MAX_AUDIO_RECOVERY_ATTEMPTS = 2 private const val FPS_SAMPLE_COUNT = 8 + + /** Per-frame "video is at X" logcat stream (tag AssFrameCb) for diagnosing + * ASS subtitle lag against the libass pipeline's render/swap lines. */ + private const val ASS_FRAME_LOGS = false private const val TS_TIMESTAMP_SEARCH_PACKETS = 1800 private val DV_CODEC_PROFILE_REGEX = Regex("""(?:^|,)\s*dvh[1e]\.(\d{2})""") @@ -123,6 +126,9 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { private var subtitleView: SubtitleView? = null private var videoZoomScale: Float = 1.0f private var assHandler: AssHandler? = null + private var assSubtitleView: AssSubtitleSurfaceView? = null + private var assForceMargins = false + private var lastAssMargins: IntArray? = null private var overlayLayoutListener: ViewTreeObserver.OnGlobalLayoutListener? = null private var lastVideoSize: VideoSize? = null private var exoPlayer: ExoPlayer? = null @@ -139,6 +145,11 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { private var tunnelingDisabledForVideoCodec: Boolean = false private var tunnelingDisabledForDecodedTrueHdPcm: Boolean = false private var tunnelingDisabledForAudioRecovery: Boolean = false + + // Tunneled playback never fires the VideoFrameMetadataListener (media3 releases + // frames inside the codec), which is the libass pipeline's only render trigger — + // ASS subs would freeze. Correctness over tunneling while an ASS track is active. + private var tunnelingDisabledForAssSubtitles: Boolean = false private val tunnelingDisabledForCodec: Boolean get() = tunnelingDisabledForAudioCodec || tunnelingDisabledForVideoCodec || tunnelingDisabledForDecodedTrueHdPcm || tunnelingDisabledForAudioRecovery private var currentTunneledPlayback: Boolean = false @@ -478,8 +489,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { .setTsExtractorTimestampSearchBytes(TS_TIMESTAMP_SEARCH_PACKETS * TsExtractor.TS_PACKET_SIZE) // Inline buildWithAssSupport to retain AssHandler reference for font scale control. - val renderType = AssRenderType.OVERLAY_OPEN_GL - val handler = AssHandler(renderType) + val handler = AssHandler() assHandler = handler val assParserFactory = AssSubtitleParserFactory(handler) @@ -567,26 +577,28 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { .setRenderersFactory(wrappedRenderersFactory) .build() - // Add ASS overlay view inside videoAspectContainer, sibling of the video - // SurfaceView, so it inherits the video's resize-mode measurement, zoom - // transform, and crop: the libass frame (= surface size) maps 1:1 onto the - // displayed video rect at any fit mode or zoom level. Parenting it outside - // the container desyncs ASS positioning whenever the video rect exceeds the - // screen (cover mode, zoom > 100%). - // We use AssSubtitleSurfaceView directly (not AssSubtitleView) so we get a - // SurfaceFlinger-layer-backed overlay that eglPresentationTimeANDROID can - // vsync-pin. Z-order: video SurfaceView (-2) < this MediaOverlay-flagged + // Add ASS overlay view to the full-screen surfaceContainer (NOT the zoom-scaled + // videoAspectContainer): the libass frame = screen, and mpv-style ass_set_margins + // describe where the video dst rect sits inside it (negative when zoomed past the + // edges) — see updateAssMargins(). Non-positioned dialogue can then be forced + // on-screen (sub-ass-force-margins) while positioned/typeset events stay glued to + // the video rect, matching mpv. Also keeps the subtitle surface unscaled (crisp + // text, no per-gesture geometry churn). + // AssSubtitleSurfaceView gives us a SurfaceFlinger-layer-backed overlay that + // eglPresentationTimeANDROID can vsync-pin to the video frame. + // Z-order: video SurfaceView (-2) < this MediaOverlay-flagged // SurfaceView (-1) < parent canvas < Flutter SurfaceView (+1) in the window. - // Both punches run while this subtree draws, BEFORE the later subtitleView - // sibling renders non-ASS cues on the parent canvas. - var assSubtitleSurfaceView: AssSubtitleSurfaceView? = null - videoAspectContainer?.let { container -> + // Inserted before subtitleView so both punches run before SRT/VTT cues draw + // on the parent canvas. + surfaceContainer?.let { container -> val assView = AssSubtitleSurfaceView(container.context, handler) - assSubtitleSurfaceView = assView + assSubtitleView = assView // Pre-36 sublayer is already set by the view's own setZOrderMediaOverlay(true). FlutterOverlayHelper.applyCompositionOrder(assView, -1) + val subtitleIndex = container.indexOfChild(subtitleView) container.addView( assView, + if (subtitleIndex >= 0) subtitleIndex else container.childCount, FrameLayout.LayoutParams( FrameLayout.LayoutParams.MATCH_PARENT, FrameLayout.LayoutParams.MATCH_PARENT @@ -623,7 +635,19 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { exoPlayer!!.addAnalyticsListener(decoderHangListener) exoPlayer!!.setVideoFrameMetadataListener { presentationTimeUs, releaseTimeNs, _, _ -> // ASS bypasses Media3's text renderer, so apply sub-delay before libass renders. - assSubtitleSurfaceView?.requestRender(presentationTimeUs - subtitleDelayUs.get(), releaseTimeNs) + assSubtitleView?.requestRender(presentationTimeUs - subtitleDelayUs.get(), releaseTimeNs) + if (ASS_FRAME_LOGS) { + // Reference stream for subtitle-lag diagnosis: the video frame ExoPlayer + // is releasing right now and how far ahead of its vsync we are. Subtitle + // "render pts=" lines lagging these pts values = pipeline behind; + // budgetMs far from ~10-50 = release-time clock-domain trouble. + val budgetMs = (releaseTimeNs - System.nanoTime()) / 1_000_000 + Log.d( + "AssFrameCb", + "video pts=${presentationTimeUs / 1000}ms budgetMs=$budgetMs" + + (subtitleDelayUs.get().takeIf { it != 0L }?.let { " subDelayMs=${it / 1000}" } ?: "") + ) + } val count = fpsTimestampCount if (count < FPS_SAMPLE_COUNT) { fpsTimestamps[count] = presentationTimeUs @@ -737,7 +761,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { // Player.Listener override fun onCues(cueGroup: CueGroup) { - // With OVERLAY_CANVAS mode, ASS subtitles are rendered directly by AssSubtitleView + // ASS subtitles are rendered by the libass overlay surface. // This callback is for non-ASS subtitles (SRT, VTT, etc.) val incoming = cueGroup.cues lastSubtitleCues = incoming @@ -919,6 +943,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { evaluateAudioCodecForTunneling() evaluateVideoCodecForTunneling() + evaluateAssSubtitlesForTunneling(tracks) updateTunnelingState("tracks changed") emitTrackList() } @@ -1231,6 +1256,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { videoAspectContainer?.setAspectRatio(videoAspect) } updateSubtitleViewSize(videoWidth, videoHeight, pixelRatio) + updateAssMargins() } private fun updateSubtitleViewSize(videoWidth: Int, videoHeight: Int, pixelRatio: Float) { @@ -1243,8 +1269,8 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { val containerHeight = contentView.height if (containerWidth == 0 || containerHeight == 0) return - // Sizes the non-ASS SubtitleView only (the ASS overlay lives inside - // videoAspectContainer and tracks the video rect automatically). + // Sizes the non-ASS SubtitleView only (the ASS overlay is screen-sized and + // tracks the video rect via libass margins — see updateAssMargins()). // In cover/stretch/zoomed-in modes text cues stay at container size so they // never get cropped. In letterbox mode they follow the visible video rect. val isLetterbox = videoAspectContainer?.resizeMode == AspectRatioFrameLayout.RESIZE_MODE_FIT @@ -1287,6 +1313,83 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { } } + // Pushes mpv-style libass margins: the offsets of the video dst rect within the + // full-screen container (= the libass frame). Negative when the video extends past + // the screen (cover mode, zoom > 1) — libass supports that explicitly. Pure math + + // a native setter (no view mutation), so it is safe to run per layout pass and per + // pinch-zoom tick (#1261 no-churn invariant). + private fun updateAssMargins() { + if (disposing) return + val handler = assHandler ?: return + val vs = lastVideoSize ?: return + if (vs.width == 0 || vs.height == 0) return + + activity.runOnUiThread { + if (disposing) return@runOnUiThread + val containerWidth = surfaceContainer?.width ?: 0 + val containerHeight = surfaceContainer?.height ?: 0 + if (containerWidth == 0 || containerHeight == 0) return@runOnUiThread + + val videoAspect = (vs.width * vs.pixelWidthHeightRatio) / vs.height + val containerAspect = containerWidth.toFloat() / containerHeight + val resizeMode = videoAspectContainer?.resizeMode ?: AspectRatioFrameLayout.RESIZE_MODE_FIT + + // Mirror AspectRatioFrameLayout.onMeasure: aspect mismatches <= 1% are + // absorbed by stretching to the container instead of resizing. + val (baseWidth, baseHeight) = if (kotlin.math.abs(videoAspect / containerAspect - 1f) <= 0.01f) { + containerWidth.toFloat() to containerHeight.toFloat() + } else { + when (resizeMode) { + // Cover: scale up to fill the container, cropping the overflow + AspectRatioFrameLayout.RESIZE_MODE_ZOOM -> + if (videoAspect > containerAspect) { + containerHeight * videoAspect to containerHeight.toFloat() + } else { + containerWidth.toFloat() to containerWidth / videoAspect + } + // Stretch: video fills the container, aspect overridden + AspectRatioFrameLayout.RESIZE_MODE_FILL -> + containerWidth.toFloat() to containerHeight.toFloat() + // Fit: letterbox within the container + else -> + if (videoAspect > containerAspect) { + containerWidth.toFloat() to containerWidth / videoAspect + } else { + containerHeight * videoAspect to containerHeight.toFloat() + } + } + } + + // videoAspectContainer is centered and zoom-scaled about its center. + val videoWidth = Math.round(baseWidth * videoZoomScale) + val videoHeight = Math.round(baseHeight * videoZoomScale) + val left = (containerWidth - videoWidth) / 2 + val top = (containerHeight - videoHeight) / 2 + val right = containerWidth - videoWidth - left + val bottom = containerHeight - videoHeight - top + + val margins = intArrayOf(top, bottom, left, right) + if (lastAssMargins?.contentEquals(margins) == true) return@runOnUiThread + lastAssMargins = margins + handler.setMargins(top, bottom, left, right) + // Repaint at the current position so changes are visible while paused; during + // playback the next video frame's render supersedes it (latest-wins). + assSubtitleView?.invalidateSubtitles() + } + } + + // mpv's sub-ass-force-margins, live-applied from the Dart-managed property: lay out + // non-positioned ASS events against the visible screen instead of the video rect. + fun setAssForceMargins(force: Boolean) { + if (disposing) return + activity.runOnUiThread { + if (disposing || assForceMargins == force) return@runOnUiThread + assForceMargins = force + assHandler?.setUseMargins(force) + assSubtitleView?.invalidateSubtitles() + } + } + private fun boxFitModeToResizeMode(mode: Int): Int = when (mode) { 1 -> AspectRatioFrameLayout.RESIZE_MODE_ZOOM 2 -> AspectRatioFrameLayout.RESIZE_MODE_FILL @@ -1305,6 +1408,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { updateSubtitleViewSize(vs.width, vs.height, vs.pixelWidthHeightRatio) } } + updateAssMargins() } } @@ -1321,6 +1425,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { updateSubtitleViewSize(vs.width, vs.height, vs.pixelWidthHeightRatio) } } + updateAssMargins() } } @@ -1974,6 +2079,31 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { applyTrackSelectorPolicy(reason = reason, forceSelector = forceSelector) } + /** True when [format] is an ASS/SSA subtitle track (rendered by libass). */ + private fun isAssSubtitleFormat(format: Format): Boolean = + format.sampleMimeType == MimeTypes.TEXT_SSA || format.codecs == MimeTypes.TEXT_SSA + + /** Sets the ASS-subtitles tunneling block; returns true when the flag changed. */ + private fun updateAssSubtitlesForTunneling(assActive: Boolean): Boolean { + if (assActive == tunnelingDisabledForAssSubtitles) return false + tunnelingDisabledForAssSubtitles = assActive + emitLog( + "info", + "tunneling", + if (assActive) "ASS subtitle track selected: tunneling DISABLED (frame metadata required for libass)" + else "ASS subtitle track deselected: tunneling unblocked" + ) + return true + } + + private fun evaluateAssSubtitlesForTunneling(tracks: Tracks) { + val assSelected = tracks.groups.any { group -> + group.type == C.TRACK_TYPE_TEXT && group.isSelected && + (0 until group.length).any { isAssSubtitleFormat(group.getTrackFormat(it)) } + } + updateAssSubtitlesForTunneling(assSelected) + } + private fun applyTrackSelectorPolicy( reason: String, forceSelector: Boolean = false, @@ -2022,7 +2152,8 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { private fun calculateTunnelingEnabled(): Boolean? { val player = exoPlayer ?: return null val audioDelayActive = (renderersFactory?.audioDelayUs?.get() ?: 0L) != 0L - return tunnelingUserEnabled && (player.playbackParameters.speed == 1f) && !tunnelingDisabledForCodec && !audioDelayActive + return tunnelingUserEnabled && (player.playbackParameters.speed == 1f) && !tunnelingDisabledForCodec && + !tunnelingDisabledForAssSubtitles && !audioDelayActive } private fun updateCurrentTunnelingState(reason: String, shouldTunnel: Boolean): Boolean { @@ -2030,7 +2161,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { currentTunneledPlayback = shouldTunnel val speed = exoPlayer?.playbackParameters?.speed ?: 1f val audioDelayActive = (renderersFactory?.audioDelayUs?.get() ?: 0L) != 0L - emitLog("info", "tunneling", "Toggling tunneling=$shouldTunnel (reason=$reason, user=$tunnelingUserEnabled, speed=$speed, audioCodecDisabled=$tunnelingDisabledForAudioCodec, videoCodecDisabled=$tunnelingDisabledForVideoCodec, decodedTrueHdPcmDisabled=$tunnelingDisabledForDecodedTrueHdPcm, audioRecoveryDisabled=$tunnelingDisabledForAudioRecovery, audioDelay=$audioDelayActive)") + emitLog("info", "tunneling", "Toggling tunneling=$shouldTunnel (reason=$reason, user=$tunnelingUserEnabled, speed=$speed, audioCodecDisabled=$tunnelingDisabledForAudioCodec, videoCodecDisabled=$tunnelingDisabledForVideoCodec, decodedTrueHdPcmDisabled=$tunnelingDisabledForDecodedTrueHdPcm, audioRecoveryDisabled=$tunnelingDisabledForAudioRecovery, assSubtitlesDisabled=$tunnelingDisabledForAssSubtitles, audioDelay=$audioDelayActive)") return true } @@ -2423,6 +2554,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { tunnelingDisabledForVideoCodec = false tunnelingDisabledForDecodedTrueHdPcm = false tunnelingDisabledForAudioRecovery = false + tunnelingDisabledForAssSubtitles = false currentTunneledPlayback = false pendingStartPositionMs = startPositionMs pendingPlayWhenReady = autoPlay @@ -2605,6 +2737,9 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { fun selectSubtitleTrack(trackId: String?) { if (trackId == null || trackId == "no") { selectedSubtitleTrackId = "no" + // Flip the tunneling block in the same parameters update as the text + // disable so the renderer re-initializes once, not twice. + updateAssSubtitlesForTunneling(false) applyTrackSelectorPolicy( reason = "subtitle disabled", textDisabled = true @@ -2615,6 +2750,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { val trackGroup = subtitleTrackGroupMap[trackId] ?: return selectedSubtitleTrackId = trackId + updateAssSubtitlesForTunneling(isAssSubtitleFormat(trackGroup.getFormat(0))) applyTrackSelectorPolicy( reason = "subtitle track selected", textOverride = TrackSelectionOverride(trackGroup, 0), @@ -2898,6 +3034,26 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { "videoDecoderName" to videoDecoderName, "videoDroppedFrames" to player.videoDecoderCounters?.droppedBufferCount, "videoRenderedFrames" to player.videoDecoderCounters?.renderedOutputBufferCount, + // ASS overlay swap timing (vsync-pinned; late = past the swap-time budget) + "subSwapCount" to assSubtitleView?.swapCount, + "subLateSwaps" to assSubtitleView?.lateSwapCount, + "subMaxLateMs" to assSubtitleView?.maxLateMs, + // ASS libass render cost (changed renders rewrite the atlas; histogram + // buckets: ≤10/≤25/≤42/≤84/>84 ms) + "subRenderCount" to assSubtitleView?.renderCount, + "subChangedRenders" to assSubtitleView?.changedRenderCount, + "subOverflows" to assSubtitleView?.overflowCount, + "subLibassLastMs" to assSubtitleView?.lastLibassMs, + "subLibassMaxMs" to assSubtitleView?.maxLibassMs, + "subLibassHist" to assSubtitleView?.libassMsHistogram, + // ASS render-ahead: hits = served from a pre-rendered frame (GL-only path); + // minLead ≥ 0 means changed content reached the queue before the video + // frame's vsync — the frame-perfection signal. + "subSpecHits" to assSubtitleView?.specHits, + "subSpecMisses" to assSubtitleView?.specMisses, + "subSpecSkips" to assSubtitleView?.specSkips, + "subPrefetches" to assSubtitleView?.prefetchCount, + "subMinLeadMs" to assSubtitleView?.minLeadChangedMs, // Color info "colorSpace" to videoFormat?.colorInfo?.colorSpace, "colorRange" to videoFormat?.colorInfo?.colorRange, @@ -2992,6 +3148,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { if (tunnelingDisabledForDecodedTrueHdPcm) return "Off (decoded TrueHD PCM)" if (tunnelingDisabledForVideoCodec) return "Off (video codec unsupported)" if (tunnelingDisabledForAudioCodec) return "Off (no HW audio decoder)" + if (tunnelingDisabledForAssSubtitles) return "Off (ASS subtitles active)" return "Off" } @@ -3036,6 +3193,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { tunnelingDisabledForVideoCodec = false tunnelingDisabledForDecodedTrueHdPcm = false tunnelingDisabledForAudioRecovery = false + tunnelingDisabledForAssSubtitles = false currentTunneledPlayback = false pendingStartPositionMs = 0L pendingPlayWhenReady = null @@ -3070,6 +3228,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { videoAspectContainer = null surfaceView = null subtitleView = null + assSubtitleView = null // Remove layout listener synchronously overlayLayoutListener?.let { listener -> diff --git a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerPlugin.kt b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerPlugin.kt index fcd18bda..f7ef585e 100644 --- a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerPlugin.kt +++ b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerPlugin.kt @@ -622,6 +622,9 @@ class ExoPlayerPlugin : when (name) { "audio-delay" -> playerCore?.setAudioDelay(value.toDoubleOrNull() ?: 0.0) "sub-delay" -> playerCore?.setSubtitleDelay(value.toDoubleOrNull() ?: 0.0) + // mpv semantics mirrored on the libass overlay: anchor non-positioned ASS + // events to the visible screen (Dart sets 'yes' for cover mode / zoom > 1) + "sub-ass-force-margins" -> playerCore?.setAssForceMargins(value == "yes") } } diff --git a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ZlibMatroskaExtractor.kt b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ZlibMatroskaExtractor.kt index 5b00ed90..696640e3 100644 --- a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ZlibMatroskaExtractor.kt +++ b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ZlibMatroskaExtractor.kt @@ -7,8 +7,8 @@ import androidx.media3.extractor.SeekMap import androidx.media3.extractor.TrackOutput import androidx.media3.extractor.mkv.MatroskaExtractor import androidx.media3.extractor.text.SubtitleParser -import io.github.peerless2012.ass.media.AssHandler -import io.github.peerless2012.ass.media.extractor.AssMatroskaExtractor +import com.edde746.plezy.libass.media.AssHandler +import com.edde746.plezy.libass.media.extractor.AssMatroskaExtractor /** * Extends AssMatroskaExtractor to add support for MKV ContentCompAlgo 0 (zlib). diff --git a/android/build.gradle.kts b/android/build.gradle.kts index 89176ef4..3fa01898 100644 --- a/android/build.gradle.kts +++ b/android/build.gradle.kts @@ -2,6 +2,28 @@ allprojects { repositories { google() mavenCentral() + if (providers.gradleProperty("localAssCore").isPresent) { + // Opt-in A/B of a locally built libass native core: build it in the + // libass-android fork with `./gradlew :lib_ass:publishToMavenLocal + // -PVERSION_NAME=0.4.0-local`, then build this app with -PlocalAssCore. + mavenLocal() + } else { + // Production libass native core: the -O3/NEON/asm AAR published on + // the edde746/libass-android fork's releases (the upstream Maven + // artifact io.github.peerless2012:ass ships un-optimized natives — + // see the fork's pinned libass-cmake fix). Resolved as + // /ass-.aar with no metadata probing. + exclusiveContent { + forRepository { + ivy { + url = uri("https://github.com/edde746/libass-android/releases/download") + patternLayout { artifact("[revision]/[artifact]-[revision].[ext]") } + metadataSources { artifact() } + } + } + filter { includeModule("io.github.peerless2012", "ass") } + } + } } } diff --git a/android/libass/build.gradle.kts b/android/libass/build.gradle.kts new file mode 100644 index 00000000..05dee797 --- /dev/null +++ b/android/libass/build.gradle.kts @@ -0,0 +1,77 @@ +// libass ASS subtitle rendering: Kotlin/JNI bindings + Media3 integration +// (extractor, parsers, AssHandler, GL atlas overlay). The native libass core +// (libass.so + prefab headers) comes from the Maven artifact +// io.github.peerless2012:ass; this module compiles its JNI against it. +plugins { + id("com.android.library") + id("org.jetbrains.kotlin.android") +} + +android { + namespace = "com.edde746.plezy.libass" + compileSdk = 36 + ndkVersion = "28.2.13676358" // matches flutter.ndkVersion so only one NDK is provisioned + + defaultConfig { + minSdk = 21 + consumerProguardFiles("consumer-rules.pro") + if (project.hasProperty("localAssCore")) { + // The locally published A/B core only ships device ABIs (x86 would need + // nasm on the host); match it so prefab resolution doesn't fail. + ndk { + abiFilters += listOf("armeabi-v7a", "arm64-v8a") + } + } + externalNativeBuild { + cmake { + // libass.so in the prefab AAR is built against c++_shared (abi.json: stl=c++_shared); + // prefab validates consumer STL compatibility. + arguments += listOf("-DANDROID_STL=c++_shared") + } + } + } + + buildFeatures { + prefab = true + } + + packaging { + jniLibs { + // libass.so is a prefab IMPORTED target (linked, not owned) — the app packages + // it from the io.github.peerless2012:ass AAR; don't duplicate it here. + excludes.add("**/libass.so") + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 + } + + kotlinOptions { + jvmTarget = JavaVersion.VERSION_11.toString() + } + + externalNativeBuild { + cmake { + path = file("src/main/cpp/CMakeLists.txt") + version = "3.22.1" + } + } +} + +dependencies { + // compileOnly: prefab headers + link-time libass.so come from the AAR; runtime + // packaging of libass.so is the app's implementation dependency. + // -PlocalAssCore swaps in a mavenLocal()-published core for A/B tests (must + // match the app module's version so one libass.so is linked and packaged). + val assCoreVersion = if (project.hasProperty("localAssCore")) "0.4.0-local" else "0.4.1-plezy.1" + compileOnly("io.github.peerless2012:ass:$assCoreVersion@aar") + + implementation("androidx.annotation:annotation:1.9.1") + implementation("androidx.annotation:annotation-experimental:1.5.1") + implementation("androidx.media3:media3-exoplayer:1.9.2") + implementation("androidx.media3:media3-ui:1.9.2") + + testImplementation("junit:junit:4.13.2") +} diff --git a/android/libass/consumer-rules.pro b/android/libass/consumer-rules.pro new file mode 100644 index 00000000..9ab5d082 --- /dev/null +++ b/android/libass/consumer-rules.pro @@ -0,0 +1,6 @@ +# Constructed from JNI via FindClass/NewObject (AssKt.c). +-keep class com.edde746.plezy.libass.AssAtlasFrame { *; } +# JNI exports bind by name (Java_com_edde746_plezy_libass_*); keep the names stable. +-keepclasseswithmembernames class com.edde746.plezy.libass.* { + native ; +} diff --git a/android/libass/src/main/AndroidManifest.xml b/android/libass/src/main/AndroidManifest.xml new file mode 100644 index 00000000..a5918e68 --- /dev/null +++ b/android/libass/src/main/AndroidManifest.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/android/libass/src/main/cpp/AssKt.c b/android/libass/src/main/cpp/AssKt.c new file mode 100644 index 00000000..892a77c9 --- /dev/null +++ b/android/libass/src/main/cpp/AssKt.c @@ -0,0 +1,436 @@ +// JNI bindings for libass. Exports use standard Java___ +// naming so no RegisterNatives/JNI_OnLoad registration is needed. +#include +#include +#include +#include +#include +#include + +static inline long long nowMs(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (long long)ts.tv_sec * 1000 + ts.tv_nsec / 1000000; +} + +#include "ass/ass.h" + +#define LOG_TAG "SubtitleRenderer" + +static void assMessageCallback(int level, const char* fmt, va_list args, void* data) { + if (level > 4) return; + + if (level >= 2) { + __android_log_vprint(ANDROID_LOG_WARN, LOG_TAG, fmt, args); + } else { + __android_log_vprint(ANDROID_LOG_ERROR, LOG_TAG, fmt, args); + } +} + +// --- Ass (library) --- + +JNIEXPORT jlong JNICALL Java_com_edde746_plezy_libass_Ass_nativeAssInit(JNIEnv* env, jclass clazz) { + ASS_Library* assLibrary = ass_library_init(); + ass_set_message_cb(assLibrary, assMessageCallback, NULL); + ass_set_extract_fonts(assLibrary, 1); + return (jlong)assLibrary; +} + +JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_Ass_nativeAssAddFont( + JNIEnv* env, jclass clazz, jlong ass, jstring name, jbyteArray byteArray) { + jsize length = (*env)->GetArrayLength(env, byteArray); + jbyte* bytePtr = (*env)->GetByteArrayElements(env, byteArray, NULL); + if (bytePtr == NULL) { + return; + } + const char* cName = (*env)->GetStringUTFChars(env, name, NULL); + ass_add_font(((ASS_Library*)ass), cName, (char*)bytePtr, length); + (*env)->ReleaseByteArrayElements(env, byteArray, bytePtr, 0); + if (cName != NULL) { + (*env)->ReleaseStringUTFChars(env, name, cName); + } +} + +JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_Ass_nativeAssDeinit(JNIEnv* env, jclass clazz, jlong ass) { + if (ass) { + ass_library_done((ASS_Library*)ass); + } +} + +// --- AssTrack --- + +JNIEXPORT jlong JNICALL +Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackInit(JNIEnv* env, jclass clazz, jlong ass) { + return (jlong)ass_new_track((ASS_Library*)ass); +} + +// Shared body of readBuffer/readChunk: pins the byte array and feeds libass. +// chunked != 0 routes to ass_process_chunk (timed dialogue), else ass_process_data. +static void processTrackBytes( + JNIEnv* env, jlong track, jbyteArray buffer, jint offset, jint length, jlong start, jlong duration, int chunked) { + if (!track) return; + jbyte* elements = (*env)->GetByteArrayElements(env, buffer, NULL); + if (elements == NULL) { + return; + } + if (chunked) { + ass_process_chunk((ASS_Track*)track, (char*)(elements + offset), length, start, duration); + } else { + ass_process_data((ASS_Track*)track, (char*)(elements + offset), length); + } + (*env)->ReleaseByteArrayElements(env, buffer, elements, 0); +} + +JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackReadBuffer( + JNIEnv* env, jclass clazz, jlong track, jbyteArray buffer, jint offset, jint length) { + processTrackBytes(env, track, buffer, offset, length, 0, 0, 0); +} + +JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackReadChunk( + JNIEnv* env, jclass clazz, jlong track, jlong start, jlong duration, jbyteArray buffer, jint offset, jint length) { + processTrackBytes(env, track, buffer, offset, length, start, duration, 1); +} + +JNIEXPORT void JNICALL +Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackDeinit(JNIEnv* env, jclass clazz, jlong track) { + if (!track) return; + ass_free_track((ASS_Track*)track); +} + +// Earliest event Start strictly after afterMs, or -1. Lets the render pipeline +// pre-render (cache-warm) the next upcoming event during idle stretches so +// heavy typesetting doesn't pay its cache-cold rasterization at appearance. +JNIEXPORT jlong JNICALL Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackNextEventStart( + JNIEnv* env, jclass clazz, jlong track, jlong afterMs) { + if (!track) return -1; + ASS_Track* t = (ASS_Track*)track; + long long best = -1; + for (int i = 0; i < t->n_events; i++) { + const long long start = t->events[i].Start; + if (start > afterMs && (best < 0 || start < best)) best = start; + } + return (jlong)best; +} + +// Earliest visible-content boundary (event Start OR End) strictly after afterMs, +// or -1. A cache-warming prefetch is only invisible while no boundary passes: +// the render pipeline uses this to ensure nothing on screen is due to change +// before the event it is about to warm. +JNIEXPORT jlong JNICALL Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackNextEventChange( + JNIEnv* env, jclass clazz, jlong track, jlong afterMs) { + if (!track) return -1; + ASS_Track* t = (ASS_Track*)track; + long long best = -1; + for (int i = 0; i < t->n_events; i++) { + const long long start = t->events[i].Start; + const long long end = start + t->events[i].Duration; + if (start > afterMs && (best < 0 || start < best)) best = start; + if (end > afterMs && (best < 0 || end < best)) best = end; + } + return (jlong)best; +} + +// --- AssRender --- + +JNIEXPORT jlong JNICALL +Java_com_edde746_plezy_libass_AssRender_nativeAssRenderInit(JNIEnv* env, jclass clazz, jlong ass) { + ASS_Renderer* assRenderer = ass_renderer_init((ASS_Library*)ass); + ass_set_fonts(assRenderer, NULL, "sans-serif", ASS_FONTPROVIDER_FONTCONFIG, NULL, 1); + return (jlong)assRenderer; +} + +JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderSetFontScale( + JNIEnv* env, jclass clazz, jlong render, jfloat scale) { + if (!render) return; + ass_set_font_scale((ASS_Renderer*)render, scale); +} + +JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderSetCacheLimit( + JNIEnv* env, jclass clazz, jlong render, jint glyphMax, jint bitmapMaxSize) { + if (!render) return; + ass_set_cache_limits((ASS_Renderer*)render, glyphMax, bitmapMaxSize); +} + +JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderSetFrameSize( + JNIEnv* env, jclass clazz, jlong render, jint width, jint height) { + if (!render) return; + ass_set_frame_size((ASS_Renderer*)render, width, height); +} + +JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderSetStorageSize( + JNIEnv* env, jclass clazz, jlong render, jint width, jint height) { + if (!render) return; + ass_set_storage_size((ASS_Renderer*)render, width, height); +} + +JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderSetMargins( + JNIEnv* env, jclass clazz, jlong render, jint top, jint bottom, jint left, jint right) { + if (!render) return; + ass_set_margins((ASS_Renderer*)render, top, bottom, left, right); +} + +JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderSetUseMargins( + JNIEnv* env, jclass clazz, jlong render, jboolean use) { + if (!render) return; + ass_set_use_margins((ASS_Renderer*)render, use ? 1 : 0); +} + +JNIEXPORT void JNICALL +Java_com_edde746_plezy_libass_AssRender_nativeAssRenderDeinit(JNIEnv* env, jclass clazz, jlong render) { + if (render) { + ass_renderer_done((ASS_Renderer*)render); + } +} + +// (image, original-list index) pair so packing can run in height-sorted order +// while slots stay keyed by list position (= blend order) for pass 2. +typedef struct { + ASS_Image* img; + int idx; +} PackItem; + +static int comparePackItemsByHeightDesc(const void* a, const void* b) { + const PackItem* ia = (const PackItem*)a; + const PackItem* ib = (const PackItem*)b; + return ib->img->h - ia->img->h; +} + +// Throttle for truncation warnings (shared across renderers; logging only). +static int truncationLogCounter = 0; + +// Renders a frame into the provided atlas + vertex direct ByteBuffers. +// +// - atlasBuf holds packed ALPHA_8 pixels with row stride atlasMaxW. Only the first +// atlasHeight rows are written; the caller uploads that region to a texture +// allocated once at atlasMaxW × atlasMaxH. +// - vertexBuf holds a per-quad vertex stream (6 vertices × (2 pos + 2 uv + 4 color) +// floats = 48 floats = 192 bytes per quad). Must match BYTES_PER_QUAD/VERTEX in +// AssSubtitleAtlasPipeline.kt. Ready for a single glDrawArrays(GL_TRIANGLES, 0, N * 6). +// - UVs are normalized against atlasMaxW × atlasMaxH (the allocated texture dims), +// not the packed region, so the texture never needs reallocation. +// - Images are packed in height-sorted rows (minimizes packed height) but vertices +// are emitted in original list order — the list order is libass's painter order. +// +// Never fails on content size: images that don't fit the remaining atlas/vertex +// capacity are dropped and counted in AssAtlasFrame.truncated, so a heavy frame +// degrades instead of going stale. Returns NULL only for missing buffers/handles. +// On changed == 0, returns (0, 0, 0, changed, 0) without touching the buffers — +// caller reuses the atlas texture already on the GPU. +JNIEXPORT jobject JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderFrameAtlas( + JNIEnv* env, jclass clazz, jlong render, jlong track, jlong time, jobject atlasBuf, jint atlasMaxW, + jint atlasMaxH, jobject vertexBuf) { + if (!render || !track || !atlasBuf || !vertexBuf || atlasMaxW <= 0 || atlasMaxH <= 0) return NULL; + + jclass atlasFrameClass = (*env)->FindClass(env, "com/edde746/plezy/libass/AssAtlasFrame"); + if (!atlasFrameClass) return NULL; + jmethodID ctor = (*env)->GetMethodID(env, atlasFrameClass, "", "(IIIII)V"); + if (!ctor) return NULL; + + const long long t0 = nowMs(); + int changed; + ASS_Image* image = ass_render_frame((ASS_Renderer*)render, (ASS_Track*)track, time, &changed); + const long long tAss = nowMs(); + + if (changed == 0 || image == NULL) { + if (tAss - t0 > 40) { + __android_log_print( + ANDROID_LOG_WARN, LOG_TAG, "slow render t=%lldms: ass=%lldms (changed=%d, no output)", (long long)time, + tAss - t0, changed); + } + return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, 0); + } + + uint8_t* atlasPixels = (uint8_t*)(*env)->GetDirectBufferAddress(env, atlasBuf); + jlong atlasCap = (*env)->GetDirectBufferCapacity(env, atlasBuf); + float* vertices = (float*)(*env)->GetDirectBufferAddress(env, vertexBuf); + jlong vertexCap = (*env)->GetDirectBufferCapacity(env, vertexBuf); + if (!atlasPixels || !vertices) return NULL; + if ((jlong)atlasMaxW * atlasMaxH > atlasCap) { + __android_log_print( + ANDROID_LOG_ERROR, LOG_TAG, "atlas buffer smaller than %dx%d (capacity %lld bytes)", atlasMaxW, atlasMaxH, + (long long)atlasCap); + return NULL; + } + // 48 floats per quad × 4 bytes = 192 bytes/quad + const int maxQuads = (int)(vertexCap / 192); + + int total = 0; + for (ASS_Image* img = image; img != NULL; img = img->next) { + if (img->w > 0 && img->h > 0) total++; + } + if (total == 0) { + return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, 0); + } + + // Pass 1: assign packing slots in height-sorted order so mixed-size frames pack + // tight rows. slotX/slotY are keyed by the image's position in the original + // list; -1 marks images dropped for capacity. + PackItem* items = (PackItem*)malloc(sizeof(PackItem) * (size_t)total); + int* slotX = (int*)malloc(sizeof(int) * (size_t)total); + int* slotY = (int*)malloc(sizeof(int) * (size_t)total); + if (!items || !slotX || !slotY) { + free(items); + free(slotX); + free(slotY); + return NULL; + } + int n = 0; + for (ASS_Image* img = image; img != NULL; img = img->next) { + if (img->w > 0 && img->h > 0) { + items[n].img = img; + items[n].idx = n; + n++; + } + } + qsort(items, (size_t)n, sizeof(PackItem), comparePackItemsByHeightDesc); + + int cursorX = 0, cursorY = 0, rowH = 0; + int truncated = 0; + int packedH = 0; + int accepted = 0; + long long srcPixels = 0; + for (int i = 0; i < n; i++) { + ASS_Image* img = items[i].img; + srcPixels += (long long)img->w * img->h; + int sx = -1, sy = -1; + if (img->w <= atlasMaxW && accepted < maxQuads) { + int cx = cursorX, cy = cursorY, rh = rowH; + if (cx + img->w > atlasMaxW) { + cy += rh; + cx = 0; + rh = 0; + } + if (cy + img->h <= atlasMaxH) { + sx = cx; + sy = cy; + cursorX = cx + img->w; + cursorY = cy; + rowH = (img->h > rh) ? img->h : rh; + if (cy + img->h > packedH) packedH = cy + img->h; + accepted++; + } + } + if (sx < 0) truncated++; + slotX[items[i].idx] = sx; + slotY[items[i].idx] = sy; + } + + if (truncated > 0 && (truncationLogCounter++ & 63) == 0) { + __android_log_print( + ANDROID_LOG_WARN, LOG_TAG, "atlas truncation: %d of %d images dropped (atlas %dx%d, %d quads max)", + truncated, n, atlasMaxW, atlasMaxH, maxQuads); + } + if (accepted == 0) { + free(items); + free(slotX); + free(slotY); + return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, truncated); + } + + memset(atlasPixels, 0, (size_t)atlasMaxW * packedH); + + // Pass 2: walk the original list (= libass's painter/blend order), copying each + // accepted image into its assigned slot and emitting its quad. + int qi = 0; + int k = 0; + for (ASS_Image* img = image; img != NULL; img = img->next) { + if (img->w <= 0 || img->h <= 0) continue; + const int px = slotX[k]; + const int py = slotY[k]; + k++; + if (px < 0) continue; + + for (int y = 0; y < img->h; y++) { + uint8_t* dst = atlasPixels + (size_t)(py + y) * atlasMaxW + px; + const uint8_t* src = img->bitmap + (size_t)y * img->stride; + memcpy(dst, src, (size_t)img->w); + } + + const float x0 = (float)img->dst_x; + const float y0 = (float)img->dst_y; + const float x1 = x0 + (float)img->w; + const float y1 = y0 + (float)img->h; + const float u0 = (float)px / (float)atlasMaxW; + const float v0 = (float)py / (float)atlasMaxH; + const float u1 = (float)(px + img->w) / (float)atlasMaxW; + const float v1 = (float)(py + img->h) / (float)atlasMaxH; + const unsigned int c = img->color; + const float r = (float)((c >> 24) & 0xFFu) / 255.0f; + const float g = (float)((c >> 16) & 0xFFu) / 255.0f; + const float b = (float)((c >> 8) & 0xFFu) / 255.0f; + const float a = (float)(0xFFu - (c & 0xFFu)) / 255.0f; + + float* vx = vertices + (size_t)qi * 48; + // 8 floats per vertex: x, y, u, v, r, g, b, a. + // Triangle 1: (x0,y0) (x1,y0) (x0,y1) + vx[0] = x0; + vx[1] = y0; + vx[2] = u0; + vx[3] = v0; + vx[4] = r; + vx[5] = g; + vx[6] = b; + vx[7] = a; + vx[8] = x1; + vx[9] = y0; + vx[10] = u1; + vx[11] = v0; + vx[12] = r; + vx[13] = g; + vx[14] = b; + vx[15] = a; + vx[16] = x0; + vx[17] = y1; + vx[18] = u0; + vx[19] = v1; + vx[20] = r; + vx[21] = g; + vx[22] = b; + vx[23] = a; + // Triangle 2: (x1,y0) (x1,y1) (x0,y1) + vx[24] = x1; + vx[25] = y0; + vx[26] = u1; + vx[27] = v0; + vx[28] = r; + vx[29] = g; + vx[30] = b; + vx[31] = a; + vx[32] = x1; + vx[33] = y1; + vx[34] = u1; + vx[35] = v1; + vx[36] = r; + vx[37] = g; + vx[38] = b; + vx[39] = a; + vx[40] = x0; + vx[41] = y1; + vx[42] = u0; + vx[43] = v1; + vx[44] = r; + vx[45] = g; + vx[46] = b; + vx[47] = a; + + qi++; + } + + free(items); + free(slotX); + free(slotY); + + // Slow-render breakdown: separates libass's own cost (rasterize/blur/shape) + // from this function's packing + memcpy, so device logs attribute the time. + const long long tEnd = nowMs(); + if (tEnd - t0 > 40) { + __android_log_print( + ANDROID_LOG_WARN, LOG_TAG, + "slow render t=%lldms: total=%lldms ass=%lldms pack+copy=%lldms images=%d srcPx=%lldk atlas=%dx%d quads=%d", + (long long)time, tEnd - t0, tAss - t0, tEnd - tAss, n, srcPixels / 1000, atlasMaxW, packedH, qi); + } + + // atlasWidth is the full row stride (GLES2 can't upload with stride ≠ width); + // atlasHeight is the packed height — the only rows worth uploading. + return (*env)->NewObject(env, atlasFrameClass, ctor, atlasMaxW, packedH, qi, changed, truncated); +} diff --git a/android/libass/src/main/cpp/CMakeLists.txt b/android/libass/src/main/cpp/CMakeLists.txt new file mode 100644 index 00000000..05cbbd75 --- /dev/null +++ b/android/libass/src/main/cpp/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.22.1) + +project("asskt") + +# libass headers + libass.so come from the io.github.peerless2012:ass AAR via prefab. +add_library(${CMAKE_PROJECT_NAME} SHARED AssKt.c) +find_package(lib_ass REQUIRED CONFIG) +target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE + lib_ass::ass + log) diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/Ass.kt b/android/libass/src/main/java/com/edde746/plezy/libass/Ass.kt new file mode 100644 index 00000000..a3b1b117 --- /dev/null +++ b/android/libass/src/main/java/com/edde746/plezy/libass/Ass.kt @@ -0,0 +1,62 @@ +package com.edde746.plezy.libass + +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock + +class Ass { + + companion object { + + init { + System.loadLibrary("asskt") + } + + @JvmStatic + external fun nativeAssInit(): Long + + @JvmStatic + external fun nativeAssAddFont(ptr: Long, name: String, buffer: ByteArray) + + @JvmStatic + external fun nativeAssDeinit(ptr: Long) + } + + /** Single lock for all libass calls on this library instance. */ + val lock = ReentrantLock() + + private var nativeAss: Long = nativeAssInit() + + @Volatile + var released = false + private set + + private fun create(block: (Long) -> T): T = lock.withLock { + check(!released && nativeAss != 0L) { "Ass already released" } + block(nativeAss) + } + + fun createTrack(): AssTrack = create { AssTrack(it, lock) } + + fun createRender(): AssRender = create { AssRender(it, lock) } + + fun addFont(name: String, buffer: ByteArray) { + lock.withLock { + if (!released && nativeAss != 0L) nativeAssAddFont(nativeAss, name, buffer) + } + } + + fun release() { + lock.withLock { + if (released) return + released = true + if (nativeAss != 0L) { + nativeAssDeinit(nativeAss) + nativeAss = 0 + } + } + } + + protected fun finalize() { + release() + } +} diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/AssAtlasFrame.kt b/android/libass/src/main/java/com/edde746/plezy/libass/AssAtlasFrame.kt new file mode 100644 index 00000000..af628d94 --- /dev/null +++ b/android/libass/src/main/java/com/edde746/plezy/libass/AssAtlasFrame.kt @@ -0,0 +1,21 @@ +package com.edde746.plezy.libass + +/** + * Result of a packed-atlas render. The atlas pixel data is stored in the direct ByteBuffer + * that was passed into [AssRender.renderFrameAtlas]; the vertex stream is in the other. + * + * @param atlasWidth atlas row stride in pixels (= the allocated width; 0 when [changed] == 0) + * @param atlasHeight packed atlas height in pixels — the rows worth uploading + * @param quadCount number of quads; the vertex buffer holds [quadCount] * 6 vertices + * @param changed libass change flag (0 = no change, 1 = positions, 2 = content) + * @param truncated images dropped because they exceeded the atlas/vertex capacity; + * the frame is incomplete but never stale (> 0 should be rare — + * it means even the GL-max-sized atlas couldn't fit the frame) + */ +class AssAtlasFrame( + val atlasWidth: Int, + val atlasHeight: Int, + val quadCount: Int, + val changed: Int, + val truncated: Int +) diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/AssRender.kt b/android/libass/src/main/java/com/edde746/plezy/libass/AssRender.kt new file mode 100644 index 00000000..c5d06a2a --- /dev/null +++ b/android/libass/src/main/java/com/edde746/plezy/libass/AssRender.kt @@ -0,0 +1,164 @@ +package com.edde746.plezy.libass + +import java.nio.ByteBuffer +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock + +class AssRender(nativeAss: Long, private val lock: ReentrantLock) { + + companion object { + + @JvmStatic + external fun nativeAssRenderInit(ass: Long): Long + + @JvmStatic + external fun nativeAssRenderSetFontScale(render: Long, scale: Float) + + @JvmStatic + external fun nativeAssRenderSetCacheLimit(render: Long, glyphMax: Int, bitmapMaxSize: Int) + + @JvmStatic + external fun nativeAssRenderSetStorageSize(render: Long, width: Int, height: Int) + + @JvmStatic + external fun nativeAssRenderSetFrameSize(render: Long, width: Int, height: Int) + + @JvmStatic + external fun nativeAssRenderSetMargins(render: Long, top: Int, bottom: Int, left: Int, right: Int) + + @JvmStatic + external fun nativeAssRenderSetUseMargins(render: Long, use: Boolean) + + @JvmStatic + external fun nativeAssRenderFrameAtlas( + render: Long, + track: Long, + time: Long, + atlasBuf: ByteBuffer, + atlasMaxWidth: Int, + atlasMaxHeight: Int, + vertexBuf: ByteBuffer + ): AssAtlasFrame? + + @JvmStatic + external fun nativeAssRenderDeinit(render: Long) + } + + private var nativeRender: Long = nativeAssRenderInit(nativeAss) + + @Volatile + var released = false + private set + + private var track: AssTrack? = null + + /** + * Bumped on every renderer-state mutation (track, sizes, margins, font scale). + * Lets the render-ahead pipeline detect that a speculatively rendered frame was + * produced against stale state and must not be presented. + */ + private val generation = java.util.concurrent.atomic.AtomicInteger(0) + + /** Current renderer-state generation; see [generation]. */ + val stateGeneration: Int get() = generation.get() + + /** Runs [block] with the native handle under the shared libass lock; no-op once released. */ + private inline fun withNative(block: (Long) -> Unit) { + lock.withLock { + if (!released && nativeRender != 0L) block(nativeRender) + } + } + + fun setTrack(track: AssTrack?) { + generation.incrementAndGet() + lock.withLock { this.track = track } + } + + fun setFontScale(scale: Float) { + generation.incrementAndGet() + withNative { nativeAssRenderSetFontScale(it, scale) } + } + + fun setCacheLimit(glyphMax: Int, bitmapMaxSize: Int) = withNative { nativeAssRenderSetCacheLimit(it, glyphMax, bitmapMaxSize) } + + fun setStorageSize(width: Int, height: Int) { + generation.incrementAndGet() + withNative { nativeAssRenderSetStorageSize(it, width, height) } + } + + fun setFrameSize(width: Int, height: Int) { + generation.incrementAndGet() + withNative { nativeAssRenderSetFrameSize(it, width, height) } + } + + /** + * mpv-style frame margins: offsets of the video dst rect within the frame set by + * [setFrameSize]. Negative when the video extends beyond the frame (zoomed in / cover). + */ + fun setMargins(top: Int, bottom: Int, left: Int, right: Int) { + generation.incrementAndGet() + withNative { nativeAssRenderSetMargins(it, top, bottom, left, right) } + } + + /** + * mpv's sub-ass-force-margins: lay out non-positioned events against the full frame + * (kept on the visible screen) instead of the video rect between the margins. + */ + fun setUseMargins(use: Boolean) { + generation.incrementAndGet() + withNative { nativeAssRenderSetUseMargins(it, use) } + } + + /** + * Renders a frame into a packed ALPHA_8 texture atlas plus a single vertex stream + * ready for `glDrawArrays(GL_TRIANGLES, 0, quadCount * 6)`. + * + * UVs are normalized against ([atlasMaxW], [atlasMaxH]) — the allocated texture + * dims — so the caller can allocate the texture once and `glTexSubImage2D` only + * the packed rows. Images that exceed the capacity are dropped and counted in + * [AssAtlasFrame.truncated]; the render never fails on content size. + * + * @param atlasBuf direct ByteBuffer receiving the packed pixels (≥ atlasMaxW × atlasMaxH) + * @param atlasMaxW atlas row stride in pixels (bound by `GL_MAX_TEXTURE_SIZE`) + * @param atlasMaxH atlas height in pixels (bound by `GL_MAX_TEXTURE_SIZE`) + * @param vertexBuf direct ByteBuffer receiving the vertex stream (192 bytes per quad) + */ + /** How long the most recent [renderFrameAtlas] waited to acquire the shared + * libass lock (contended by track dialogue/font feeding), in milliseconds. */ + @Volatile + var lastLockWaitMs: Long = 0 + private set + + fun renderFrameAtlas( + time: Long, + atlasBuf: ByteBuffer, + atlasMaxW: Int, + atlasMaxH: Int, + vertexBuf: ByteBuffer + ): AssAtlasFrame? { + val tQueue = System.nanoTime() + lock.withLock { + lastLockWaitMs = (System.nanoTime() - tQueue) / 1_000_000 + if (released || nativeRender == 0L) return null + val t = track ?: return null + if (t.released || t.nativeAssTrack == 0L) return null + return nativeAssRenderFrameAtlas(nativeRender, t.nativeAssTrack, time, atlasBuf, atlasMaxW, atlasMaxH, vertexBuf) + } + } + + fun release() { + lock.withLock { + if (released) return + released = true + track = null + if (nativeRender != 0L) { + nativeAssRenderDeinit(nativeRender) + nativeRender = 0 + } + } + } + + protected fun finalize() { + release() + } +} diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/AssTrack.kt b/android/libass/src/main/java/com/edde746/plezy/libass/AssTrack.kt new file mode 100644 index 00000000..25362b42 --- /dev/null +++ b/android/libass/src/main/java/com/edde746/plezy/libass/AssTrack.kt @@ -0,0 +1,77 @@ +package com.edde746.plezy.libass + +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock + +class AssTrack(private val ass: Long, private val lock: ReentrantLock) { + + companion object { + + @JvmStatic + external fun nativeAssTrackInit(track: Long): Long + + @JvmStatic + external fun nativeAssTrackReadBuffer(track: Long, byteArray: ByteArray, offset: Int, length: Int) + + @JvmStatic + external fun nativeAssTrackReadChunk(track: Long, start: Long, duration: Long, byteArray: ByteArray, offset: Int, length: Int) + + @JvmStatic + external fun nativeAssTrackDeinit(track: Long) + + @JvmStatic + external fun nativeAssTrackNextEventStart(track: Long, afterMs: Long): Long + + @JvmStatic + external fun nativeAssTrackNextEventChange(track: Long, afterMs: Long): Long + } + + var nativeAssTrack = nativeAssTrackInit(ass) + private set + + @Volatile + var released = false + private set + + /** Runs [block] with the native handle under the shared libass lock; no-op once released. */ + private inline fun withNative(block: (Long) -> Unit) { + lock.withLock { + if (!released && nativeAssTrack != 0L) block(nativeAssTrack) + } + } + + fun readBuffer(array: ByteArray, offset: Int = 0, length: Int = array.size) = withNative { nativeAssTrackReadBuffer(it, array, offset, length) } + + fun readChunk(start: Long, duration: Long, array: ByteArray, offset: Int = 0, length: Int = array.size) = withNative { nativeAssTrackReadChunk(it, start, duration, array, offset, length) } + + /** Earliest event start strictly after [afterMs], or -1 if none (yet). */ + fun nextEventStartMs(afterMs: Long): Long { + lock.withLock { + if (released || nativeAssTrack == 0L) return -1 + return nativeAssTrackNextEventStart(nativeAssTrack, afterMs) + } + } + + /** Earliest event start OR end strictly after [afterMs], or -1 if none (yet). */ + fun nextEventChangeMs(afterMs: Long): Long { + lock.withLock { + if (released || nativeAssTrack == 0L) return -1 + return nativeAssTrackNextEventChange(nativeAssTrack, afterMs) + } + } + + fun release() { + lock.withLock { + if (released) return + released = true + if (nativeAssTrack != 0L) { + nativeAssTrackDeinit(nativeAssTrack) + nativeAssTrack = 0 + } + } + } + + protected fun finalize() { + release() + } +} diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/media/AssHandler.kt b/android/libass/src/main/java/com/edde746/plezy/libass/media/AssHandler.kt new file mode 100644 index 00000000..e271a877 --- /dev/null +++ b/android/libass/src/main/java/com/edde746/plezy/libass/media/AssHandler.kt @@ -0,0 +1,378 @@ +package com.edde746.plezy.libass.media + +import android.util.Log +import androidx.annotation.OptIn +import androidx.media3.common.Format +import androidx.media3.common.MediaItem +import androidx.media3.common.MimeTypes +import androidx.media3.common.MimeTypes.TEXT_SSA +import androidx.media3.common.Player.Listener +import androidx.media3.common.Tracks +import androidx.media3.common.VideoSize +import androidx.media3.common.util.Size +import androidx.media3.common.util.UnstableApi +import androidx.media3.exoplayer.ExoPlayer +import androidx.media3.exoplayer.video.VideoFrameMetadataListener +import com.edde746.plezy.libass.Ass +import com.edde746.plezy.libass.AssRender +import com.edde746.plezy.libass.AssTrack +import com.edde746.plezy.libass.media.parser.AssHeaderParser + +/** + * Handles ASS subtitle rendering and integration with ExoPlayer. + * + * This class listens to ExoPlayer events and manages the creation, selection, and rendering of ASS + * subtitle tracks. Rendering always happens on a GL overlay surface (atlas pipeline) — the + * libass frame is the overlay surface, not the video. + */ +@OptIn(UnstableApi::class) +class AssHandler( + val config: AssHandlerConfig = AssHandlerConfig() +) : Listener { + + /** The ASS instance used for creating tracks and renderers. This is lazy to avoid loading + * libass if the played media does not have ASS tracks. */ + private val assDelegate = lazy { Ass() } + val ass by assDelegate + + /** The current ASS renderer. It's created as soon as a ASS track is detected. */ + var render: AssRender? = null + private set + + /** + * AssRender changed callback + */ + var renderCallback: ((AssRender?) -> Unit)? = null + + /** The currently selected ASS track. */ + var track: AssTrack? = null + private set + + /** The available ASS tracks in the current media. */ + private val availableTracks = mutableMapOf() + + /** Fonts encountered before any ASS track was created. Flushed in [createTrack]. */ + private val pendingFonts = mutableListOf>() + + /** The size of the video track. */ + var videoSize = Size.ZERO + private set + + /** The size of the surface on which subtitles are rendered. */ + var surfaceSize = Size.ZERO + private set + + /** + * True once an overlay widget reported its surface size via [setOverlaySurfaceSize]. + * From then on the Player.Listener surface size (= the video output surface, which + * may differ from the subtitle overlay once the overlay is parented elsewhere) is + * ignored as a frame-size source. + */ + private var overlaySizeFromWidget = false + + /** mpv-style margins of the video rect within the frame: top, bottom, left, right. */ + private var margins: IntArray? = null + + /** mpv's sub-ass-force-margins: anchor non-positioned events to the visible frame. */ + private var useMargins = false + + /** + * Per-video-frame callback. Fired by ExoPlayer just before MediaCodec releases the frame + * to the output surface. Carries the exact PTS of the frame and the System.nanoTime() + * domain target release time, so subtitle renderers can align composition to the same + * display vsync as the video. + * + * - [presentationTimeUs] is track-relative microseconds matching the subtitle track's PTS. + * - [releaseTimeNs] may be [androidx.media3.common.C.TIME_UNSET] in rare paths. + * + * Invoked on the playback thread. + */ + var videoFrameCallback: ((presentationTimeUs: Long, releaseTimeNs: Long) -> Unit)? = null + + private val videoFrameMetadataListener = VideoFrameMetadataListener { pts, releaseNs, _, _ -> + videoFrameCallback?.invoke(pts, releaseNs) + } + + private var player: ExoPlayer? = null + + /** The current selected ass format. */ + private var format: Format? = null + + /** + * Initializes the handler with the provided ExoPlayer instance. + * @param player The ExoPlayer instance to attach to. + */ + fun init(player: ExoPlayer) { + this.player = player + player.addListener(this) + player.setVideoFrameMetadataListener(videoFrameMetadataListener) + } + + /** + * Handles transitions between media items in the player and resets everything to the initial + * state. + */ + override fun onMediaItemTransition(mediaItem: MediaItem?, reason: Int) { + super.onMediaItemTransition(mediaItem, reason) + Log.i("AssHandler", "onMediaItemTransition: item = $mediaItem, reason = $reason") + render = null + track = null + availableTracks.clear() + pendingFonts.clear() + videoSize = Size.ZERO + renderCallback?.invoke(null) + } + + /** + * Handles changes to the tracks available in the current media. + * Configures the selected ASS track if available. + * @param tracks The selected tracks. + */ + override fun onTracksChanged(tracks: Tracks) { + Log.i("AssHandler", "onTracksChanged $tracks") + + val selectedVideoTrack = getSelectedVideoTrack(tracks) + if (selectedVideoTrack != null) { + setVideoSize(selectedVideoTrack.width, selectedVideoTrack.height) + } + + format = getSelectedAssTrack(tracks) + if (format == null) { + Log.i("AssHandler", "subtitle track disabled") + track = null + render?.setTrack(null) + return + } + + updateTrack() + } + + private fun updateTrack() { + val track = availableTracks.firstNotNullOfOrNull { + // When media without external subtitles, format id will not change. + // When media with external subtitles, format will become like 1:1 . + // So to compat both situation, we extract the actual id after the colon. + if (format?.id?.substringAfter(":") == it.key) { + it.value + } else { + null + } + } + if (track == null || this.track == track) return + + Log.i("AssHandler", "subtitle track changed to $format") + this.track = track + val render = requireNotNull(render) + applyRenderState(render) + render.setTrack(track) + } + + /** + * (Re)applies sizing and margin state to [render]. Called whenever a render is + * (re)created or the selected track changes, so renderer state survives both. + */ + private fun applyRenderState(render: AssRender) { + if (videoSize.isValid) render.setStorageSize(videoSize.width, videoSize.height) + when { + surfaceSize.isValid -> render.setFrameSize(surfaceSize.width, surfaceSize.height) + // Fallback frame until the overlay surface reports its size. + videoSize.isValid -> render.setFrameSize(videoSize.width, videoSize.height) + } + margins?.let { m -> render.setMargins(m[0], m[1], m[2], m[3]) } + render.setUseMargins(useMargins) + } + + /** + * Handles changes to the surface size for video playback. + * Notifies the callback if the size has changed. + * @param width The new width of the surface. + * @param height The new height of the surface. + */ + override fun onSurfaceSizeChanged(width: Int, height: Int) { + super.onSurfaceSizeChanged(width, height) + Log.i("AssHandler", "onSurfaceSizeChanged: width = $width, height = $height") + // The video output surface is only a frame-size proxy until an overlay widget + // reports its own size — they diverge when the overlay isn't video-rect-sized. + if (overlaySizeFromWidget) return + if (surfaceSize.width == width && surfaceSize.height == height) return + surfaceSize = Size(width, height) + } + + /** + * Reports the subtitle overlay widget's actual surface size — the authoritative + * libass frame size for OVERLAY render types. + */ + fun setOverlaySurfaceSize(width: Int, height: Int) { + overlaySizeFromWidget = true + if (surfaceSize.width == width && surfaceSize.height == height) return + Log.i("AssHandler", "setOverlaySurfaceSize: width = $width, height = $height") + surfaceSize = Size(width, height) + render?.setFrameSize(width, height) + } + + /** + * Sets mpv-style frame margins: the offsets of the video dst rect within the libass + * frame ([setOverlaySurfaceSize]); negative when the video extends beyond the frame. + * Applied to the current render and re-applied whenever a render is (re)created. + */ + fun setMargins(top: Int, bottom: Int, left: Int, right: Int) { + margins = intArrayOf(top, bottom, left, right) + render?.setMargins(top, bottom, left, right) + } + + /** mpv's sub-ass-force-margins: anchor non-positioned events to the visible frame. */ + fun setUseMargins(use: Boolean) { + useMargins = use + render?.setUseMargins(use) + } + + override fun onVideoSizeChanged(videoSize: VideoSize) { + super.onVideoSizeChanged(videoSize) + this.videoSize = Size(videoSize.width, videoSize.height) + Log.i("AssHandler", "onVideoSizeChanged: width = ${videoSize.width}, height = ${videoSize.height}") + } + + /** + * Updates the video size for the ASS renderer. Called as soon as the video size is known in + * order to properly render subtitles. + * @param width The width of the video. + * @param height The height of the video. + */ + fun setVideoSize(width: Int, height: Int) { + Log.i("AssHandler", "setVideoSize: width = $width, height = $height") + videoSize = Size(width, height) + } + + /** + * Returns true if the current media has ASS tracks, false otherwise. + */ + fun hasTracks(): Boolean = availableTracks.isNotEmpty() + + /** + * Adds a font to the ASS library. If no tracks have been created yet, the font is buffered + * and will be added when the first track is created via [createTrack]. + */ + @Synchronized + fun addFont(name: String, data: ByteArray) { + if (hasTracks()) { + ass.addFont(name, data) + } else { + pendingFonts.add(name to data) + } + } + + /** + * Creates a new ASS track from the given format and saves it in the [availableTracks]. + * The renderer and libass are also created if needed. + * @param format The format of the ASS track. + * @return The created ASS track. + */ + @Synchronized + fun createTrack(format: Format): AssTrack { + Log.i("AssHandler", "createTrack: format = $format") + // Ensure the renderer is created before creating tracks. + createRenderIfNeeded() + + // Flush any fonts that were buffered before the first track was created. + if (pendingFonts.isNotEmpty()) { + for ((name, data) in pendingFonts) { + ass.addFont(name, data) + } + pendingFonts.clear() + } + + val track = ass.createTrack() + if (format.initializationData.size > 0) { + val header = AssHeaderParser.parse(format) + track.readBuffer(header) + } + availableTracks[format.id!!] = track + + updateTrack() + + return track + } + + /** + * Ensures the ASS renderer is created if it does not already exist. + */ + private fun createRenderIfNeeded() { + if (render != null) return + Log.i("AssHandler", "createRender (cacheSize: ${config.cacheSize}MB, glyphSize: ${config.glyphSize})") + render = ass.createRender().also { render -> + render.setCacheLimit(config.glyphSize, config.cacheSize) + applyRenderState(render) + } + renderCallback?.invoke(render) + } + + /** + * Reads a dialogue into the track of the given [trackId]. + * Thread-safe: AssTrack.readChunk internally acquires the shared libass lock. + */ + fun readTrackDialogue( + trackId: String?, + start: Long, + duration: Long, + data: ByteArray, + offset: Int = 0, + length: Int = data.size + ) { + val t = availableTracks[trackId] ?: return + t.readChunk(start, duration, data, offset, length) + } + + /** + * Retrieves the selected video track, if any. + */ + private fun getSelectedVideoTrack(tracks: Tracks): Format? = tracks.groups.find { group -> + if (group.isSelected) { + (0 until group.length).any { index -> + val track = group.getTrackFormat(index) + MimeTypes.isVideo(track.sampleMimeType) + } + } else { + false + } + }?.getTrackFormat(0) + + /** + * Retrieves the ID of the selected ASS track, if any. + * @param tracks The selected tracks. + * @return The ID of the selected ASS track, or null if none. + */ + private fun getSelectedAssTrack(tracks: Tracks): Format? = tracks.groups.find { group -> + if (group.isSelected) { + (0 until group.length).any { index -> + val track = group.getTrackFormat(index) + track.sampleMimeType == TEXT_SSA || track.codecs == TEXT_SSA + } + } else { + false + } + }?.getTrackFormat(0) + + /** + * Releases all native resources held by this handler. + */ + fun release() { + videoFrameCallback = null + player?.clearVideoFrameMetadataListener(videoFrameMetadataListener) + player = null + render?.release() + render = null + availableTracks.values.forEach { it.release() } + availableTracks.clear() + track = null + pendingFonts.clear() + if (assDelegate.isInitialized()) { + ass.release() + } + } + + /** + * Checks if the size is valid (both width and height are greater than 0). + */ + private val Size.isValid + get() = width > 0 && height > 0 +} diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/media/AssHandlerConfig.kt b/android/libass/src/main/java/com/edde746/plezy/libass/media/AssHandlerConfig.kt new file mode 100644 index 00000000..fb382a4b --- /dev/null +++ b/android/libass/src/main/java/com/edde746/plezy/libass/media/AssHandlerConfig.kt @@ -0,0 +1,6 @@ +package com.edde746.plezy.libass.media + +data class AssHandlerConfig( + val glyphSize: Int = 10000, + val cacheSize: Int = 128 +) diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/media/extractor/AssMatroskaExtractor.kt b/android/libass/src/main/java/com/edde746/plezy/libass/media/extractor/AssMatroskaExtractor.kt new file mode 100644 index 00000000..2eb5ea3d --- /dev/null +++ b/android/libass/src/main/java/com/edde746/plezy/libass/media/extractor/AssMatroskaExtractor.kt @@ -0,0 +1,127 @@ +package com.edde746.plezy.libass.media.extractor + +import androidx.annotation.OptIn +import androidx.media3.common.util.ParsableByteArray +import androidx.media3.common.util.UnstableApi +import androidx.media3.extractor.ExtractorInput +import androidx.media3.extractor.ExtractorOutput +import androidx.media3.extractor.mkv.EbmlProcessor +import androidx.media3.extractor.mkv.MatroskaExtractor +import androidx.media3.extractor.text.SubtitleParser +import com.edde746.plezy.libass.media.AssHandler +import com.edde746.plezy.libass.media.text.AssSubtitleExtractorOutput + +@OptIn(UnstableApi::class) +open class AssMatroskaExtractor( + subtitleParserFactory: SubtitleParser.Factory, + private val assHandler: AssHandler, + flags: Int = 0 +) : MatroskaExtractor(subtitleParserFactory, flags) { + + private var currentAttachmentName: String? = null + private var currentAttachmentMime: String? = null + + internal val subtitleSample = subtitleSampleField.get(this) as ParsableByteArray + + override fun getElementType(id: Int): Int = when (id) { + ID_ATTACHMENTS -> EbmlProcessor.ELEMENT_TYPE_MASTER + ID_ATTACHED_FILE -> EbmlProcessor.ELEMENT_TYPE_MASTER + ID_FILE_NAME -> EbmlProcessor.ELEMENT_TYPE_STRING + ID_FILE_MIME_TYPE -> EbmlProcessor.ELEMENT_TYPE_STRING + ID_FILE_DATA -> EbmlProcessor.ELEMENT_TYPE_BINARY + else -> super.getElementType(id) + } + + override fun isLevel1Element(id: Int): Boolean = super.isLevel1Element(id) || id == ID_ATTACHMENTS + + override fun startMasterElement(id: Int, contentPosition: Long, contentSize: Long) { + when (id) { + ID_EBML -> { + val currentExtractor = extractorOutput.get(this) as ExtractorOutput + if (currentExtractor !is AssSubtitleExtractorOutput) { + extractorOutput.set( + this, + AssSubtitleExtractorOutput(currentExtractor, assHandler, this) + ) + } + super.startMasterElement(id, contentPosition, contentSize) + } + ID_ATTACHED_FILE -> clearAttachment() + else -> super.startMasterElement(id, contentPosition, contentSize) + } + } + + override fun endMasterElement(id: Int) { + when (id) { + ID_VIDEO -> { + // We need to get the video dimensions very early + val track = getCurrentTrack(id) + assHandler.setVideoSize(track.width, track.height) + super.endMasterElement(id) + } + ID_ATTACHED_FILE -> clearAttachment() + else -> super.endMasterElement(id) + } + } + + override fun stringElement(id: Int, value: String) { + when (id) { + ID_FILE_NAME -> currentAttachmentName = value + ID_FILE_MIME_TYPE -> currentAttachmentMime = value + else -> super.stringElement(id, value) + } + } + + override fun binaryElement(id: Int, contentSize: Int, input: ExtractorInput) { + when (id) { + ID_FILE_DATA -> { + val attachmentName = requireNotNull(currentAttachmentName) + val attachmentMime = requireNotNull(currentAttachmentMime) + + if (attachmentMime in fontMimeTypes) { + val data = ByteArray(contentSize) + input.readFully(data, 0, contentSize) + assHandler.addFont(attachmentName, data) + } else { + input.skipFully(contentSize) + } + } + else -> super.binaryElement(id, contentSize, input) + } + } + + private fun clearAttachment() { + currentAttachmentName = null + currentAttachmentMime = null + } + + companion object { + const val ID_EBML = 0x1A45DFA3 + const val ID_VIDEO = 0xE0 + const val ID_ATTACHMENTS = 0x1941A469 + const val ID_ATTACHED_FILE = 0x61A7 + const val ID_FILE_NAME = 0x466E + const val ID_FILE_MIME_TYPE = 0x4660 + const val ID_FILE_DATA = 0x465C + + val fontMimeTypes = listOf( + "font/ttf", + "font/otf", + "font/sfnt", + "font/woff", + "font/woff2", + "application/font-sfnt", + "application/font-woff", + "application/x-truetype-font", + "application/vnd.ms-opentype", + "application/x-font-ttf" + ) + + val extractorOutput = MatroskaExtractor::class.java.getDeclaredField("extractorOutput").apply { + isAccessible = true + } + val subtitleSampleField = MatroskaExtractor::class.java.getDeclaredField("subtitleSample").apply { + isAccessible = true + } + } +} diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/media/parser/AssFullSubtitleParser.kt b/android/libass/src/main/java/com/edde746/plezy/libass/media/parser/AssFullSubtitleParser.kt new file mode 100644 index 00000000..f98ae52f --- /dev/null +++ b/android/libass/src/main/java/com/edde746/plezy/libass/media/parser/AssFullSubtitleParser.kt @@ -0,0 +1,29 @@ +package com.edde746.plezy.libass.media.parser + +import androidx.media3.common.Format +import androidx.media3.common.util.Consumer +import androidx.media3.common.util.UnstableApi +import androidx.media3.extractor.text.CuesWithTiming +import androidx.media3.extractor.text.SubtitleParser +import com.edde746.plezy.libass.AssTrack + +/** + * Parser for full (non-embedded) ASS documents, e.g. external sidecar files. + * Feeds the whole document into the libass track; rendering happens on the + * overlay surface, so no Media3 cues are emitted. + */ +@UnstableApi +class AssFullSubtitleParser(private val track: AssTrack) : SubtitleParser { + + override fun parse( + data: ByteArray, + offset: Int, + length: Int, + outputOptions: SubtitleParser.OutputOptions, + output: Consumer + ) { + track.readBuffer(data, offset, length) + } + + override fun getCueReplacementBehavior(): Int = Format.CUE_REPLACEMENT_BEHAVIOR_REPLACE +} diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/media/parser/AssHeaderParser.kt b/android/libass/src/main/java/com/edde746/plezy/libass/media/parser/AssHeaderParser.kt new file mode 100644 index 00000000..c60fb494 --- /dev/null +++ b/android/libass/src/main/java/com/edde746/plezy/libass/media/parser/AssHeaderParser.kt @@ -0,0 +1,30 @@ +package com.edde746.plezy.libass.media.parser + +import androidx.annotation.OptIn +import androidx.media3.common.Format +import androidx.media3.common.util.UnstableApi + +@OptIn(UnstableApi::class) +object AssHeaderParser { + + private const val ASS_EVENTS = "[Events]\n" + + "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text" + + /** + * Fix some ass header with error end. + * https://github.com/jellyfin/jellyfin-ffmpeg/issues/506 + */ + private fun fixAssHeaderIfNeed(buffer: ByteArray): ByteArray = if (buffer[buffer.size - 1] != 0.toByte()) { + // validate ass header + buffer + } else { + // remote the last null character and append the events tag + (String(buffer, 0, buffer.size - 1) + "\n" + ASS_EVENTS).toByteArray() + } + + /** + * Parses the headers from the initialization data of the given [format]. The original + * headers are preserved (duplication checks are handled by libass). + */ + fun parse(format: Format): ByteArray = fixAssHeaderIfNeed(format.initializationData[1]) +} diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/media/parser/AssNoOpSubtitleParser.kt b/android/libass/src/main/java/com/edde746/plezy/libass/media/parser/AssNoOpSubtitleParser.kt new file mode 100644 index 00000000..6fbb4817 --- /dev/null +++ b/android/libass/src/main/java/com/edde746/plezy/libass/media/parser/AssNoOpSubtitleParser.kt @@ -0,0 +1,24 @@ +package com.edde746.plezy.libass.media.parser + +import androidx.media3.common.Format +import androidx.media3.common.util.Consumer +import androidx.media3.common.util.UnstableApi +import androidx.media3.extractor.text.CuesWithTiming +import androidx.media3.extractor.text.SubtitleParser + +/** + * No operation subtitle parser. + */ +@UnstableApi +class AssNoOpSubtitleParser : SubtitleParser { + override fun parse( + p0: ByteArray, + p1: Int, + p2: Int, + p3: SubtitleParser.OutputOptions, + p4: Consumer + ) { + } + + override fun getCueReplacementBehavior(): Int = Format.CUE_REPLACEMENT_BEHAVIOR_REPLACE +} diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/media/parser/AssSubtitleParserFactory.kt b/android/libass/src/main/java/com/edde746/plezy/libass/media/parser/AssSubtitleParserFactory.kt new file mode 100644 index 00000000..458061af --- /dev/null +++ b/android/libass/src/main/java/com/edde746/plezy/libass/media/parser/AssSubtitleParserFactory.kt @@ -0,0 +1,32 @@ +package com.edde746.plezy.libass.media.parser + +import androidx.media3.common.Format +import androidx.media3.common.MimeTypes +import androidx.media3.common.util.UnstableApi +import androidx.media3.extractor.text.DefaultSubtitleParserFactory +import androidx.media3.extractor.text.SubtitleParser +import com.edde746.plezy.libass.media.AssHandler + +@UnstableApi +class AssSubtitleParserFactory(private val assHandler: AssHandler) : SubtitleParser.Factory { + + private val defaultSubtitleParserFactory = DefaultSubtitleParserFactory() + + override fun supportsFormat(format: Format): Boolean = defaultSubtitleParserFactory.supportsFormat(format) + + override fun getCueReplacementBehavior(format: Format): Int = defaultSubtitleParserFactory.getCueReplacementBehavior(format) + + override fun create(format: Format): SubtitleParser = if (format.sampleMimeType == MimeTypes.TEXT_SSA) { + val embeddedSubtitles = MimeTypes.VIDEO_MATROSKA + .contentEquals(format.containerMimeType) + val track = assHandler.createTrack(format) + if (embeddedSubtitles) { + // Embedded dialogue lines reach libass via AssTrackOutput; nothing to parse here. + AssNoOpSubtitleParser() + } else { + AssFullSubtitleParser(track) + } + } else { + defaultSubtitleParserFactory.create(format) + } +} diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/media/text/AssSubtitleExtractorOutput.kt b/android/libass/src/main/java/com/edde746/plezy/libass/media/text/AssSubtitleExtractorOutput.kt new file mode 100644 index 00000000..ed4836cf --- /dev/null +++ b/android/libass/src/main/java/com/edde746/plezy/libass/media/text/AssSubtitleExtractorOutput.kt @@ -0,0 +1,26 @@ +package com.edde746.plezy.libass.media.text + +import androidx.media3.common.C +import androidx.media3.common.util.UnstableApi +import androidx.media3.extractor.ExtractorOutput +import androidx.media3.extractor.TrackOutput +import com.edde746.plezy.libass.media.AssHandler +import com.edde746.plezy.libass.media.extractor.AssMatroskaExtractor + +/** + * This class is only used by the overlay renderer. It's needed to get the start time of the subtitles. + */ +@UnstableApi +class AssSubtitleExtractorOutput( + private val delegate: ExtractorOutput, + private val assHandler: AssHandler, + private val extractor: AssMatroskaExtractor +) : ExtractorOutput by delegate { + override fun track(id: Int, type: Int): TrackOutput = if (type == C.TRACK_TYPE_TEXT) { + // We can't know at this time if the subtitle track is ASS or other format, so we wrap + // every subtitle track + AssTrackOutput(delegate.track(id, type), assHandler, extractor) + } else { + delegate.track(id, type) + } +} diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/media/text/AssTrackOutput.kt b/android/libass/src/main/java/com/edde746/plezy/libass/media/text/AssTrackOutput.kt new file mode 100644 index 00000000..b68d6c05 --- /dev/null +++ b/android/libass/src/main/java/com/edde746/plezy/libass/media/text/AssTrackOutput.kt @@ -0,0 +1,95 @@ +package com.edde746.plezy.libass.media.text + +import androidx.media3.common.C +import androidx.media3.common.Format +import androidx.media3.common.MimeTypes +import androidx.media3.common.util.UnstableApi +import androidx.media3.common.util.Util +import androidx.media3.extractor.TrackOutput +import com.edde746.plezy.libass.media.AssHandler +import com.edde746.plezy.libass.media.extractor.AssMatroskaExtractor +import java.util.regex.Pattern + +/** + * This class is only used by the overlay renderer. It's needed to get the start time of the subtitles. + */ +@UnstableApi +class AssTrackOutput( + private val delegate: TrackOutput, + private val assHandler: AssHandler, + private val extractor: AssMatroskaExtractor +) : TrackOutput by delegate { + + private var isAss = false + + private var trackId: String? = null + + override fun format(format: Format) { + if (format.sampleMimeType == MimeTypes.TEXT_SSA || format.codecs == MimeTypes.TEXT_SSA) { + isAss = true + trackId = format.id + } + delegate.format(format) + } + + override fun sampleMetadata( + timeUs: Long, + flags: Int, + size: Int, + offset: Int, + cryptoData: TrackOutput.CryptoData? + ) { + if (isAss && timeUs.isValidTs) { + val sample = extractor.subtitleSample + val endIndex = findTokenIndex(sample.data, 1) + val lineIndex = findTokenIndex(sample.data, 2) + + val rawDuration = sample.data.decodeToString(endIndex, lineIndex - 1) + val durationUs = parseTimecodeUs(rawDuration) + + assHandler.readTrackDialogue( + trackId = trackId, + start = timeUs / 1000, + duration = durationUs / 1000, + data = sample.data, + offset = lineIndex, + length = sample.limit() - lineIndex + ) + } + delegate.sampleMetadata(timeUs, flags, size, offset, cryptoData) + } + + private fun parseTimecodeUs(timeString: String): Long { + val matcher = SSA_TIMECODE_PATTERN.matcher(timeString.trim { it <= ' ' }) + if (!matcher.matches()) { + return C.TIME_UNSET + } + var timestampUs = + Util.castNonNull(matcher.group(1)).toLong() * 60 * 60 * C.MICROS_PER_SECOND + timestampUs += Util.castNonNull(matcher.group(2)).toLong() * 60 * C.MICROS_PER_SECOND + timestampUs += Util.castNonNull(matcher.group(3)).toLong() * C.MICROS_PER_SECOND + timestampUs += Util.castNonNull(matcher.group(4)).toLong() * 10000 + return timestampUs + } + + private fun findTokenIndex(array: ByteArray, tokenNumber: Int): Int { + if (tokenNumber == 0) return 0 + var tokensFound = 0 + array.forEachIndexed { index, byte -> + if (byte == COMMA && ++tokensFound == tokenNumber) { + return index + 1 + } + } + return 0 + } + + private val Long.isValidTs + get() = this != C.TIME_UNSET + + private companion object { + val SSA_TIMECODE_PATTERN: Pattern = + Pattern.compile("""(?:(\d+):)?(\d+):(\d+)[:.](\d+)""") + + const val COMMA = ','.code.toByte() + } +} diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/AssSubtitleAtlasPipeline.kt b/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/AssSubtitleAtlasPipeline.kt new file mode 100644 index 00000000..b3a74011 --- /dev/null +++ b/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/AssSubtitleAtlasPipeline.kt @@ -0,0 +1,991 @@ +package com.edde746.plezy.libass.media.widget + +import android.opengl.EGL14 +import android.opengl.EGLContext +import android.opengl.EGLDisplay +import android.opengl.EGLExt +import android.opengl.EGLSurface +import android.opengl.GLES20 +import android.os.Handler +import android.os.HandlerThread +import android.os.Process +import android.util.Log +import android.view.Surface +import androidx.media3.common.C +import androidx.media3.common.util.GlProgram +import androidx.media3.common.util.GlUtil +import androidx.media3.common.util.Size +import androidx.media3.common.util.UnstableApi +import com.edde746.plezy.libass.AssAtlasFrame +import com.edde746.plezy.libass.media.AssHandler +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.concurrent.atomic.AtomicReference + +/** + * Atlas-rendering pipeline behind [AssSubtitleSurfaceView]. + * Runs libass on its own [HandlerThread] into a packed + * ALPHA_8 texture atlas plus a single vertex stream, and a GL thread that uploads + * both and issues one `glDrawArrays` per frame. Each swap is pinned to the video's + * target release time via [EGLExt.eglPresentationTimeANDROID]: SurfaceFlinger holds + * the buffer and composes it on the same vsync as the corresponding video frame. + */ +@UnstableApi +internal object AssAtlasPipelineConfig { + /** Flip to `true` for `adb logcat -s AssSurfaceGlThread:D AssLibassThread:D` traces. */ + internal const val TIMING_LOGS = false + + /** Fallback atlas row width when GL caps are unknown (EGL init failed/slow). */ + internal const val FALLBACK_ATLAS_W = 2048 + + /** Fallback atlas height; 2048 × 4096 = 8 MB matches the long-proven default. */ + internal const val FALLBACK_ATLAS_H = 4096 + + /** + * Pixel budget per atlas slot once GL caps are known: 4096×4096 or 2048×8192 + * (16 MB ALPHA_8). Sized so heavy typesetting (full-screen gradients/blurs) + * fits where the old fixed 2048×4096 overflowed and dropped frames. + */ + internal const val ATLAS_PIXEL_BUDGET = 16 * 1024 * 1024 + + /** Preallocated vertex-stream capacity (192 bytes × 16384 = 3 MB per buffer). */ + internal const val MAX_QUADS = 16384 + + /** Must match the byte layout produced by `nativeAssRenderFrameAtlas` in AssKt.c. */ + internal const val BYTES_PER_VERTEX = 32 + internal const val BYTES_PER_QUAD = BYTES_PER_VERTEX * 6 + + /** + * Kill switch for speculative render-ahead (see [SpecRenderEngine]) and the + * event prefetch. With it off (or on low-RAM devices, which stay at 2 slots) + * every request renders on-demand inside the video frame's release deadline — + * the pre-speculation behavior. + * + * Only pays off when typical changed renders fit the speculation coverage + * window (~frame interval + release budget): with the un-optimized (-O0) + * native core's 90ms+ renders it could only add overhead; with the -O3/NEON + * core's ~35-50ms renders it converts near-misses into on-time latches. + */ + internal const val SPECULATION_ENABLED = true +} + +/** Payload handed from the libass worker to the GL thread. */ +internal class AtlasPayload( + val slotIndex: Int, + val atlasBuf: ByteBuffer, + val vertexBuf: ByteBuffer, + var frame: AssAtlasFrame, + var presentationTimeUs: Long, + var releaseTimeNs: Long, + /** Bumped on every content-changing render; lets the GL thread tell "same slot, + * new content" apart from "same slot, same content" when deciding to re-upload. */ + var contentSeq: Long = 0L +) + +/** Payload slots plus the atlas dims their buffers were sized for. */ +internal class AtlasSlots( + val payloads: Array, + val atlasW: Int, + val atlasH: Int +) + +/** + * Owns both the libass worker and the GL thread. The two talk via a single-slot + * atomic — a newer payload always replaces a pending one so the GL thread never + * falls behind. + */ +@UnstableApi +internal class AssAtlasPipeline( + surface: Surface, + width: Int, + height: Int, + assHandler: AssHandler, + lowRamDevice: Boolean = false +) { + private val surfaceWidth = width + private val surfaceHeight = height + + // 3 slots give the render-ahead engine a writable target while one slot is + // posted and another is in GL's hands; low-RAM devices stay at 2 slots with + // speculation off (the legacy on-demand behavior, ~19 MB less in buffers). + private val slotCount = if (lowRamDevice) 2 else 3 + private val speculationEnabled = AssAtlasPipelineConfig.SPECULATION_ENABLED && slotCount >= 3 + + // Atlas dims, resolved exactly once (first-wins): normally by the GL thread from + // GL_MAX_TEXTURE_SIZE right after EGL init; by the libass thread's 1 s fallback + // if GL never comes up. Both threads then agree on the dims, which matters + // because the C side bakes UV denominators = these dims into the vertex stream + // and the GL side allocates the texture once at these dims. + private val dimsResolved = java.util.concurrent.atomic.AtomicBoolean(false) + private val dimsLatch = java.util.concurrent.CountDownLatch(1) + + @Volatile private var atlasW = 0 + + @Volatile private var atlasH = 0 + + private fun resolveAtlasDims(maxTextureSize: Int): Pair { + if (dimsResolved.compareAndSet(false, true)) { + if (maxTextureSize < 4096) { + // Query failed (or an ancient GPU): keep the long-proven fixed size. + atlasW = AssAtlasPipelineConfig.FALLBACK_ATLAS_W + atlasH = AssAtlasPipelineConfig.FALLBACK_ATLAS_H + } else { + val w = if (surfaceWidth > 2048) 4096 else 2048 + atlasW = w + atlasH = minOf(maxTextureSize, AssAtlasPipelineConfig.ATLAS_PIXEL_BUDGET / w) + } + if (AssAtlasPipelineConfig.TIMING_LOGS) { + Log.d("AssAtlasPipeline", "atlas dims ${atlasW}x$atlasH (glMaxTexture=$maxTextureSize)") + } + dimsLatch.countDown() + } + return atlasW to atlasH + } + + // Lazily allocated on the libass thread once atlas dims are known; ~19 MB of + // direct buffers per slot at the full budget, so don't pay it before the + // first actual render. Confined to the libass thread after creation. + private var slots: AtlasSlots? = null + + private fun acquireSlots(): AtlasSlots { + slots?.let { return it } + if (!dimsLatch.await(1, java.util.concurrent.TimeUnit.SECONDS)) { + resolveAtlasDims(0) // first-wins: no-op if the GL thread resolved meanwhile + } + val w = atlasW + val h = atlasH + val payloads = Array(slotCount) { index -> + AtlasPayload( + slotIndex = index, + atlasBuf = ByteBuffer.allocateDirect(w * h).order(ByteOrder.nativeOrder()), + vertexBuf = ByteBuffer.allocateDirect( + AssAtlasPipelineConfig.MAX_QUADS * AssAtlasPipelineConfig.BYTES_PER_QUAD + ).order(ByteOrder.nativeOrder()), + frame = AssAtlasFrame(0, 0, 0, 0, 0), + presentationTimeUs = 0L, + releaseTimeNs = C.TIME_UNSET + ) + } + return AtlasSlots(payloads, w, h).also { slots = it } + } + + /** Slot index the GL thread most recently took for drawing; the libass side + * never writes it. Written only inside [takePending] on the GL thread. */ + @Volatile private var glLastTakenSlot = -1 + + private val pendingPayload = AtomicReference(null) + private val glThread = AtlasGlThread( + surface, + width, + height, + assHandler, + takePending = { + pendingPayload.getAndSet(null)?.also { glLastTakenSlot = it.slotIndex } + }, + resolveAtlasDims = ::resolveAtlasDims + ) + private val libassThread = AtlasLibassThread( + assHandler, + acquireSlots = ::acquireSlots, + speculationEnabled = speculationEnabled, + glTakenSlot = { glLastTakenSlot }, + onFrameReady = { payload -> + pendingPayload.set(payload) + glThread.triggerDraw() + } + ) + + fun start() { + if (AssAtlasPipelineConfig.TIMING_LOGS) { + Log.d( + "AssAtlasPipeline", + "start surface=${surfaceWidth}x$surfaceHeight slots=$slotCount speculation=$speculationEnabled" + ) + } + glThread.start() + libassThread.start() + } + + fun requestRender(presentationTimeUs: Long, releaseTimeNs: Long) { + libassThread.enqueue(presentationTimeUs, releaseTimeNs) + } + + /** + * Re-renders the last requested position — for renderer state changes (margins, + * use-margins) that must become visible while playback is paused. Safe during + * playback: the next video frame's [requestRender] supersedes it (latest-wins). + */ + fun invalidate() { + libassThread.invalidate() + } + + fun onSurfaceSizeChanged(width: Int, height: Int) { + glThread.onSurfaceSizeChanged(width, height) + } + + /** Vsync-pinned swaps performed (excludes untimed invalidate repaints). */ + val swapCount: Long get() = glThread.swapCount + + /** Pinned swaps that finished past the swap-time budget (possible missed vsync). */ + val lateSwapCount: Long get() = glThread.lateSwapCount + + /** Worst observed swap lateness past the target release time, in milliseconds. */ + val maxLateMs: Long get() = glThread.maxLateMs + + /** Total libass renders performed (one per serviced request). */ + val renderCount: Long get() = libassThread.renderCount + + /** Renders where libass reported changed content (atlas/vertex rewritten). */ + val changedRenderCount: Long get() = libassThread.changedRenderCount + + /** Renders that overflowed the atlas/vertex capacity (frame content incomplete). */ + val overflowCount: Long get() = libassThread.overflowCount + + /** Duration of the most recent libass render, in milliseconds. */ + val lastLibassMs: Long get() = libassThread.lastLibassMs + + /** Worst observed libass render duration, in milliseconds. */ + val maxLibassMs: Long get() = libassThread.maxLibassMs + + /** Changed-render duration histogram: [≤10ms, ≤25ms, ≤42ms, ≤84ms, >84ms]. */ + val libassMsHistogram: List get() = libassThread.histogramSnapshot() + + /** Requests served from a pre-rendered (speculative) frame — GL-only hot path. */ + val specHits: Long get() = libassThread.specHits + + /** Requests where a speculative frame existed but didn't match (seek, state change). */ + val specMisses: Long get() = libassThread.specMisses + + /** Speculation rounds skipped (paused, pending request, no confident cadence). */ + val specSkips: Long get() = libassThread.specSkips + + /** Cache-warming prefetch renders of upcoming events. */ + val prefetchCount: Long get() = libassThread.prefetchCount + + /** Worst (minimum) lead of a changed-content pinned swap vs its target release + * time, in ms; negative = the new content was queued after the video frame's + * vsync. Long.MAX_VALUE until a changed pinned swap happened. */ + val minLeadChangedMs: Long get() = glThread.minLeadChangedMs + + fun releaseAndWait() { + libassThread.releaseAndWait() + glThread.releaseAndWait() + } +} + +/** + * Stops a [Handler] synchronously: posts [releaseWhat] with an [Ack], waits up to 1 s + * for the handler to invoke [onReleased] (on its own looper) and signal the latch. + */ +private fun postShutdownAndWait( + handler: Handler, + releaseWhat: Int, + onReleased: () -> Unit +) { + val latch = Object() + synchronized(latch) { + handler.obtainMessage(releaseWhat, Ack(latch, onReleased)).sendToTarget() + try { + latch.wait(1_000) + } catch (_: InterruptedException) { + Thread.currentThread().interrupt() + } + } +} + +/** Transport for [postShutdownAndWait] — the handler callback calls [release] then notifies. */ +private class Ack(val latch: Any, val release: () -> Unit) + +/** + * Runs libass off the GL thread into a packed atlas + vertex stream. Latest-wins: + * older pending renders are dropped when a newer one arrives. Slot choice, the + * changed-flag bookkeeping and speculative render-ahead live in [SpecRenderEngine]; + * this thread owns the buffers, the timing/stat accounting and the GL handoff. + */ +@UnstableApi +private class AtlasLibassThread( + private val assHandler: AssHandler, + private val acquireSlots: () -> AtlasSlots, + private val speculationEnabled: Boolean, + private val glTakenSlot: () -> Int, + private val onFrameReady: (AtlasPayload) -> Unit +) : HandlerThread(TAG, Process.THREAD_PRIORITY_DISPLAY) { + + /** Immutable (pts, release) request — handed off through a single atomic so a + * concurrent enqueue can neither be lost by drain's consume nor torn in half. + * [enqueueNs] timestamps the handoff so drain can report how long the request + * sat behind an in-flight render (the queue-wait component of subtitle lag). */ + private class PendingFrame(val ptsUs: Long, val releaseNs: Long, val enqueueNs: Long = System.nanoTime()) + + private lateinit var handler: Handler + + private val pending = AtomicReference(null) + + @Volatile private var lastRequestedPtsUs = UNSET + private var contentSeqCounter = 0L + + // Thread-confined; created on first render so non-ASS playback never allocates. + private var engine: SpecRenderEngine? = null + + val specHits: Long get() = engine?.specHits ?: 0L + val specMisses: Long get() = engine?.specMisses ?: 0L + val specSkips: Long get() = engine?.specSkips ?: 0L + val prefetchCount: Long get() = engine?.prefetchCount ?: 0L + + // Telemetry; single-writer (this thread), read from the stats path. + @Volatile var renderCount = 0L + private set + + @Volatile var changedRenderCount = 0L + private set + + @Volatile var overflowCount = 0L + private set + + @Volatile var lastLibassMs = 0L + private set + + @Volatile var maxLibassMs = 0L + private set + + /** Changed-render durations bucketed at ≤10 / ≤25 / ≤42 / ≤84 / >84 ms. */ + private val histogram = java.util.concurrent.atomic.AtomicLongArray(5) + + fun histogramSnapshot(): List = List(histogram.length()) { histogram.get(it) } + + private fun recordChangedRenderMs(ms: Long) { + val bucket = when { + ms <= 10 -> 0 + ms <= 25 -> 1 + ms <= 42 -> 2 + ms <= 84 -> 3 + else -> 4 + } + histogram.incrementAndGet(bucket) + } + + override fun start() { + super.start() + handler = Handler(looper) { msg -> + when (msg.what) { + MSG_RENDER -> drainAndRender() + MSG_RELEASE -> { + val ack = msg.obj as Ack + ack.release() + quit() + synchronized(ack.latch) { (ack.latch as Object).notifyAll() } + } + } + true + } + } + + fun enqueue(presentationTimeUs: Long, releaseTimeNs: Long) { + if (!::handler.isInitialized) return + val dropped = pending.getAndSet(PendingFrame(presentationTimeUs, releaseTimeNs)) + if (dropped != null && AssAtlasPipelineConfig.TIMING_LOGS) { + // A request was coalesced away — the renderer is behind by at least one + // frame. agedMs = how long the dropped request had been waiting. + Log.d( + TAG, + "drop pts=${dropped.ptsUs / 1000}ms agedMs=${(System.nanoTime() - dropped.enqueueNs) / 1_000_000} " + + "replacedBy=${presentationTimeUs / 1000}ms" + ) + } + handler.removeMessages(MSG_RENDER) + handler.sendEmptyMessage(MSG_RENDER) + } + + /** Re-enqueues the last requested PTS (renderer state changed, possibly while paused). */ + fun invalidate() { + val pts = lastRequestedPtsUs + if (pts == UNSET) return + // TIME_UNSET release time => the GL thread swaps immediately instead of + // vsync-pinning to a video frame that may never come while paused. + enqueue(pts, C.TIME_UNSET) + } + + private fun ensureEngine(slots: AtlasSlots): SpecRenderEngine { + engine?.let { return it } + return SpecRenderEngine( + slotCount = slots.payloads.size, + speculationEnabled = speculationEnabled, + renderAt = { timeMs, slot -> timedRender(timeMs, slots, slot) }, + // Renderer identity in the high bits + its state generation in the low bits: + // a recreated renderer (media item transition) can never alias a stale + // speculation, even if the new generation counter happens to match. + stateGeneration = { + assHandler.render?.let { + (System.identityHashCode(it).toLong() shl 32) or (it.stateGeneration.toLong() and 0xffffffffL) + } ?: -1L + }, + glTakenSlot = glTakenSlot, + debugLog = if (AssAtlasPipelineConfig.TIMING_LOGS) ({ msg -> Log.d(TAG, msg) }) else null + ).also { engine = it } + } + + /** Consecutive renders that reported no content change — a static screen. + * Reset by any changed render (dialogue flips, karaoke, animated signs). */ + private var unchangedStreak = 0 + + /** Renders into [slot]'s buffers, owning the per-render timing/stat accounting + * for both on-demand and speculative renders. */ + private fun timedRender(timeMs: Long, slots: AtlasSlots, slot: Int): AssAtlasFrame? { + val render = assHandler.render ?: return null + val payload = slots.payloads[slot] + val t0 = System.nanoTime() + val frame = render.renderFrameAtlas(timeMs, payload.atlasBuf, slots.atlasW, slots.atlasH, payload.vertexBuf) + ?: return null + val libassMs = (System.nanoTime() - t0) / 1_000_000 + renderCount++ + lastLibassMs = libassMs + if (libassMs > maxLibassMs) maxLibassMs = libassMs + if (frame.truncated > 0) overflowCount++ + if (frame.changed != 0) { + changedRenderCount++ + recordChangedRenderMs(libassMs) + unchangedStreak = 0 + } else { + unchangedStreak++ + } + return frame + } + + private fun drainAndRender() { + val request = pending.getAndSet(null) ?: return + val pts = request.ptsUs + val releaseNs = request.releaseNs + lastRequestedPtsUs = pts + val tDrain = System.nanoTime() + // How long the request sat in the handoff (behind an in-flight on-demand or + // speculative render) — the queue-wait component of any subtitle lag. + val waitMs = (tDrain - request.enqueueNs) / 1_000_000 + // Before any ASS render exists (SRT/VTT or no subs) do nothing — this also + // keeps the slot buffers unallocated for non-ASS playback. + if (assHandler.render == null) return + val slots = acquireSlots() + val engine = ensureEngine(slots) + val pinned = releaseNs != C.TIME_UNSET + // Budget left until the video frame's vsync when we START servicing. + val budgetMs = if (pinned) (releaseNs - tDrain) / 1_000_000 else -1L + + when (val outcome = engine.service(pts, pinned)) { + is SpecRenderEngine.Outcome.Post -> { + val payload = slots.payloads[outcome.slot] + if (outcome.newContent) { + payload.frame = outcome.frame + payload.contentSeq = ++contentSeqCounter + } + payload.presentationTimeUs = pts + payload.releaseTimeNs = releaseNs + onFrameReady(payload) + if (AssAtlasPipelineConfig.TIMING_LOGS) { + Log.d( + TAG, + "render pts=${pts / 1000}ms seq=${payload.contentSeq} waitMs=$waitMs budgetMs=$budgetMs " + + "libassMs=$lastLibassMs lockWaitMs=${assHandler.render?.lastLockWaitMs} " + + "specHit=${outcome.specHit} changed=${payload.frame.changed} quads=${payload.frame.quadCount} " + + "atlas=${payload.frame.atlasWidth}x${payload.frame.atlasHeight} truncated=${payload.frame.truncated}" + ) + } + } + SpecRenderEngine.Outcome.Skip -> { + if (AssAtlasPipelineConfig.TIMING_LOGS) { + Log.d(TAG, "skip pts=${pts / 1000}ms waitMs=$waitMs budgetMs=$budgetMs (no content)") + } + } + } + + // Pre-render the predicted next frame in the dead time between requests so the + // next service is (usually) a GL-only hit. Never delays a waiting request. + engine.speculateAfter(pts, pinned, hasPending = pending.get() != null)?.let { write -> + val payload = slots.payloads[write.slot] + payload.frame = write.frame + payload.contentSeq = ++contentSeqCounter + if (AssAtlasPipelineConfig.TIMING_LOGS) { + Log.d( + TAG, + "spec after=${pts / 1000}ms seq=${payload.contentSeq} libassMs=$lastLibassMs " + + "lockWaitMs=${assHandler.render?.lastLockWaitMs} slot=${write.slot} quads=${write.frame.quadCount}" + ) + } + } + + maybePrefetch(engine, slots, pts, pinned) + } + + /** Start time of the last event boundary we cache-warmed (libass/track ms). */ + private var lastPrefetchedStartMs = Long.MIN_VALUE + + /** Wall time of the last prefetch render, for the cooldown. */ + private var lastPrefetchNs = Long.MIN_VALUE / 2 + + /** + * Cache-warms the next upcoming subtitle event so heavy typesetting pays its + * cache-cold rasterization (seconds on weak devices) before the sign appears + * instead of at appearance. + * + * The render blocks this thread, and during playback a new request is never + * more than one frame interval away — so a prefetch is only allowed when its + * delay cannot be SEEN, not merely when the queue is momentarily empty + * (the v1 mistake, which thrashed on densely-authored per-frame events and + * stalled visible dialogue): + * - the screen must be static ([unchangedStreak]): requests delayed behind + * the prefetch re-render identical content, so their lateness is invisible; + * - the warmed event must be the NEXT on-screen change (no other event start + * or end before it) — this also kills dense per-frame event sections, + * where the next change is always ≤ one frame away; + * - a cooldown bounds the worst-case overhead to one render per window. + */ + private fun maybePrefetch(engine: SpecRenderEngine, slots: AtlasSlots, ptsUs: Long, pinned: Boolean) { + if (!speculationEnabled) return + if (!pinned) return + if (pending.get() != null) return + if (unchangedStreak < PREFETCH_STATIC_STREAK) return + val now = System.nanoTime() + if (now - lastPrefetchNs < PREFETCH_COOLDOWN_NS) return + val track = assHandler.track ?: return + val nowMs = ptsUs / 1000 + // Events closer than MIN_AHEAD are the regular per-frame path's business; + // beyond HORIZON the warmed bitmaps may be evicted before they're needed. + val targetMs = track.nextEventStartMs(nowMs + PREFETCH_MIN_AHEAD_MS) + if (targetMs < 0 || targetMs == lastPrefetchedStartMs) return + if (targetMs > nowMs + PREFETCH_HORIZON_MS) return + // Invisibility gate, time-budgeted: a prefetch is invisible as long as it + // finishes before anything on screen is due to change (the screen is static + // per the streak gate, so requests delayed behind it re-render identical + // content). Estimate the cost from the worst render seen this session — + // an overestimate only skips warming; an underestimate delays one boundary + // by the shortfall. nextEventChangeMs also sees ends, so a dialogue line + // due to disappear inside the budget skips the prefetch. + val nextChangeMs = track.nextEventChangeMs(nowMs) + if (nextChangeMs in 0 until targetMs) { + val runwayMs = nextChangeMs - nowMs + val costEstimateMs = (maxLibassMs * 5 / 4).coerceIn(PREFETCH_COST_FLOOR_MS, PREFETCH_COST_CEIL_MS) + if (runwayMs < costEstimateMs + PREFETCH_SAFETY_MS) return + } + lastPrefetchedStartMs = targetMs + lastPrefetchNs = now + engine.prefetch(targetMs * 1000)?.let { write -> + val payload = slots.payloads[write.slot] + payload.frame = write.frame + payload.contentSeq = ++contentSeqCounter + } + if (AssAtlasPipelineConfig.TIMING_LOGS) { + Log.d( + TAG, + "prefetch evt=${targetMs}ms aheadMs=${targetMs - nowMs} " + + "tookMs=${(System.nanoTime() - now) / 1_000_000} libassMs=$lastLibassMs" + ) + } + } + + fun releaseAndWait() { + if (!::handler.isInitialized) { + quit() + return + } + postShutdownAndWait(handler, MSG_RELEASE) { /* nothing thread-local to tear down */ } + } + + companion object { + private const val TAG = "AssLibassThread" + private const val MSG_RENDER = 1 + private const val MSG_RELEASE = 2 + private const val UNSET = Long.MIN_VALUE + + /** Don't prefetch events the per-frame speculation will reach imminently. */ + private const val PREFETCH_MIN_AHEAD_MS = 1_000L + + /** Don't warm caches so early that the bitmaps could be evicted again. */ + private const val PREFETCH_HORIZON_MS = 15_000L + + /** Static-screen requirement before a prefetch may block this thread. */ + private const val PREFETCH_STATIC_STREAK = 3 + + /** Minimum spacing between prefetch renders. */ + private const val PREFETCH_COOLDOWN_NS = 2_000_000_000L + + /** Cost-estimate clamp for the time-budgeted invisibility gate: never + * assume a prefetch cheaper than the floor (estimator may not have seen a + * heavy frame yet) nor pointlessly demand more runway than the ceiling. */ + private const val PREFETCH_COST_FLOOR_MS = 250L + private const val PREFETCH_COST_CEIL_MS = 1_500L + + /** Slack added to the cost estimate when checking the static runway. */ + private const val PREFETCH_SAFETY_MS = 150L + } +} + +/** + * Owns the EGL surface, uploads the atlas + vertex stream and issues a single + * `glDrawArrays` per frame. Swaps immediately with the swap pinned to the video's + * target release time via [EGLExt.eglPresentationTimeANDROID]; SurfaceFlinger + * holds the buffer until then, so the thread is never blocked waiting for a vsync. + */ +@UnstableApi +private class AtlasGlThread( + private val surface: Surface, + @Volatile private var width: Int, + @Volatile private var height: Int, + private val assHandler: AssHandler, + private val takePending: () -> AtlasPayload?, + private val resolveAtlasDims: (maxTextureSize: Int) -> Pair +) : HandlerThread(TAG, Process.THREAD_PRIORITY_DISPLAY) { + + private lateinit var handler: Handler + private var eglDisplay: EGLDisplay = EGL14.EGL_NO_DISPLAY + private var eglContext: EGLContext = EGL14.EGL_NO_CONTEXT + private var eglSurface: EGLSurface = EGL14.EGL_NO_SURFACE + + private val renderer = AtlasRenderer(assHandler) + private var lastUploadedPayload: AtlasPayload? = null + private var lastUploadedSeq = -1L + private var lastSwappedSeq = -1L + + // Lateness telemetry; single-writer (this thread), read from the stats path. + @Volatile var swapCount = 0L + private set + + @Volatile var lateSwapCount = 0L + private set + + @Volatile var maxLateMs = 0L + private set + + /** Minimum lead (release target − swap completion) over changed-content pinned + * swaps; negative = content queued after the video frame's vsync. */ + @Volatile var minLeadChangedMs = Long.MAX_VALUE + private set + + override fun start() { + super.start() + handler = Handler(looper) { msg -> + try { + when (msg.what) { + MSG_INIT -> initEgl() + MSG_DRAW -> drawAndSwap() + MSG_SIZE_CHANGED -> sizeChanged(width, height) + MSG_RELEASE -> { + val ack = msg.obj as Ack + ack.release() + quit() + synchronized(ack.latch) { (ack.latch as Object).notifyAll() } + } + } + } catch (e: Exception) { + Log.e(TAG, "GL thread error", e) + releaseEgl() + } + true + } + handler.sendEmptyMessage(MSG_INIT) + } + + fun onSurfaceSizeChanged(width: Int, height: Int) { + this.width = width + this.height = height + handler.sendEmptyMessage(MSG_SIZE_CHANGED) + } + + fun triggerDraw() { + if (!::handler.isInitialized) return + handler.removeMessages(MSG_DRAW) + handler.sendEmptyMessage(MSG_DRAW) + } + + fun releaseAndWait() { + if (!::handler.isInitialized) { + quit() + return + } + postShutdownAndWait(handler, MSG_RELEASE) { releaseEgl() } + } + + private fun initEgl() { + try { + eglDisplay = GlUtil.getDefaultEglDisplay() + eglContext = GlUtil.createEglContext(eglDisplay) + eglSurface = GlUtil.createEglSurface(eglDisplay, surface, C.COLOR_TRANSFER_SDR, false) + EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext) + renderer.onSurfaceCreated() + // Resolve atlas dims from real GL caps (first-wins against the libass + // thread's fallback) and allocate the texture once at those dims — uploads + // are glTexSubImage2D of the packed rows from then on. + val maxTexture = IntArray(1) + GLES20.glGetIntegerv(GLES20.GL_MAX_TEXTURE_SIZE, maxTexture, 0) + val (atlasW, atlasH) = resolveAtlasDims(maxTexture[0]) + renderer.allocateAtlasTexture(atlasW, atlasH) + sizeChanged(width, height) + } catch (e: GlUtil.GlException) { + Log.e(TAG, "Failed to initialize EGL", e) + } + } + + private fun sizeChanged(width: Int, height: Int) { + renderer.onSurfaceChanged(width, height) + if (eglDisplay != EGL14.EGL_NO_DISPLAY) { + GlUtil.clearFocusedBuffers() + EGL14.eglSwapBuffers(eglDisplay, eglSurface) + } + } + + private fun drawAndSwap() { + if (eglDisplay == EGL14.EGL_NO_DISPLAY) return + val payload = takePending() ?: return + + // Render immediately (GL commands queue on the GPU). Re-upload only when the + // slot's content actually changed — identity alone is not enough because the + // libass side rewrites slot buffers in place (contentSeq tracks the rewrites). + val t0 = System.nanoTime() + val reuse = payload === lastUploadedPayload && payload.contentSeq == lastUploadedSeq + renderer.onDrawFrame(payload, reuseUploads = reuse) + lastUploadedPayload = payload + lastUploadedSeq = payload.contentSeq + val t1 = System.nanoTime() + + // Swap immediately with the presentation time set: SurfaceFlinger holds the + // queued buffer until the video frame's target release time, so the subtitle + // can never appear early, and the GL thread is free again within a couple of + // milliseconds. Sleeping here until near the target vsync (as a removed + // TextureView path once required) made this thread blind for almost a whole + // frame interval — the single-slot latest-wins handoff would then drop an + // intermediate subtitle state (e.g. the transition to blank), showing the + // previous state one frame too long. + if (payload.releaseTimeNs != C.TIME_UNSET) { + EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, payload.releaseTimeNs) + } + EGL14.eglSwapBuffers(eglDisplay, eglSurface) + val t2 = System.nanoTime() + if (payload.releaseTimeNs != C.TIME_UNSET) { + swapCount++ + val lateNs = t2 - payload.releaseTimeNs + if (lateNs > LATE_THRESHOLD_NS) { + lateSwapCount++ + val lateMs = lateNs / 1_000_000 + if (lateMs > maxLateMs) maxLateMs = lateMs + } + // Lead of changed-content swaps is the frame-perfection signal: ≥ 0 means + // the new subtitle content reached the queue before the video frame's vsync. + if (payload.contentSeq != lastSwappedSeq) { + val leadMs = -lateNs / 1_000_000 + if (leadMs < minLeadChangedMs) minLeadChangedMs = leadMs + } + } + lastSwappedSeq = payload.contentSeq + if (AssAtlasPipelineConfig.TIMING_LOGS) { + val pinned = payload.releaseTimeNs != C.TIME_UNSET + // headroomMs: slack before the target vsync when GL STARTED; leadMs: slack + // when the buffer was actually queued (negative = queued after the vsync). + val headroomMs = if (pinned) (payload.releaseTimeNs - t0) / 1_000_000 else -1L + val leadMs = if (pinned) (payload.releaseTimeNs - t2) / 1_000_000 else -1L + Log.d( + TAG, + "swap pts=${payload.presentationTimeUs / 1000}ms seq=${payload.contentSeq} " + + "quads=${payload.frame.quadCount} reused=$reuse drawMs=${(t1 - t0) / 1_000_000} " + + "swapMs=${(t2 - t1) / 1_000_000} headroomMs=$headroomMs leadMs=$leadMs pinned=$pinned" + ) + } + } + + private fun releaseEgl() { + if (eglDisplay != EGL14.EGL_NO_DISPLAY) { + try { + renderer.onSurfaceDestroyed() + GlUtil.destroyEglSurface(eglDisplay, eglSurface) + GlUtil.destroyEglContext(eglDisplay, eglContext) + } catch (e: GlUtil.GlException) { + Log.e(TAG, "Failed to release EGL", e) + } finally { + eglDisplay = EGL14.EGL_NO_DISPLAY + eglContext = EGL14.EGL_NO_CONTEXT + eglSurface = EGL14.EGL_NO_SURFACE + } + } + } + + companion object { + private const val TAG = "AssSurfaceGlThread" + private const val MSG_INIT = 1 + private const val MSG_DRAW = 2 + private const val MSG_SIZE_CHANGED = 3 + private const val MSG_RELEASE = 4 + + /** + * Swaps finishing this far past the target release time arrived after the + * buffer should already have been queued and may miss the frame's vsync — + * actual slack depends on the display's vsync offset from the release time. + */ + private const val LATE_THRESHOLD_NS = 4_000_000L + } +} + +/** + * GL-side work for the atlas-based path. Maintains a single atlas texture and a + * single vertex buffer; uploads them per frame (unless the payload identity + * matches the last upload) and issues one `glDrawArrays` for the whole frame. + */ +@UnstableApi +private class AtlasRenderer(private val assHandler: AssHandler) { + + private val vertexShaderCode = """ + attribute vec2 a_Position; + attribute vec2 a_TexCoord; + attribute vec4 a_Color; + uniform vec2 u_SurfaceSize; + varying vec2 v_TexCoord; + varying vec4 v_Color; + void main() { + vec2 clip = (a_Position / u_SurfaceSize) * 2.0 - 1.0; + clip.y = -clip.y; + gl_Position = vec4(clip, 0.0, 1.0); + v_TexCoord = a_TexCoord; + v_Color = a_Color; + } + """.trimIndent() + + private val fragmentShaderCode = """ + precision mediump float; + varying vec2 v_TexCoord; + varying vec4 v_Color; + uniform sampler2D u_Texture; + void main() { + float alpha = texture2D(u_Texture, v_TexCoord).a; + gl_FragColor = v_Color * alpha; + } + """.trimIndent() + + private var surfaceSize = Size.ZERO + private lateinit var glProgram: GlProgram + + private var atlasTexId = 0 + private var vertexBufferId = 0 + + private var aPosition = 0 + private var aTexCoord = 0 + private var aColor = 0 + private var uTexture = 0 + private var uSurfaceSize = 0 + + private var atlasAllocatedW = 0 + private var atlasAllocatedH = 0 + + /** + * Allocates the atlas texture once at the resolved dims. The C side bakes UV + * denominators = these dims into the vertex stream, so per-frame uploads can be + * partial ([uploadAtlas]) without ever reallocating — drivers keep one stable + * texture allocation instead of churning on packed-height changes. + */ + fun allocateAtlasTexture(width: Int, height: Int) { + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, atlasTexId) + GLES20.glTexImage2D( + GLES20.GL_TEXTURE_2D, 0, GLES20.GL_ALPHA, + width, height, 0, + GLES20.GL_ALPHA, GLES20.GL_UNSIGNED_BYTE, null + ) + atlasAllocatedW = width + atlasAllocatedH = height + } + + fun onSurfaceCreated() { + glProgram = GlProgram(vertexShaderCode, fragmentShaderCode) + GlUtil.checkGlError() + glProgram.use() + + aPosition = glProgram.getAttributeArrayLocationAndEnable("a_Position") + aTexCoord = glProgram.getAttributeArrayLocationAndEnable("a_TexCoord") + aColor = glProgram.getAttributeArrayLocationAndEnable("a_Color") + uTexture = glProgram.getUniformLocation("u_Texture") + uSurfaceSize = glProgram.getUniformLocation("u_SurfaceSize") + + val tex = IntArray(1) + GLES20.glGenTextures(1, tex, 0) + atlasTexId = tex[0] + GLES20.glActiveTexture(GLES20.GL_TEXTURE0) + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, atlasTexId) + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE) + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE) + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR) + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR) + GLES20.glUniform1i(uTexture, 0) + + val buf = IntArray(1) + GLES20.glGenBuffers(1, buf, 0) + vertexBufferId = buf[0] + + GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1) + GLES20.glEnable(GLES20.GL_BLEND) + GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA) + } + + fun onSurfaceChanged(width: Int, height: Int) { + surfaceSize = Size(width, height) + assHandler.render?.setFrameSize(width, height) + GLES20.glViewport(0, 0, width, height) + GLES20.glUniform2f(uSurfaceSize, width.toFloat(), height.toFloat()) + } + + fun onDrawFrame(payload: AtlasPayload, reuseUploads: Boolean) { + GlUtil.clearFocusedBuffers() + + val frame = payload.frame + val quadCount = frame.quadCount + if (quadCount == 0) return + + if (!reuseUploads) { + uploadAtlas(payload.atlasBuf, frame.atlasWidth, frame.atlasHeight) + uploadVertices(payload.vertexBuf, quadCount) + } + + GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vertexBufferId) + val stride = AssAtlasPipelineConfig.BYTES_PER_VERTEX + GLES20.glVertexAttribPointer(aPosition, 2, GLES20.GL_FLOAT, false, stride, 0) + GLES20.glVertexAttribPointer(aTexCoord, 2, GLES20.GL_FLOAT, false, stride, 8) + GLES20.glVertexAttribPointer(aColor, 4, GLES20.GL_FLOAT, false, stride, 16) + GLES20.glDrawArrays(GLES20.GL_TRIANGLES, 0, quadCount * 6) + } + + private fun uploadAtlas(atlasBuf: ByteBuffer, atlasW: Int, atlasH: Int) { + atlasBuf.position(0).limit(atlasW * atlasH) + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, atlasTexId) + if (atlasW == atlasAllocatedW && atlasH <= atlasAllocatedH) { + // Steady state: packed rows into the once-allocated texture. + GLES20.glTexSubImage2D( + GLES20.GL_TEXTURE_2D, 0, 0, 0, atlasW, atlasH, + GLES20.GL_ALPHA, GLES20.GL_UNSIGNED_BYTE, atlasBuf + ) + } else { + // Defensive: dims disagree with the allocation (shouldn't happen — both + // sides resolve dims through the same first-wins gate). + Log.w("AssAtlasRenderer", "atlas upload ${atlasW}x$atlasH outside allocation ${atlasAllocatedW}x$atlasAllocatedH") + GLES20.glTexImage2D( + GLES20.GL_TEXTURE_2D, 0, GLES20.GL_ALPHA, + atlasW, atlasH, 0, + GLES20.GL_ALPHA, GLES20.GL_UNSIGNED_BYTE, atlasBuf + ) + atlasAllocatedW = atlasW + atlasAllocatedH = atlasH + } + } + + private fun uploadVertices(vertexBuf: ByteBuffer, quadCount: Int) { + val size = quadCount * AssAtlasPipelineConfig.BYTES_PER_QUAD + vertexBuf.position(0).limit(size) + GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vertexBufferId) + GLES20.glBufferData(GLES20.GL_ARRAY_BUFFER, size, vertexBuf, GLES20.GL_STREAM_DRAW) + } + + fun onSurfaceDestroyed() { + if (atlasTexId != 0) { + val tex = intArrayOf(atlasTexId) + GLES20.glDeleteTextures(1, tex, 0) + atlasTexId = 0 + } + if (vertexBufferId != 0) { + val buf = intArrayOf(vertexBufferId) + GLES20.glDeleteBuffers(1, buf, 0) + vertexBufferId = 0 + } + if (::glProgram.isInitialized) glProgram.delete() + } +} diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/AssSubtitleSurfaceView.kt b/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/AssSubtitleSurfaceView.kt new file mode 100644 index 00000000..b1ed14f9 --- /dev/null +++ b/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/AssSubtitleSurfaceView.kt @@ -0,0 +1,103 @@ +package com.edde746.plezy.libass.media.widget + +import android.app.ActivityManager +import android.content.Context +import android.graphics.PixelFormat +import android.view.SurfaceHolder +import android.view.SurfaceView +import androidx.media3.common.util.UnstableApi +import com.edde746.plezy.libass.media.AssHandler + +/** + * Subtitle overlay rendered through a dedicated [SurfaceView] layer. + * + * Uses a SurfaceFlinger layer directly, which lets the atlas-based pipeline + * vsync-align its swap with the corresponding video frame via + * `eglPresentationTimeANDROID`. + */ +@UnstableApi +class AssSubtitleSurfaceView( + context: Context, + private val assHandler: AssHandler +) : SurfaceView(context), + SurfaceHolder.Callback { + + private var pipeline: AssAtlasPipeline? = null + + init { + setZOrderMediaOverlay(true) + holder.setFormat(PixelFormat.TRANSLUCENT) + holder.addCallback(this) + } + + fun requestRender(presentationTimeUs: Long, releaseTimeNs: Long) { + pipeline?.requestRender(presentationTimeUs, releaseTimeNs) + } + + /** Re-renders the last position, e.g. after margin changes while paused. */ + fun invalidateSubtitles() { + pipeline?.invalidate() + } + + /** Vsync-pinned swaps performed by the current pipeline. */ + val swapCount: Long get() = pipeline?.swapCount ?: 0L + + /** Pinned swaps that finished past the swap-time budget (possible missed vsync). */ + val lateSwapCount: Long get() = pipeline?.lateSwapCount ?: 0L + + /** Worst observed swap lateness past the target release time, in milliseconds. */ + val maxLateMs: Long get() = pipeline?.maxLateMs ?: 0L + + /** Total libass renders performed by the current pipeline. */ + val renderCount: Long get() = pipeline?.renderCount ?: 0L + + /** Renders where libass reported changed content. */ + val changedRenderCount: Long get() = pipeline?.changedRenderCount ?: 0L + + /** Renders that overflowed the atlas/vertex capacity. */ + val overflowCount: Long get() = pipeline?.overflowCount ?: 0L + + /** Duration of the most recent libass render, in milliseconds. */ + val lastLibassMs: Long get() = pipeline?.lastLibassMs ?: 0L + + /** Worst observed libass render duration, in milliseconds. */ + val maxLibassMs: Long get() = pipeline?.maxLibassMs ?: 0L + + /** Changed-render duration histogram: [≤10ms, ≤25ms, ≤42ms, ≤84ms, >84ms]. */ + val libassMsHistogram: List get() = pipeline?.libassMsHistogram ?: emptyList() + + /** Requests served from a pre-rendered (speculative) frame. */ + val specHits: Long get() = pipeline?.specHits ?: 0L + + /** Requests where speculation existed but didn't match (seek, state change). */ + val specMisses: Long get() = pipeline?.specMisses ?: 0L + + /** Speculation rounds skipped (paused, pending request, no confident cadence). */ + val specSkips: Long get() = pipeline?.specSkips ?: 0L + + /** Cache-warming prefetch renders of upcoming events. */ + val prefetchCount: Long get() = pipeline?.prefetchCount ?: 0L + + /** Minimum lead of changed-content pinned swaps vs the video frame's release + * time, in ms (negative = late); null until one happened. */ + val minLeadChangedMs: Long? get() = pipeline?.minLeadChangedMs?.takeIf { it != Long.MAX_VALUE } + + override fun surfaceCreated(holder: SurfaceHolder) { + val rect = holder.surfaceFrame + assHandler.setOverlaySurfaceSize(rect.width(), rect.height()) + val lowRam = (context.getSystemService(Context.ACTIVITY_SERVICE) as? ActivityManager) + ?.isLowRamDevice ?: false + pipeline = AssAtlasPipeline(holder.surface, rect.width(), rect.height(), assHandler, lowRam) + .also { it.start() } + } + + override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) { + assHandler.setOverlaySurfaceSize(width, height) + pipeline?.onSurfaceSizeChanged(width, height) + } + + override fun surfaceDestroyed(holder: SurfaceHolder) { + pipeline?.releaseAndWait() + pipeline = null + } +} diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/SpecRenderEngine.kt b/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/SpecRenderEngine.kt new file mode 100644 index 00000000..abaf5cef --- /dev/null +++ b/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/SpecRenderEngine.kt @@ -0,0 +1,265 @@ +package com.edde746.plezy.libass.media.widget + +import com.edde746.plezy.libass.AssAtlasFrame +import kotlin.math.abs + +/** + * Decision core of the speculative render-ahead pipeline. + * + * The problem it solves: ExoPlayer's frame-metadata callback gives only ~40 ms + * between "video frame scheduled" and that frame's vsync, while a changed-content + * libass render can take longer — so a render started at the callback lands a + * frame (or more) late. The engine renders the *predicted next* PTS right after + * servicing the current one, in the dead time between frames; when the next + * request matches the prediction, only the GL upload+swap (~6–11 ms) sits inside + * the deadline. + * + * Pure logic, single-threaded (the libass worker) — rendering, buffers and GL + * are behind injected closures, so the state machine is unit-testable. Only the + * stat fields are read from other threads. + * + * Slot ownership invariant: a slot handed to GL ([lastPostedSlot]) or currently + * read by GL ([glTakenSlot]) is never chosen as a render target. With 3 slots and + * at most 2 exclusions a target always exists; with 2 slots (low-RAM devices) + * speculation must be disabled and the legacy alternation behavior falls out. + * + * Key subtlety inherited from libass: `ass_render_frame`'s `changed` flag compares + * against libass's *previous render* — which, with speculation, may be content + * that never reached the screen. The engine therefore tracks [libassLastSlot] + * (the slot holding libass's most recent render output) rather than the last + * *posted* slot, and prefers it as the render target: on `changed == 0` the + * buffers were untouched and that slot already holds exactly the right content. + */ +internal class SpecRenderEngine( + private val slotCount: Int, + private val speculationEnabled: Boolean, + /** + * Renders the current track at [timeMs] into the slot's buffers. Returns null + * when the renderer/track is gone. The closure owns timing/stat accounting. + */ + private val renderAt: (timeMs: Long, slot: Int) -> AssAtlasFrame?, + /** + * Renderer identity + state generation as one comparable value. Must change + * whenever a speculatively rendered frame could be stale: margins, sizes, + * track switches, renderer recreation. + */ + private val stateGeneration: () -> Long, + /** Slot the GL thread most recently took for drawing, or -1. Never write it. */ + private val glTakenSlot: () -> Int, + /** Diagnostic sink for hit/miss/skip decisions; null disables (no string cost). */ + private val debugLog: ((String) -> Unit)? = null +) { + + /** What the caller should do after [service]. */ + sealed class Outcome { + /** Nothing to post (no renderer, or no rendered content exists yet). */ + object Skip : Outcome() + + /** + * Post [slot] to GL. [newContent] means this call wrote fresh content into + * the slot (the caller bumps the payload's content seq). [specHit] means the + * content was pre-rendered — no libass render ran inside the deadline. + */ + class Post(val slot: Int, val frame: AssAtlasFrame, val newContent: Boolean, val specHit: Boolean) : Outcome() + } + + /** A speculative render that wrote new content into [slot] (bump its seq). */ + class SpecWrite(val slot: Int, val frame: AssAtlasFrame) + + // Speculation state: content for specPtsUs is pre-rendered and lives either in + // specSlot, or — when the spec render returned changed == 0 — is identical to + // libassLastSlot's content (specIsLibassLast). + private var specPtsUs = UNSET + private var specSlot = -1 + private var specIsLibassLast = false + private var specGen = 0L + + // The slot holding libass's most recent render output and its frame. + private var libassLastSlot = -1 + private var libassLastFrame: AssAtlasFrame? = null + private var lastPostedSlot = -1 + + // Request-cadence estimator over pinned (playing) requests: median of the last + // 8 PTS deltas, valid after 4, reset on any non-monotonic or > 250 ms jump. + private val deltas = LongArray(DELTA_SAMPLES) + private var deltaCount = 0 + private var deltaIndex = 0 + private var lastPinnedPtsUs = UNSET + + // Stats; single-writer (the libass thread), read from the stats path. + @Volatile + var specHits = 0L + private set + + @Volatile + var specMisses = 0L + private set + + @Volatile + var specSkips = 0L + private set + + /** + * Services a render request for [ptsUs]. [pinned] is false for invalidate + * repaints (paused margin changes etc.), which never feed the cadence + * estimator and never count as speculation misses against playback. + */ + fun service(ptsUs: Long, pinned: Boolean): Outcome { + if (pinned) updateDeltaEstimator(ptsUs) + + if (specPtsUs != UNSET) { + val eps = epsilonUs() + val genNow = stateGeneration() + val hit = genNow == specGen && eps > 0 && abs(ptsUs - specPtsUs) <= eps + val slot = if (specIsLibassLast) libassLastSlot else specSlot + val frame = libassLastFrame + val specPts = specPtsUs + specPtsUs = UNSET + if (hit && slot >= 0 && frame != null) { + specHits++ + lastPostedSlot = slot + debugLog?.invoke("hit pts=${ptsUs / 1000}ms spec=${specPts / 1000}ms d=${(ptsUs - specPts) / 1000}ms slot=$slot") + return Outcome.Post(slot, frame, newContent = false, specHit = true) + } + specMisses++ + debugLog?.invoke( + "miss pts=${ptsUs / 1000}ms spec=${specPts / 1000}ms d=${(ptsUs - specPts) / 1000}ms eps=${eps / 1000}ms " + + "gen=${if (genNow == specGen) "ok" else "CHANGED"} slot=$slot frame=${frame != null}" + ) + } else { + debugLog?.invoke("no-spec pts=${ptsUs / 1000}ms") + } + + // On-demand render. Preferring libassLastSlot as the target makes changed == 0 + // unambiguous: the buffers were untouched and already hold the right content. + val target = renderTargetSlot() ?: return Outcome.Skip + val frame = renderAt(ptsUs / 1000, target) ?: return Outcome.Skip + if (frame.changed == 0) { + val lastSlot = libassLastSlot + val lastFrame = libassLastFrame ?: return Outcome.Skip + if (lastSlot < 0) return Outcome.Skip + lastPostedSlot = lastSlot + return Outcome.Post(lastSlot, lastFrame, newContent = false, specHit = false) + } + libassLastSlot = target + libassLastFrame = frame + lastPostedSlot = target + return Outcome.Post(target, frame, newContent = true, specHit = false) + } + + /** + * Speculatively renders the predicted next request ([servicedPtsUs] + median + * delta) into a free slot. Call after posting the current frame; skipped while + * paused ([pinned] false), when a newer request is already waiting, or while + * the cadence estimator has no confident delta. + */ + fun speculateAfter(servicedPtsUs: Long, pinned: Boolean, hasPending: Boolean): SpecWrite? { + if (!speculationEnabled) return null + if (!pinned || hasPending || !deltaValid()) { + specSkips++ + debugLog?.invoke( + "spec-skip after=${servicedPtsUs / 1000}ms pinned=$pinned pending=$hasPending cadence=${deltaValid()}" + ) + return null + } + val target = renderTargetSlot() ?: run { + specSkips++ + debugLog?.invoke("spec-skip after=${servicedPtsUs / 1000}ms no-free-slot") + return null + } + val gen = stateGeneration() + val specPts = servicedPtsUs + medianDeltaUs() + val frame = renderAt(specPts / 1000, target) ?: run { + specSkips++ + return null + } + specGen = gen + specPtsUs = specPts + if (frame.changed == 0) { + // Content at specPts is identical to libass's last render — nothing was + // written; a hit will repost libassLastSlot (and GL will skip the upload). + specIsLibassLast = true + specSlot = -1 + return null + } + libassLastSlot = target + libassLastFrame = frame + specIsLibassLast = false + specSlot = target + return SpecWrite(target, frame) + } + + /** + * Pre-renders [ptsUs] (an upcoming event's start) purely to warm the + * renderer's glyph/bitmap caches before that content is actually needed — + * heavy typesetting otherwise pays its cache-cold rasterization (measured + * 0.8–2.6 s on weak devices) exactly when the sign appears. The result is + * never posted; like any render it becomes libass's last-rendered content, + * so any pending speculation is invalidated first (its slot/content + * references would go stale). + * + * Returns the slot written (caller bumps its content seq) or null when + * nothing was rendered. + */ + fun prefetch(ptsUs: Long): SpecWrite? { + specPtsUs = UNSET + val target = renderTargetSlot() ?: return null + val frame = renderAt(ptsUs / 1000, target) ?: return null + prefetchCount++ + if (frame.changed == 0) return null + libassLastSlot = target + libassLastFrame = frame + return SpecWrite(target, frame) + } + + /** Cache-warming prefetch renders performed; single-writer, read cross-thread. */ + @Volatile + var prefetchCount = 0L + private set + + private fun renderTargetSlot(): Int? { + // The GL exclusion only exists in ≥3-slot mode; with 2 slots this reduces to + // the legacy "don't write the posted slot" alternation. + val taken = if (slotCount > 2) glTakenSlot() else -1 + val last = libassLastSlot + if (last >= 0 && last != lastPostedSlot && last != taken) return last + for (s in 0 until slotCount) { + if (s != lastPostedSlot && s != taken) return s + } + return null + } + + private fun updateDeltaEstimator(ptsUs: Long) { + val prev = lastPinnedPtsUs + lastPinnedPtsUs = ptsUs + if (prev == UNSET) return + val d = ptsUs - prev + if (d <= 0 || d > MAX_DELTA_US) { + // Seek/discontinuity (or duplicate PTS): forget the cadence. + deltaCount = 0 + deltaIndex = 0 + return + } + deltas[deltaIndex] = d + deltaIndex = (deltaIndex + 1) % DELTA_SAMPLES + if (deltaCount < DELTA_SAMPLES) deltaCount++ + } + + private fun deltaValid() = deltaCount >= MIN_DELTA_SAMPLES + + private fun medianDeltaUs(): Long { + val copy = deltas.copyOfRange(0, deltaCount) + copy.sort() + return copy[deltaCount / 2] + } + + private fun epsilonUs(): Long = if (deltaValid()) minOf(medianDeltaUs() / 2, EPSILON_CAP_US) else 0L + + private companion object { + const val UNSET = Long.MIN_VALUE + const val DELTA_SAMPLES = 8 + const val MIN_DELTA_SAMPLES = 4 + const val MAX_DELTA_US = 250_000L + const val EPSILON_CAP_US = 8_000L + } +} diff --git a/android/libass/src/test/java/com/edde746/plezy/libass/media/widget/SpecRenderEngineTest.kt b/android/libass/src/test/java/com/edde746/plezy/libass/media/widget/SpecRenderEngineTest.kt new file mode 100644 index 00000000..ba4fe872 --- /dev/null +++ b/android/libass/src/test/java/com/edde746/plezy/libass/media/widget/SpecRenderEngineTest.kt @@ -0,0 +1,332 @@ +package com.edde746.plezy.libass.media.widget + +import com.edde746.plezy.libass.AssAtlasFrame +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * Tests for the speculative render-ahead state machine. The render function is + * scripted: each call records (timeMs, slot) and returns the next queued frame. + */ +class SpecRenderEngineTest { + + private class RenderCall(val timeMs: Long, val slot: Int) + + private class Harness( + slotCount: Int = 3, + speculationEnabled: Boolean = true + ) { + val calls = mutableListOf() + val script = ArrayDeque() + var generation = 0L + var glTaken = -1 + + val engine = SpecRenderEngine( + slotCount = slotCount, + speculationEnabled = speculationEnabled, + renderAt = { timeMs, slot -> + calls.add(RenderCall(timeMs, slot)) + if (script.isEmpty()) changed() else script.removeFirst() + }, + stateGeneration = { generation }, + glTakenSlot = { glTaken } + ) + + /** Primes the cadence estimator with [n] steady requests; returns last pts. */ + fun prime(n: Int = 5, startUs: Long = 0L, deltaUs: Long = 42_000L): Long { + var pts = startUs + repeat(n) { + engine.service(pts, pinned = true) + engine.speculateAfter(pts, pinned = true, hasPending = false) + pts += deltaUs + } + return pts - deltaUs + } + } + + private companion object { + const val DELTA = 42_000L + + fun changed(quads: Int = 5) = AssAtlasFrame(2048, 100, quads, 2, 0) + + fun unchanged() = AssAtlasFrame(0, 0, 0, 0, 0) + } + + @Test + fun `steady state hits serve without rendering`() { + val h = Harness() + val last = h.prime() + + // Cadence is confident by now: the last speculateAfter pre-rendered last+Δ. + val before = h.calls.size + val outcome = h.engine.service(last + DELTA, pinned = true) + + assertTrue(outcome is SpecRenderEngine.Outcome.Post) + outcome as SpecRenderEngine.Outcome.Post + assertTrue(outcome.specHit) + assertFalse(outcome.newContent) // content was written (and seq-bumped) at spec time + assertEquals("service must not render on a hit", before, h.calls.size) + assertEquals(1L, h.engine.specHits) + } + + @Test + fun `hit tolerates jitter within epsilon`() { + val h = Harness() + val last = h.prime() + + val outcome = h.engine.service(last + DELTA + 3_000, pinned = true) + + assertTrue((outcome as SpecRenderEngine.Outcome.Post).specHit) + } + + @Test + fun `seek misses and renders on demand into the spec slot`() { + val h = Harness() + val last = h.prime() + val specSlot = h.calls.last().slot // where the speculative content went + + val before = h.calls.size + h.script.add(changed()) + val outcome = h.engine.service(last + 1_000_000, pinned = true) + + assertTrue(outcome is SpecRenderEngine.Outcome.Post) + outcome as SpecRenderEngine.Outcome.Post + assertFalse(outcome.specHit) + assertTrue(outcome.newContent) + assertEquals(before + 1, h.calls.size) + assertEquals("miss must render into the slot holding libass's last content", specSlot, h.calls.last().slot) + assertEquals(1L, h.engine.specMisses) + } + + @Test + fun `miss with changed 0 posts the spec slot not stale screen content`() { + // The trap: spec rendered NEW content (never posted); a mismatched request + // then returns changed == 0 (identical to libass's LAST render = the spec + // content). Posting "last posted" content would be stale — the engine must + // post the slot libass last wrote. + val h = Harness() + val last = h.prime() + val specSlot = h.calls.last().slot + + h.script.add(unchanged()) + val outcome = h.engine.service(last + 200_000, pinned = true) // > ε, miss; estimator resets too + + assertTrue(outcome is SpecRenderEngine.Outcome.Post) + outcome as SpecRenderEngine.Outcome.Post + assertFalse(outcome.specHit) + assertFalse(outcome.newContent) + assertEquals("changed==0 must repost libass's last-rendered slot", specSlot, outcome.slot) + } + + @Test + fun `state generation change invalidates speculation`() { + val h = Harness() + val last = h.prime() + + h.generation++ // margins/zoom/track switch between spec render and the request + h.script.add(changed()) + val before = h.calls.size + val outcome = h.engine.service(last + DELTA, pinned = true) + + assertTrue(outcome is SpecRenderEngine.Outcome.Post) + assertFalse((outcome as SpecRenderEngine.Outcome.Post).specHit) + assertEquals("stale spec must be re-rendered", before + 1, h.calls.size) + assertEquals(1L, h.engine.specMisses) + } + + @Test + fun `unpinned requests never feed the estimator or speculate`() { + val h = Harness() + val last = h.prime() + + // Paused invalidate-repaint at an arbitrary position. + h.script.add(changed()) + h.engine.service(last, pinned = false) + val specBefore = h.calls.size + assertNull(h.engine.speculateAfter(last, pinned = false, hasPending = false)) + assertEquals("unpinned must not speculate", specBefore, h.calls.size) + + // Cadence survives the unpinned request: the next pinned pair still hits. + h.engine.service(last + DELTA, pinned = true) + h.engine.speculateAfter(last + DELTA, pinned = true, hasPending = false) + val outcome = h.engine.service(last + 2 * DELTA, pinned = true) + assertTrue((outcome as SpecRenderEngine.Outcome.Post).specHit) + } + + @Test + fun `speculation skipped while a request is pending`() { + val h = Harness() + val last = h.prime() + val before = h.calls.size + + assertNull(h.engine.speculateAfter(last, pinned = true, hasPending = true)) + assertEquals(before, h.calls.size) + assertTrue(h.engine.specSkips > 0) + } + + @Test + fun `no speculation until cadence is confident`() { + val h = Harness() + h.engine.service(0, pinned = true) + // Only one delta sample so far (needs 4). + h.engine.service(DELTA, pinned = true) + val before = h.calls.size + assertNull(h.engine.speculateAfter(DELTA, pinned = true, hasPending = false)) + assertEquals(before, h.calls.size) + } + + @Test + fun `render target avoids posted and gl taken slots`() { + val h = Harness() + h.script.add(changed()) + val first = h.engine.service(0, pinned = true) as SpecRenderEngine.Outcome.Post + val posted = first.slot + + h.glTaken = (posted + 1) % 3 + h.script.add(changed()) + val second = h.engine.service(1_000_000, pinned = true) as SpecRenderEngine.Outcome.Post + + val expected = (0 until 3).first { it != posted && it != h.glTaken } + assertEquals(expected, second.slot) + } + + @Test + fun `changed 0 before any content skips`() { + val h = Harness() + h.script.add(unchanged()) + assertEquals(SpecRenderEngine.Outcome.Skip, h.engine.service(0, pinned = true)) + } + + @Test + fun `renderer gone skips`() { + val h = Harness() + h.script.add(null) + assertEquals(SpecRenderEngine.Outcome.Skip, h.engine.service(0, pinned = true)) + } + + @Test + fun `two slot mode alternates and never speculates`() { + val h = Harness(slotCount = 2, speculationEnabled = false) + h.script.add(changed()) + val a = h.engine.service(0, pinned = true) as SpecRenderEngine.Outcome.Post + assertNull(h.engine.speculateAfter(0, pinned = true, hasPending = false)) + + h.script.add(changed()) + val b = h.engine.service(DELTA, pinned = true) as SpecRenderEngine.Outcome.Post + assertTrue(a.slot != b.slot) + assertTrue(a.slot in 0..1 && b.slot in 0..1) + assertEquals(0, h.calls.count { it.slot >= 2 }) + } + + @Test + fun `static dialogue hit reposts the same slot without new content`() { + // Spec render returns changed == 0 (nothing moves): a hit must repost the + // last-rendered slot so GL skips the upload entirely. + val h = Harness() + h.script.add(changed()) + val first = h.engine.service(0, pinned = true) as SpecRenderEngine.Outcome.Post + var pts = 0L + repeat(4) { // build cadence; renders return changed for simplicity + pts += DELTA + h.engine.service(pts, pinned = true) + } + h.script.add(unchanged()) // speculative render: nothing changes at pts+Δ + assertNull(h.engine.speculateAfter(pts, pinned = true, hasPending = false)) + + val before = h.calls.size + val outcome = h.engine.service(pts + DELTA, pinned = true) as SpecRenderEngine.Outcome.Post + assertTrue(outcome.specHit) + assertFalse(outcome.newContent) + assertEquals(before, h.calls.size) + assertNotNull(first) // first slot existed; hit reposts whichever slot was last rendered + } + + @Test + fun `speculative write reports slot for seq bump`() { + val h = Harness() + val last = h.prime() + h.script.add(changed()) + val write = h.engine.speculateAfter(last, pinned = true, hasPending = false) + assertNotNull(write) + assertEquals(h.calls.last().slot, write!!.slot) + } + + @Test + fun `spec predicts pts plus median delta`() { + val h = Harness() + val last = h.prime() + h.script.add(changed()) + h.engine.speculateAfter(last, pinned = true, hasPending = false) + assertEquals((last + DELTA) / 1000, h.calls.last().timeMs) + } + + @Test + fun `prefetch renders the requested future pts and reports its slot`() { + val h = Harness() + h.script.add(changed()) + h.engine.service(0, pinned = true) // some content on screen + + h.script.add(changed()) + val write = h.engine.prefetch(5_000_000) + + assertNotNull(write) + assertEquals(5_000L, h.calls.last().timeMs) + assertEquals(h.calls.last().slot, write!!.slot) + assertEquals(1L, h.engine.prefetchCount) + } + + @Test + fun `prefetch invalidates pending speculation`() { + // Prefetch rewrites libass's last-rendered content (and possibly the spec + // slot itself); a stale spec hit would post future-event content. + val h = Harness() + val last = h.prime() // leaves a valid spec for last+Δ + + h.script.add(changed()) + h.engine.prefetch(last + 5_000_000) + + val before = h.calls.size + h.script.add(changed()) + val outcome = h.engine.service(last + DELTA, pinned = true) + + assertTrue(outcome is SpecRenderEngine.Outcome.Post) + assertFalse((outcome as SpecRenderEngine.Outcome.Post).specHit) + assertEquals("post-prefetch request must render on demand", before + 1, h.calls.size) + } + + @Test + fun `render after prefetch with changed 0 posts the prefetched slot`() { + // changed==0 after a prefetch means "identical to libass's last render" = + // the prefetched content — correct exactly when playback reached the + // prefetched event. The engine must post that slot, not older content. + val h = Harness() + h.script.add(changed()) + h.engine.service(0, pinned = true) + + h.script.add(changed()) + val write = h.engine.prefetch(5_000_000)!! + + h.script.add(unchanged()) + val outcome = h.engine.service(5_000_000, pinned = true) as SpecRenderEngine.Outcome.Post + assertEquals(write.slot, outcome.slot) + assertFalse(outcome.newContent) + } + + @Test + fun `prefetch avoids posted and gl taken slots`() { + val h = Harness() + h.script.add(changed()) + val posted = (h.engine.service(0, pinned = true) as SpecRenderEngine.Outcome.Post).slot + h.glTaken = (posted + 1) % 3 + + h.script.add(changed()) + val write = h.engine.prefetch(5_000_000) + + val expected = (0 until 3).first { it != posted && it != h.glTaken } + assertEquals(expected, write!!.slot) + } +} diff --git a/android/settings.gradle.kts b/android/settings.gradle.kts index 43394ed5..48747da0 100644 --- a/android/settings.gradle.kts +++ b/android/settings.gradle.kts @@ -19,7 +19,11 @@ pluginManagement { plugins { id("dev.flutter.flutter-plugin-loader") version "1.0.0" id("com.android.application") version "8.9.1" apply false + id("com.android.library") version "8.9.1" apply false id("org.jetbrains.kotlin.android") version "2.1.0" apply false } include(":app") +// libass subtitle module (Kotlin/JNI bindings + Media3 glue); the native libass +// core stays the upstream Maven artifact io.github.peerless2012:ass. +include(":libass") diff --git a/scripts/gen_framesync_sample.sh b/scripts/gen_framesync_sample.sh new file mode 100755 index 00000000..3e4f3249 --- /dev/null +++ b/scripts/gen_framesync_sample.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Generates MKV test assets that prove (or disprove) frame-perfect ASS rendering: +# the video has a per-frame counter burned in (top) and the MKV carries an ASS +# track flipping the same counter (bottom) on exactly the same frame boundaries. +# Any captured frame showing video number N with subtitle number != N is a sync +# failure. +# +# Outputs (in scripts/framesync/): +# framesync-2397.mkv 23.976 fps, plain per-frame counter events +# framesync-60.mkv 60 fps variant +# framesync-stress.mkv 23.976 fps + heavy animated typesetting (blur/scale/ +# rotation re-rendered every frame) to stress libass +# +# The burn-in uses, in order of preference: ffmpeg drawtext, ffmpeg subtitles +# filter, or mpv's encoding mode (mpv always bundles libass) — all three +# evaluate the counter at each frame's exact PTS, so the burned reference is +# frame-exact by construction. +# +# Verification procedure (Android, ExoPlayer path): +# 1. Play the file with the ASS track selected (SDR content and the +# ASS-tunneling block mean the layers are screen-recordable). +# 2. adb shell screenrecord /sdcard/sync.mp4 (record ~30 s, pull it) +# 3. Step through frames (e.g. ffmpeg -i sync.mp4 frames/%05d.png, or mpv with +# '.' frame-step): top (video) and bottom (player-rendered ASS) counters +# must match on EVERY captured frame. Desktop mpv is the reference player. +# 4. Cross-check at the compositor: while playing, +# adb shell dumpsys SurfaceFlinger --latency +# lists per-layer (desired, actual) present times for the video SurfaceView +# and the ASS overlay layer — matched content must share vsync timestamps. +# 5. Cheap regression: the in-app stats overlay should show subLateSwaps ≈ 0, +# subOverflows == 0, subMinLeadMs ≥ 0 and a >95% subSpecHits ratio. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")" +mkdir -p framesync +cd framesync + +DURATION_S=${DURATION_S:-60} +# drawtext resolves the family via fontconfig; override with FONT if needed. +FONT=${FONT:-Sans} + +have_filter() { ffmpeg -hide_banner -filters 2>/dev/null | grep -q " $1 "; } + +# Emits an ASS file with one Dialogue event per frame. Frame i starts at +# i*den/num seconds; timestamps are floored to ASS centisecond precision, which +# is safe because the floor error (<10 ms) is smaller than any frame interval +# generated here (16.7 / 41.7 ms) — an event can never leak onto the previous +# frame. burn=1 emits the top-aligned white reference style (for burn-in), +# burn=0 the bottom-aligned yellow player style; heavy=1 adds per-frame +# animated blur/scale/rotation events. +gen_ass() { # $1=fps_num $2=fps_den $3=frames $4=burn(0/1) $5=heavy(0/1) $6=outfile + awk -v num="$1" -v den="$2" -v frames="$3" -v burn="$4" -v heavy="$5" ' + function ts(cs, h, m, s) { + h = int(cs / 360000); cs -= h * 360000 + m = int(cs / 6000); cs -= m * 6000 + s = int(cs / 100); cs -= s * 100 + return sprintf("%d:%02d:%02d.%02d", h, m, s, cs) + } + BEGIN { + print "[Script Info]" + print "ScriptType: v4.00+" + print "PlayResX: 1280" + print "PlayResY: 720" + print "" + print "[V4+ Styles]" + print "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding" + if (burn) + print "Style: Counter,Arial,150,&H00FFFFFF,&H000000FF,&H00000000,&H00000000,-1,0,0,0,100,100,0,0,1,5,0,8,10,10,30,1" + else + print "Style: Counter,Arial,150,&H0000FFFF,&H000000FF,&H00000000,&H00000000,-1,0,0,0,100,100,0,0,1,5,0,2,10,10,30,1" + print "Style: Stress,Arial,80,&H40FF8800,&H000000FF,&H00000000,&H00000000,-1,0,0,0,100,100,0,0,1,2,0,5,10,10,10,1" + print "" + print "[Events]" + print "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text" + for (i = 0; i < frames; i++) { + start = int(i * den * 100 / num) + end = int((i + 1) * den * 100 / num) + if (end <= start) end = start + 1 + printf "Dialogue: 0,%s,%s,Counter,,0,0,0,,%d\n", ts(start), ts(end), i + if (heavy) { + # Re-emitted every frame with a frame-dependent rotation/blur so libass + # reports changed content and re-rasterizes large blurred glyphs — + # approximates typesetting storms (the measured 100-400 ms frames). + printf "Dialogue: 1,%s,%s,Stress,,0,0,0,,{\\an5\\pos(640,260)\\blur12\\fscx320\\fscy320\\frz%d}#\n", ts(start), ts(end), (i * 7) % 360 + printf "Dialogue: 1,%s,%s,Stress,,0,0,0,,{\\an5\\pos(640,500)\\blur18\\fscx500\\fscy220\\frz%d\\alpha&H80&}@\n", ts(start), ts(end), 359 - (i * 11) % 360 + } + } + } + ' /dev/null > "$6" +} + +# Produces the reference video: gray frames with the frame counter burned in at +# the top. Tries drawtext, then the subtitles filter, then mpv encoding. +gen_video() { # $1=rate $2=burn_ass $3=outfile + if have_filter drawtext; then + ffmpeg -y -v error -f lavfi -i "color=c=0x202020:s=1280x720:r=$1" \ + -vf "drawtext=font='${FONT}':text='%{n}':fontsize=150:fontcolor=white:borderw=5:bordercolor=black:x=(w-text_w)/2:y=30" \ + -t "$DURATION_S" -c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p "$3" + elif have_filter subtitles; then + ffmpeg -y -v error -f lavfi -i "color=c=0x202020:s=1280x720:r=$1" \ + -vf "subtitles=$2" \ + -t "$DURATION_S" -c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p "$3" + elif command -v mpv > /dev/null; then + ffmpeg -y -v error -f lavfi -i "color=c=0x202020:s=1280x720:r=$1" \ + -t "$DURATION_S" -c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p blank.mp4 + mpv blank.mp4 --sub-files="$2" --vf=sub --no-audio \ + --o="$3" --of=mp4 --ovc=libx264 --ovcopts=preset=veryfast,crf=18 \ + --msg-level=all=error + rm -f blank.mp4 + else + echo "error: need ffmpeg with drawtext or subtitles filter, or mpv (for the burn-in)" >&2 + exit 1 + fi +} + +# Muxes the burned video + silent audio + the player-rendered ASS track. +mux() { # $1=video $2=ass $3=outfile + ffmpeg -y -v error -i "$1" -f lavfi -i "anullsrc=r=48000:cl=stereo" -i "$2" \ + -map 0:v -map 1:a -map 2 -c:v copy -c:a aac -b:a 64k -c:s copy -shortest \ + -metadata:s:s:0 language=eng -disposition:s:0 default "$3" +} + +frames_2397=$(awk -v d="$DURATION_S" 'BEGIN { print int(d * 24000 / 1001) }') +frames_60=$(awk -v d="$DURATION_S" 'BEGIN { print d * 60 }') + +echo "Generating ${DURATION_S}s assets..." +gen_ass 24000 1001 "$frames_2397" 1 0 burn-2397.ass +gen_ass 60 1 "$frames_60" 1 0 burn-60.ass +gen_ass 24000 1001 "$frames_2397" 0 0 counter-2397.ass +gen_ass 60 1 "$frames_60" 0 0 counter-60.ass +gen_ass 24000 1001 "$frames_2397" 0 1 counter-stress.ass +gen_video "24000/1001" burn-2397.ass video-2397.mp4 +gen_video "60" burn-60.ass video-60.mp4 +mux video-2397.mp4 counter-2397.ass framesync-2397.mkv +mux video-60.mp4 counter-60.ass framesync-60.mkv +mux video-2397.mp4 counter-stress.ass framesync-stress.mkv +rm -f video-2397.mp4 video-60.mp4 burn-2397.ass burn-60.ass + +echo "Done:" +ls -la framesync-*.mkv +echo +echo "Reference check: open framesync-2397.mkv in mpv, frame-step with '.' —" +echo "top (video) and bottom (subtitle) counters must match on every frame." +echo "Then repeat in Plezy on-device and screenrecord (see header comments)."