feat(android): frame-accurate libass pipeline + optimized native core

This commit is contained in:
edde746
2026-06-12 01:45:19 +02:00
parent 7ec1c2ebdc
commit 1adb89c6b3
30 changed files with 3663 additions and 51 deletions
+1 -1
View File
@@ -70,4 +70,4 @@ duplication-report/
**/fastlane/report.xml
maestro/**/*.png
server/deploy.sh
server/deploy.shscripts/framesync/
+8 -22
View File
@@ -23,24 +23,6 @@ val downloadLibmpv by tasks.registering {
}
}
val assVersion = "fp-3"
val assDir = layout.buildDirectory.dir("libass").get().asFile
val assAars = listOf("lib_ass-release.aar", "lib_ass_kt-release.aar", "lib_ass_media-release.aar")
val downloadLibass by tasks.registering {
val stamp = File(assDir, ".version")
outputs.upToDateWhen { stamp.exists() && stamp.readText().trim() == assVersion }
doLast {
assDir.mkdirs()
val baseUrl = "https://github.com/edde746/libass-android/releases/download/$assVersion"
assAars.forEach { name ->
val dest = File(assDir, name)
exec { commandLine("curl", "-sfL", "$baseUrl/$name", "-o", dest.absolutePath) }
}
stamp.writeText(assVersion)
}
}
val doviVersion = "2.3.1"
val doviDir = layout.buildDirectory.dir("libdovi").get().asFile
val doviAbis = mapOf(
@@ -160,10 +142,9 @@ tasks.matching { it.name.contains("CMake") || it.name.contains("externalNative")
dependsOn(downloadLibdovi)
}
// Download libmpv and libass AARs before compilation
// Download the libmpv AAR before compilation
tasks.matching { it.name.startsWith("pre") && it.name.endsWith("Build") }.configureEach {
dependsOn(downloadLibmpv)
dependsOn(downloadLibass)
}
dependencies {
@@ -186,8 +167,13 @@ dependencies {
// FFmpeg audio decoder for unsupported codecs (ALAC, DTS, TrueHD, etc.)
implementation("org.jellyfin.media3:media3-ffmpeg-decoder:1.9.0+1")
// libass-android for ASS/SSA subtitle rendering
assAars.forEach { implementation(files(File(assDir, it))) }
// libass ASS/SSA subtitle rendering: optimized native core (libass.so +
// prefab headers) from the edde746/libass-android fork's releases; Kotlin/JNI
// bindings + Media3 glue live in the android/libass module. -PlocalAssCore
// swaps in a mavenLocal()-published core (0.4.0-local) for native A/B tests.
val assCoreVersion = if (project.hasProperty("localAssCore")) "0.4.0-local" else "0.4.1-plezy.1"
implementation("io.github.peerless2012:ass:$assCoreVersion@aar")
implementation(project(":libass"))
testImplementation("junit:junit:4.13.2")
}
@@ -59,14 +59,13 @@ import androidx.media3.extractor.ts.TsExtractor
import androidx.media3.ui.AspectRatioFrameLayout
import androidx.media3.ui.CaptionStyleCompat
import androidx.media3.ui.SubtitleView
import com.edde746.plezy.libass.media.AssHandler
import com.edde746.plezy.libass.media.parser.AssSubtitleParserFactory
import com.edde746.plezy.libass.media.widget.AssSubtitleSurfaceView
import com.edde746.plezy.shared.AudioFocusManager
import com.edde746.plezy.shared.DeviceQuirks
import com.edde746.plezy.shared.FlutterOverlayHelper
import com.edde746.plezy.shared.FrameRateManager
import io.github.peerless2012.ass.media.AssHandler
import io.github.peerless2012.ass.media.parser.AssSubtitleParserFactory
import io.github.peerless2012.ass.media.type.AssRenderType
import io.github.peerless2012.ass.media.widget.AssSubtitleSurfaceView
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import org.chromium.net.CronetEngine
@@ -97,6 +96,10 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
private const val DECODER_HANG_TIMEOUT_MS = 5000L
private const val MAX_AUDIO_RECOVERY_ATTEMPTS = 2
private const val FPS_SAMPLE_COUNT = 8
/** Per-frame "video is at X" logcat stream (tag AssFrameCb) for diagnosing
* ASS subtitle lag against the libass pipeline's render/swap lines. */
private const val ASS_FRAME_LOGS = false
private const val TS_TIMESTAMP_SEARCH_PACKETS = 1800
private val DV_CODEC_PROFILE_REGEX = Regex("""(?:^|,)\s*dvh[1e]\.(\d{2})""")
@@ -123,6 +126,9 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
private var subtitleView: SubtitleView? = null
private var videoZoomScale: Float = 1.0f
private var assHandler: AssHandler? = null
private var assSubtitleView: AssSubtitleSurfaceView? = null
private var assForceMargins = false
private var lastAssMargins: IntArray? = null
private var overlayLayoutListener: ViewTreeObserver.OnGlobalLayoutListener? = null
private var lastVideoSize: VideoSize? = null
private var exoPlayer: ExoPlayer? = null
@@ -139,6 +145,11 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
private var tunnelingDisabledForVideoCodec: Boolean = false
private var tunnelingDisabledForDecodedTrueHdPcm: Boolean = false
private var tunnelingDisabledForAudioRecovery: Boolean = false
// Tunneled playback never fires the VideoFrameMetadataListener (media3 releases
// frames inside the codec), which is the libass pipeline's only render trigger —
// ASS subs would freeze. Correctness over tunneling while an ASS track is active.
private var tunnelingDisabledForAssSubtitles: Boolean = false
private val tunnelingDisabledForCodec: Boolean
get() = tunnelingDisabledForAudioCodec || tunnelingDisabledForVideoCodec || tunnelingDisabledForDecodedTrueHdPcm || tunnelingDisabledForAudioRecovery
private var currentTunneledPlayback: Boolean = false
@@ -478,8 +489,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
.setTsExtractorTimestampSearchBytes(TS_TIMESTAMP_SEARCH_PACKETS * TsExtractor.TS_PACKET_SIZE)
// Inline buildWithAssSupport to retain AssHandler reference for font scale control.
val renderType = AssRenderType.OVERLAY_OPEN_GL
val handler = AssHandler(renderType)
val handler = AssHandler()
assHandler = handler
val assParserFactory = AssSubtitleParserFactory(handler)
@@ -567,26 +577,28 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
.setRenderersFactory(wrappedRenderersFactory)
.build()
// Add ASS overlay view inside videoAspectContainer, sibling of the video
// SurfaceView, so it inherits the video's resize-mode measurement, zoom
// transform, and crop: the libass frame (= surface size) maps 1:1 onto the
// displayed video rect at any fit mode or zoom level. Parenting it outside
// the container desyncs ASS positioning whenever the video rect exceeds the
// screen (cover mode, zoom > 100%).
// We use AssSubtitleSurfaceView directly (not AssSubtitleView) so we get a
// SurfaceFlinger-layer-backed overlay that eglPresentationTimeANDROID can
// vsync-pin. Z-order: video SurfaceView (-2) < this MediaOverlay-flagged
// Add ASS overlay view to the full-screen surfaceContainer (NOT the zoom-scaled
// videoAspectContainer): the libass frame = screen, and mpv-style ass_set_margins
// describe where the video dst rect sits inside it (negative when zoomed past the
// edges) — see updateAssMargins(). Non-positioned dialogue can then be forced
// on-screen (sub-ass-force-margins) while positioned/typeset events stay glued to
// the video rect, matching mpv. Also keeps the subtitle surface unscaled (crisp
// text, no per-gesture geometry churn).
// AssSubtitleSurfaceView gives us a SurfaceFlinger-layer-backed overlay that
// eglPresentationTimeANDROID can vsync-pin to the video frame.
// Z-order: video SurfaceView (-2) < this MediaOverlay-flagged
// SurfaceView (-1) < parent canvas < Flutter SurfaceView (+1) in the window.
// Both punches run while this subtree draws, BEFORE the later subtitleView
// sibling renders non-ASS cues on the parent canvas.
var assSubtitleSurfaceView: AssSubtitleSurfaceView? = null
videoAspectContainer?.let { container ->
// Inserted before subtitleView so both punches run before SRT/VTT cues draw
// on the parent canvas.
surfaceContainer?.let { container ->
val assView = AssSubtitleSurfaceView(container.context, handler)
assSubtitleSurfaceView = assView
assSubtitleView = assView
// Pre-36 sublayer is already set by the view's own setZOrderMediaOverlay(true).
FlutterOverlayHelper.applyCompositionOrder(assView, -1)
val subtitleIndex = container.indexOfChild(subtitleView)
container.addView(
assView,
if (subtitleIndex >= 0) subtitleIndex else container.childCount,
FrameLayout.LayoutParams(
FrameLayout.LayoutParams.MATCH_PARENT,
FrameLayout.LayoutParams.MATCH_PARENT
@@ -623,7 +635,19 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
exoPlayer!!.addAnalyticsListener(decoderHangListener)
exoPlayer!!.setVideoFrameMetadataListener { presentationTimeUs, releaseTimeNs, _, _ ->
// ASS bypasses Media3's text renderer, so apply sub-delay before libass renders.
assSubtitleSurfaceView?.requestRender(presentationTimeUs - subtitleDelayUs.get(), releaseTimeNs)
assSubtitleView?.requestRender(presentationTimeUs - subtitleDelayUs.get(), releaseTimeNs)
if (ASS_FRAME_LOGS) {
// Reference stream for subtitle-lag diagnosis: the video frame ExoPlayer
// is releasing right now and how far ahead of its vsync we are. Subtitle
// "render pts=" lines lagging these pts values = pipeline behind;
// budgetMs far from ~10-50 = release-time clock-domain trouble.
val budgetMs = (releaseTimeNs - System.nanoTime()) / 1_000_000
Log.d(
"AssFrameCb",
"video pts=${presentationTimeUs / 1000}ms budgetMs=$budgetMs" +
(subtitleDelayUs.get().takeIf { it != 0L }?.let { " subDelayMs=${it / 1000}" } ?: "")
)
}
val count = fpsTimestampCount
if (count < FPS_SAMPLE_COUNT) {
fpsTimestamps[count] = presentationTimeUs
@@ -737,7 +761,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
// Player.Listener
override fun onCues(cueGroup: CueGroup) {
// With OVERLAY_CANVAS mode, ASS subtitles are rendered directly by AssSubtitleView
// ASS subtitles are rendered by the libass overlay surface.
// This callback is for non-ASS subtitles (SRT, VTT, etc.)
val incoming = cueGroup.cues
lastSubtitleCues = incoming
@@ -919,6 +943,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
evaluateAudioCodecForTunneling()
evaluateVideoCodecForTunneling()
evaluateAssSubtitlesForTunneling(tracks)
updateTunnelingState("tracks changed")
emitTrackList()
}
@@ -1231,6 +1256,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
videoAspectContainer?.setAspectRatio(videoAspect)
}
updateSubtitleViewSize(videoWidth, videoHeight, pixelRatio)
updateAssMargins()
}
private fun updateSubtitleViewSize(videoWidth: Int, videoHeight: Int, pixelRatio: Float) {
@@ -1243,8 +1269,8 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
val containerHeight = contentView.height
if (containerWidth == 0 || containerHeight == 0) return
// Sizes the non-ASS SubtitleView only (the ASS overlay lives inside
// videoAspectContainer and tracks the video rect automatically).
// Sizes the non-ASS SubtitleView only (the ASS overlay is screen-sized and
// tracks the video rect via libass margins — see updateAssMargins()).
// In cover/stretch/zoomed-in modes text cues stay at container size so they
// never get cropped. In letterbox mode they follow the visible video rect.
val isLetterbox = videoAspectContainer?.resizeMode == AspectRatioFrameLayout.RESIZE_MODE_FIT
@@ -1287,6 +1313,83 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
}
}
// Pushes mpv-style libass margins: the offsets of the video dst rect within the
// full-screen container (= the libass frame). Negative when the video extends past
// the screen (cover mode, zoom > 1) — libass supports that explicitly. Pure math +
// a native setter (no view mutation), so it is safe to run per layout pass and per
// pinch-zoom tick (#1261 no-churn invariant).
private fun updateAssMargins() {
if (disposing) return
val handler = assHandler ?: return
val vs = lastVideoSize ?: return
if (vs.width == 0 || vs.height == 0) return
activity.runOnUiThread {
if (disposing) return@runOnUiThread
val containerWidth = surfaceContainer?.width ?: 0
val containerHeight = surfaceContainer?.height ?: 0
if (containerWidth == 0 || containerHeight == 0) return@runOnUiThread
val videoAspect = (vs.width * vs.pixelWidthHeightRatio) / vs.height
val containerAspect = containerWidth.toFloat() / containerHeight
val resizeMode = videoAspectContainer?.resizeMode ?: AspectRatioFrameLayout.RESIZE_MODE_FIT
// Mirror AspectRatioFrameLayout.onMeasure: aspect mismatches <= 1% are
// absorbed by stretching to the container instead of resizing.
val (baseWidth, baseHeight) = if (kotlin.math.abs(videoAspect / containerAspect - 1f) <= 0.01f) {
containerWidth.toFloat() to containerHeight.toFloat()
} else {
when (resizeMode) {
// Cover: scale up to fill the container, cropping the overflow
AspectRatioFrameLayout.RESIZE_MODE_ZOOM ->
if (videoAspect > containerAspect) {
containerHeight * videoAspect to containerHeight.toFloat()
} else {
containerWidth.toFloat() to containerWidth / videoAspect
}
// Stretch: video fills the container, aspect overridden
AspectRatioFrameLayout.RESIZE_MODE_FILL ->
containerWidth.toFloat() to containerHeight.toFloat()
// Fit: letterbox within the container
else ->
if (videoAspect > containerAspect) {
containerWidth.toFloat() to containerWidth / videoAspect
} else {
containerHeight * videoAspect to containerHeight.toFloat()
}
}
}
// videoAspectContainer is centered and zoom-scaled about its center.
val videoWidth = Math.round(baseWidth * videoZoomScale)
val videoHeight = Math.round(baseHeight * videoZoomScale)
val left = (containerWidth - videoWidth) / 2
val top = (containerHeight - videoHeight) / 2
val right = containerWidth - videoWidth - left
val bottom = containerHeight - videoHeight - top
val margins = intArrayOf(top, bottom, left, right)
if (lastAssMargins?.contentEquals(margins) == true) return@runOnUiThread
lastAssMargins = margins
handler.setMargins(top, bottom, left, right)
// Repaint at the current position so changes are visible while paused; during
// playback the next video frame's render supersedes it (latest-wins).
assSubtitleView?.invalidateSubtitles()
}
}
// mpv's sub-ass-force-margins, live-applied from the Dart-managed property: lay out
// non-positioned ASS events against the visible screen instead of the video rect.
fun setAssForceMargins(force: Boolean) {
if (disposing) return
activity.runOnUiThread {
if (disposing || assForceMargins == force) return@runOnUiThread
assForceMargins = force
assHandler?.setUseMargins(force)
assSubtitleView?.invalidateSubtitles()
}
}
private fun boxFitModeToResizeMode(mode: Int): Int = when (mode) {
1 -> AspectRatioFrameLayout.RESIZE_MODE_ZOOM
2 -> AspectRatioFrameLayout.RESIZE_MODE_FILL
@@ -1305,6 +1408,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
updateSubtitleViewSize(vs.width, vs.height, vs.pixelWidthHeightRatio)
}
}
updateAssMargins()
}
}
@@ -1321,6 +1425,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
updateSubtitleViewSize(vs.width, vs.height, vs.pixelWidthHeightRatio)
}
}
updateAssMargins()
}
}
@@ -1974,6 +2079,31 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
applyTrackSelectorPolicy(reason = reason, forceSelector = forceSelector)
}
/** True when [format] is an ASS/SSA subtitle track (rendered by libass). */
private fun isAssSubtitleFormat(format: Format): Boolean =
format.sampleMimeType == MimeTypes.TEXT_SSA || format.codecs == MimeTypes.TEXT_SSA
/** Sets the ASS-subtitles tunneling block; returns true when the flag changed. */
private fun updateAssSubtitlesForTunneling(assActive: Boolean): Boolean {
if (assActive == tunnelingDisabledForAssSubtitles) return false
tunnelingDisabledForAssSubtitles = assActive
emitLog(
"info",
"tunneling",
if (assActive) "ASS subtitle track selected: tunneling DISABLED (frame metadata required for libass)"
else "ASS subtitle track deselected: tunneling unblocked"
)
return true
}
private fun evaluateAssSubtitlesForTunneling(tracks: Tracks) {
val assSelected = tracks.groups.any { group ->
group.type == C.TRACK_TYPE_TEXT && group.isSelected &&
(0 until group.length).any { isAssSubtitleFormat(group.getTrackFormat(it)) }
}
updateAssSubtitlesForTunneling(assSelected)
}
private fun applyTrackSelectorPolicy(
reason: String,
forceSelector: Boolean = false,
@@ -2022,7 +2152,8 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
private fun calculateTunnelingEnabled(): Boolean? {
val player = exoPlayer ?: return null
val audioDelayActive = (renderersFactory?.audioDelayUs?.get() ?: 0L) != 0L
return tunnelingUserEnabled && (player.playbackParameters.speed == 1f) && !tunnelingDisabledForCodec && !audioDelayActive
return tunnelingUserEnabled && (player.playbackParameters.speed == 1f) && !tunnelingDisabledForCodec &&
!tunnelingDisabledForAssSubtitles && !audioDelayActive
}
private fun updateCurrentTunnelingState(reason: String, shouldTunnel: Boolean): Boolean {
@@ -2030,7 +2161,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
currentTunneledPlayback = shouldTunnel
val speed = exoPlayer?.playbackParameters?.speed ?: 1f
val audioDelayActive = (renderersFactory?.audioDelayUs?.get() ?: 0L) != 0L
emitLog("info", "tunneling", "Toggling tunneling=$shouldTunnel (reason=$reason, user=$tunnelingUserEnabled, speed=$speed, audioCodecDisabled=$tunnelingDisabledForAudioCodec, videoCodecDisabled=$tunnelingDisabledForVideoCodec, decodedTrueHdPcmDisabled=$tunnelingDisabledForDecodedTrueHdPcm, audioRecoveryDisabled=$tunnelingDisabledForAudioRecovery, audioDelay=$audioDelayActive)")
emitLog("info", "tunneling", "Toggling tunneling=$shouldTunnel (reason=$reason, user=$tunnelingUserEnabled, speed=$speed, audioCodecDisabled=$tunnelingDisabledForAudioCodec, videoCodecDisabled=$tunnelingDisabledForVideoCodec, decodedTrueHdPcmDisabled=$tunnelingDisabledForDecodedTrueHdPcm, audioRecoveryDisabled=$tunnelingDisabledForAudioRecovery, assSubtitlesDisabled=$tunnelingDisabledForAssSubtitles, audioDelay=$audioDelayActive)")
return true
}
@@ -2423,6 +2554,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
tunnelingDisabledForVideoCodec = false
tunnelingDisabledForDecodedTrueHdPcm = false
tunnelingDisabledForAudioRecovery = false
tunnelingDisabledForAssSubtitles = false
currentTunneledPlayback = false
pendingStartPositionMs = startPositionMs
pendingPlayWhenReady = autoPlay
@@ -2605,6 +2737,9 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
fun selectSubtitleTrack(trackId: String?) {
if (trackId == null || trackId == "no") {
selectedSubtitleTrackId = "no"
// Flip the tunneling block in the same parameters update as the text
// disable so the renderer re-initializes once, not twice.
updateAssSubtitlesForTunneling(false)
applyTrackSelectorPolicy(
reason = "subtitle disabled",
textDisabled = true
@@ -2615,6 +2750,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
val trackGroup = subtitleTrackGroupMap[trackId] ?: return
selectedSubtitleTrackId = trackId
updateAssSubtitlesForTunneling(isAssSubtitleFormat(trackGroup.getFormat(0)))
applyTrackSelectorPolicy(
reason = "subtitle track selected",
textOverride = TrackSelectionOverride(trackGroup, 0),
@@ -2898,6 +3034,26 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
"videoDecoderName" to videoDecoderName,
"videoDroppedFrames" to player.videoDecoderCounters?.droppedBufferCount,
"videoRenderedFrames" to player.videoDecoderCounters?.renderedOutputBufferCount,
// ASS overlay swap timing (vsync-pinned; late = past the swap-time budget)
"subSwapCount" to assSubtitleView?.swapCount,
"subLateSwaps" to assSubtitleView?.lateSwapCount,
"subMaxLateMs" to assSubtitleView?.maxLateMs,
// ASS libass render cost (changed renders rewrite the atlas; histogram
// buckets: ≤10/≤25/≤42/≤84/>84 ms)
"subRenderCount" to assSubtitleView?.renderCount,
"subChangedRenders" to assSubtitleView?.changedRenderCount,
"subOverflows" to assSubtitleView?.overflowCount,
"subLibassLastMs" to assSubtitleView?.lastLibassMs,
"subLibassMaxMs" to assSubtitleView?.maxLibassMs,
"subLibassHist" to assSubtitleView?.libassMsHistogram,
// ASS render-ahead: hits = served from a pre-rendered frame (GL-only path);
// minLead ≥ 0 means changed content reached the queue before the video
// frame's vsync — the frame-perfection signal.
"subSpecHits" to assSubtitleView?.specHits,
"subSpecMisses" to assSubtitleView?.specMisses,
"subSpecSkips" to assSubtitleView?.specSkips,
"subPrefetches" to assSubtitleView?.prefetchCount,
"subMinLeadMs" to assSubtitleView?.minLeadChangedMs,
// Color info
"colorSpace" to videoFormat?.colorInfo?.colorSpace,
"colorRange" to videoFormat?.colorInfo?.colorRange,
@@ -2992,6 +3148,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
if (tunnelingDisabledForDecodedTrueHdPcm) return "Off (decoded TrueHD PCM)"
if (tunnelingDisabledForVideoCodec) return "Off (video codec unsupported)"
if (tunnelingDisabledForAudioCodec) return "Off (no HW audio decoder)"
if (tunnelingDisabledForAssSubtitles) return "Off (ASS subtitles active)"
return "Off"
}
@@ -3036,6 +3193,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
tunnelingDisabledForVideoCodec = false
tunnelingDisabledForDecodedTrueHdPcm = false
tunnelingDisabledForAudioRecovery = false
tunnelingDisabledForAssSubtitles = false
currentTunneledPlayback = false
pendingStartPositionMs = 0L
pendingPlayWhenReady = null
@@ -3070,6 +3228,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
videoAspectContainer = null
surfaceView = null
subtitleView = null
assSubtitleView = null
// Remove layout listener synchronously
overlayLayoutListener?.let { listener ->
@@ -622,6 +622,9 @@ class ExoPlayerPlugin :
when (name) {
"audio-delay" -> playerCore?.setAudioDelay(value.toDoubleOrNull() ?: 0.0)
"sub-delay" -> playerCore?.setSubtitleDelay(value.toDoubleOrNull() ?: 0.0)
// mpv semantics mirrored on the libass overlay: anchor non-positioned ASS
// events to the visible screen (Dart sets 'yes' for cover mode / zoom > 1)
"sub-ass-force-margins" -> playerCore?.setAssForceMargins(value == "yes")
}
}
@@ -7,8 +7,8 @@ import androidx.media3.extractor.SeekMap
import androidx.media3.extractor.TrackOutput
import androidx.media3.extractor.mkv.MatroskaExtractor
import androidx.media3.extractor.text.SubtitleParser
import io.github.peerless2012.ass.media.AssHandler
import io.github.peerless2012.ass.media.extractor.AssMatroskaExtractor
import com.edde746.plezy.libass.media.AssHandler
import com.edde746.plezy.libass.media.extractor.AssMatroskaExtractor
/**
* Extends AssMatroskaExtractor to add support for MKV ContentCompAlgo 0 (zlib).
+22
View File
@@ -2,6 +2,28 @@ allprojects {
repositories {
google()
mavenCentral()
if (providers.gradleProperty("localAssCore").isPresent) {
// Opt-in A/B of a locally built libass native core: build it in the
// libass-android fork with `./gradlew :lib_ass:publishToMavenLocal
// -PVERSION_NAME=0.4.0-local`, then build this app with -PlocalAssCore.
mavenLocal()
} else {
// Production libass native core: the -O3/NEON/asm AAR published on
// the edde746/libass-android fork's releases (the upstream Maven
// artifact io.github.peerless2012:ass ships un-optimized natives —
// see the fork's pinned libass-cmake fix). Resolved as
// <tag>/ass-<tag>.aar with no metadata probing.
exclusiveContent {
forRepository {
ivy {
url = uri("https://github.com/edde746/libass-android/releases/download")
patternLayout { artifact("[revision]/[artifact]-[revision].[ext]") }
metadataSources { artifact() }
}
}
filter { includeModule("io.github.peerless2012", "ass") }
}
}
}
}
+77
View File
@@ -0,0 +1,77 @@
// libass ASS subtitle rendering: Kotlin/JNI bindings + Media3 integration
// (extractor, parsers, AssHandler, GL atlas overlay). The native libass core
// (libass.so + prefab headers) comes from the Maven artifact
// io.github.peerless2012:ass; this module compiles its JNI against it.
plugins {
id("com.android.library")
id("org.jetbrains.kotlin.android")
}
android {
namespace = "com.edde746.plezy.libass"
compileSdk = 36
ndkVersion = "28.2.13676358" // matches flutter.ndkVersion so only one NDK is provisioned
defaultConfig {
minSdk = 21
consumerProguardFiles("consumer-rules.pro")
if (project.hasProperty("localAssCore")) {
// The locally published A/B core only ships device ABIs (x86 would need
// nasm on the host); match it so prefab resolution doesn't fail.
ndk {
abiFilters += listOf("armeabi-v7a", "arm64-v8a")
}
}
externalNativeBuild {
cmake {
// libass.so in the prefab AAR is built against c++_shared (abi.json: stl=c++_shared);
// prefab validates consumer STL compatibility.
arguments += listOf("-DANDROID_STL=c++_shared")
}
}
}
buildFeatures {
prefab = true
}
packaging {
jniLibs {
// libass.so is a prefab IMPORTED target (linked, not owned) — the app packages
// it from the io.github.peerless2012:ass AAR; don't duplicate it here.
excludes.add("**/libass.so")
}
}
compileOptions {
sourceCompatibility = JavaVersion.VERSION_11
targetCompatibility = JavaVersion.VERSION_11
}
kotlinOptions {
jvmTarget = JavaVersion.VERSION_11.toString()
}
externalNativeBuild {
cmake {
path = file("src/main/cpp/CMakeLists.txt")
version = "3.22.1"
}
}
}
dependencies {
// compileOnly: prefab headers + link-time libass.so come from the AAR; runtime
// packaging of libass.so is the app's implementation dependency.
// -PlocalAssCore swaps in a mavenLocal()-published core for A/B tests (must
// match the app module's version so one libass.so is linked and packaged).
val assCoreVersion = if (project.hasProperty("localAssCore")) "0.4.0-local" else "0.4.1-plezy.1"
compileOnly("io.github.peerless2012:ass:$assCoreVersion@aar")
implementation("androidx.annotation:annotation:1.9.1")
implementation("androidx.annotation:annotation-experimental:1.5.1")
implementation("androidx.media3:media3-exoplayer:1.9.2")
implementation("androidx.media3:media3-ui:1.9.2")
testImplementation("junit:junit:4.13.2")
}
+6
View File
@@ -0,0 +1,6 @@
# Constructed from JNI via FindClass/NewObject (AssKt.c).
-keep class com.edde746.plezy.libass.AssAtlasFrame { *; }
# JNI exports bind by name (Java_com_edde746_plezy_libass_*); keep the names stable.
-keepclasseswithmembernames class com.edde746.plezy.libass.* {
native <methods>;
}
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
</manifest>
+436
View File
@@ -0,0 +1,436 @@
// JNI bindings for libass. Exports use standard Java_<package>_<Class>_<method>
// naming so no RegisterNatives/JNI_OnLoad registration is needed.
#include <android/log.h>
#include <jni.h>
#include <stdarg.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
static inline long long nowMs(void) {
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (long long)ts.tv_sec * 1000 + ts.tv_nsec / 1000000;
}
#include "ass/ass.h"
#define LOG_TAG "SubtitleRenderer"
static void assMessageCallback(int level, const char* fmt, va_list args, void* data) {
if (level > 4) return;
if (level >= 2) {
__android_log_vprint(ANDROID_LOG_WARN, LOG_TAG, fmt, args);
} else {
__android_log_vprint(ANDROID_LOG_ERROR, LOG_TAG, fmt, args);
}
}
// --- Ass (library) ---
JNIEXPORT jlong JNICALL Java_com_edde746_plezy_libass_Ass_nativeAssInit(JNIEnv* env, jclass clazz) {
ASS_Library* assLibrary = ass_library_init();
ass_set_message_cb(assLibrary, assMessageCallback, NULL);
ass_set_extract_fonts(assLibrary, 1);
return (jlong)assLibrary;
}
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_Ass_nativeAssAddFont(
JNIEnv* env, jclass clazz, jlong ass, jstring name, jbyteArray byteArray) {
jsize length = (*env)->GetArrayLength(env, byteArray);
jbyte* bytePtr = (*env)->GetByteArrayElements(env, byteArray, NULL);
if (bytePtr == NULL) {
return;
}
const char* cName = (*env)->GetStringUTFChars(env, name, NULL);
ass_add_font(((ASS_Library*)ass), cName, (char*)bytePtr, length);
(*env)->ReleaseByteArrayElements(env, byteArray, bytePtr, 0);
if (cName != NULL) {
(*env)->ReleaseStringUTFChars(env, name, cName);
}
}
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_Ass_nativeAssDeinit(JNIEnv* env, jclass clazz, jlong ass) {
if (ass) {
ass_library_done((ASS_Library*)ass);
}
}
// --- AssTrack ---
JNIEXPORT jlong JNICALL
Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackInit(JNIEnv* env, jclass clazz, jlong ass) {
return (jlong)ass_new_track((ASS_Library*)ass);
}
// Shared body of readBuffer/readChunk: pins the byte array and feeds libass.
// chunked != 0 routes to ass_process_chunk (timed dialogue), else ass_process_data.
static void processTrackBytes(
JNIEnv* env, jlong track, jbyteArray buffer, jint offset, jint length, jlong start, jlong duration, int chunked) {
if (!track) return;
jbyte* elements = (*env)->GetByteArrayElements(env, buffer, NULL);
if (elements == NULL) {
return;
}
if (chunked) {
ass_process_chunk((ASS_Track*)track, (char*)(elements + offset), length, start, duration);
} else {
ass_process_data((ASS_Track*)track, (char*)(elements + offset), length);
}
(*env)->ReleaseByteArrayElements(env, buffer, elements, 0);
}
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackReadBuffer(
JNIEnv* env, jclass clazz, jlong track, jbyteArray buffer, jint offset, jint length) {
processTrackBytes(env, track, buffer, offset, length, 0, 0, 0);
}
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackReadChunk(
JNIEnv* env, jclass clazz, jlong track, jlong start, jlong duration, jbyteArray buffer, jint offset, jint length) {
processTrackBytes(env, track, buffer, offset, length, start, duration, 1);
}
JNIEXPORT void JNICALL
Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackDeinit(JNIEnv* env, jclass clazz, jlong track) {
if (!track) return;
ass_free_track((ASS_Track*)track);
}
// Earliest event Start strictly after afterMs, or -1. Lets the render pipeline
// pre-render (cache-warm) the next upcoming event during idle stretches so
// heavy typesetting doesn't pay its cache-cold rasterization at appearance.
JNIEXPORT jlong JNICALL Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackNextEventStart(
JNIEnv* env, jclass clazz, jlong track, jlong afterMs) {
if (!track) return -1;
ASS_Track* t = (ASS_Track*)track;
long long best = -1;
for (int i = 0; i < t->n_events; i++) {
const long long start = t->events[i].Start;
if (start > afterMs && (best < 0 || start < best)) best = start;
}
return (jlong)best;
}
// Earliest visible-content boundary (event Start OR End) strictly after afterMs,
// or -1. A cache-warming prefetch is only invisible while no boundary passes:
// the render pipeline uses this to ensure nothing on screen is due to change
// before the event it is about to warm.
JNIEXPORT jlong JNICALL Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackNextEventChange(
JNIEnv* env, jclass clazz, jlong track, jlong afterMs) {
if (!track) return -1;
ASS_Track* t = (ASS_Track*)track;
long long best = -1;
for (int i = 0; i < t->n_events; i++) {
const long long start = t->events[i].Start;
const long long end = start + t->events[i].Duration;
if (start > afterMs && (best < 0 || start < best)) best = start;
if (end > afterMs && (best < 0 || end < best)) best = end;
}
return (jlong)best;
}
// --- AssRender ---
JNIEXPORT jlong JNICALL
Java_com_edde746_plezy_libass_AssRender_nativeAssRenderInit(JNIEnv* env, jclass clazz, jlong ass) {
ASS_Renderer* assRenderer = ass_renderer_init((ASS_Library*)ass);
ass_set_fonts(assRenderer, NULL, "sans-serif", ASS_FONTPROVIDER_FONTCONFIG, NULL, 1);
return (jlong)assRenderer;
}
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderSetFontScale(
JNIEnv* env, jclass clazz, jlong render, jfloat scale) {
if (!render) return;
ass_set_font_scale((ASS_Renderer*)render, scale);
}
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderSetCacheLimit(
JNIEnv* env, jclass clazz, jlong render, jint glyphMax, jint bitmapMaxSize) {
if (!render) return;
ass_set_cache_limits((ASS_Renderer*)render, glyphMax, bitmapMaxSize);
}
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderSetFrameSize(
JNIEnv* env, jclass clazz, jlong render, jint width, jint height) {
if (!render) return;
ass_set_frame_size((ASS_Renderer*)render, width, height);
}
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderSetStorageSize(
JNIEnv* env, jclass clazz, jlong render, jint width, jint height) {
if (!render) return;
ass_set_storage_size((ASS_Renderer*)render, width, height);
}
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderSetMargins(
JNIEnv* env, jclass clazz, jlong render, jint top, jint bottom, jint left, jint right) {
if (!render) return;
ass_set_margins((ASS_Renderer*)render, top, bottom, left, right);
}
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderSetUseMargins(
JNIEnv* env, jclass clazz, jlong render, jboolean use) {
if (!render) return;
ass_set_use_margins((ASS_Renderer*)render, use ? 1 : 0);
}
JNIEXPORT void JNICALL
Java_com_edde746_plezy_libass_AssRender_nativeAssRenderDeinit(JNIEnv* env, jclass clazz, jlong render) {
if (render) {
ass_renderer_done((ASS_Renderer*)render);
}
}
// (image, original-list index) pair so packing can run in height-sorted order
// while slots stay keyed by list position (= blend order) for pass 2.
typedef struct {
ASS_Image* img;
int idx;
} PackItem;
static int comparePackItemsByHeightDesc(const void* a, const void* b) {
const PackItem* ia = (const PackItem*)a;
const PackItem* ib = (const PackItem*)b;
return ib->img->h - ia->img->h;
}
// Throttle for truncation warnings (shared across renderers; logging only).
static int truncationLogCounter = 0;
// Renders a frame into the provided atlas + vertex direct ByteBuffers.
//
// - atlasBuf holds packed ALPHA_8 pixels with row stride atlasMaxW. Only the first
// atlasHeight rows are written; the caller uploads that region to a texture
// allocated once at atlasMaxW × atlasMaxH.
// - vertexBuf holds a per-quad vertex stream (6 vertices × (2 pos + 2 uv + 4 color)
// floats = 48 floats = 192 bytes per quad). Must match BYTES_PER_QUAD/VERTEX in
// AssSubtitleAtlasPipeline.kt. Ready for a single glDrawArrays(GL_TRIANGLES, 0, N * 6).
// - UVs are normalized against atlasMaxW × atlasMaxH (the allocated texture dims),
// not the packed region, so the texture never needs reallocation.
// - Images are packed in height-sorted rows (minimizes packed height) but vertices
// are emitted in original list order — the list order is libass's painter order.
//
// Never fails on content size: images that don't fit the remaining atlas/vertex
// capacity are dropped and counted in AssAtlasFrame.truncated, so a heavy frame
// degrades instead of going stale. Returns NULL only for missing buffers/handles.
// On changed == 0, returns (0, 0, 0, changed, 0) without touching the buffers —
// caller reuses the atlas texture already on the GPU.
JNIEXPORT jobject JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderFrameAtlas(
JNIEnv* env, jclass clazz, jlong render, jlong track, jlong time, jobject atlasBuf, jint atlasMaxW,
jint atlasMaxH, jobject vertexBuf) {
if (!render || !track || !atlasBuf || !vertexBuf || atlasMaxW <= 0 || atlasMaxH <= 0) return NULL;
jclass atlasFrameClass = (*env)->FindClass(env, "com/edde746/plezy/libass/AssAtlasFrame");
if (!atlasFrameClass) return NULL;
jmethodID ctor = (*env)->GetMethodID(env, atlasFrameClass, "<init>", "(IIIII)V");
if (!ctor) return NULL;
const long long t0 = nowMs();
int changed;
ASS_Image* image = ass_render_frame((ASS_Renderer*)render, (ASS_Track*)track, time, &changed);
const long long tAss = nowMs();
if (changed == 0 || image == NULL) {
if (tAss - t0 > 40) {
__android_log_print(
ANDROID_LOG_WARN, LOG_TAG, "slow render t=%lldms: ass=%lldms (changed=%d, no output)", (long long)time,
tAss - t0, changed);
}
return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, 0);
}
uint8_t* atlasPixels = (uint8_t*)(*env)->GetDirectBufferAddress(env, atlasBuf);
jlong atlasCap = (*env)->GetDirectBufferCapacity(env, atlasBuf);
float* vertices = (float*)(*env)->GetDirectBufferAddress(env, vertexBuf);
jlong vertexCap = (*env)->GetDirectBufferCapacity(env, vertexBuf);
if (!atlasPixels || !vertices) return NULL;
if ((jlong)atlasMaxW * atlasMaxH > atlasCap) {
__android_log_print(
ANDROID_LOG_ERROR, LOG_TAG, "atlas buffer smaller than %dx%d (capacity %lld bytes)", atlasMaxW, atlasMaxH,
(long long)atlasCap);
return NULL;
}
// 48 floats per quad × 4 bytes = 192 bytes/quad
const int maxQuads = (int)(vertexCap / 192);
int total = 0;
for (ASS_Image* img = image; img != NULL; img = img->next) {
if (img->w > 0 && img->h > 0) total++;
}
if (total == 0) {
return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, 0);
}
// Pass 1: assign packing slots in height-sorted order so mixed-size frames pack
// tight rows. slotX/slotY are keyed by the image's position in the original
// list; -1 marks images dropped for capacity.
PackItem* items = (PackItem*)malloc(sizeof(PackItem) * (size_t)total);
int* slotX = (int*)malloc(sizeof(int) * (size_t)total);
int* slotY = (int*)malloc(sizeof(int) * (size_t)total);
if (!items || !slotX || !slotY) {
free(items);
free(slotX);
free(slotY);
return NULL;
}
int n = 0;
for (ASS_Image* img = image; img != NULL; img = img->next) {
if (img->w > 0 && img->h > 0) {
items[n].img = img;
items[n].idx = n;
n++;
}
}
qsort(items, (size_t)n, sizeof(PackItem), comparePackItemsByHeightDesc);
int cursorX = 0, cursorY = 0, rowH = 0;
int truncated = 0;
int packedH = 0;
int accepted = 0;
long long srcPixels = 0;
for (int i = 0; i < n; i++) {
ASS_Image* img = items[i].img;
srcPixels += (long long)img->w * img->h;
int sx = -1, sy = -1;
if (img->w <= atlasMaxW && accepted < maxQuads) {
int cx = cursorX, cy = cursorY, rh = rowH;
if (cx + img->w > atlasMaxW) {
cy += rh;
cx = 0;
rh = 0;
}
if (cy + img->h <= atlasMaxH) {
sx = cx;
sy = cy;
cursorX = cx + img->w;
cursorY = cy;
rowH = (img->h > rh) ? img->h : rh;
if (cy + img->h > packedH) packedH = cy + img->h;
accepted++;
}
}
if (sx < 0) truncated++;
slotX[items[i].idx] = sx;
slotY[items[i].idx] = sy;
}
if (truncated > 0 && (truncationLogCounter++ & 63) == 0) {
__android_log_print(
ANDROID_LOG_WARN, LOG_TAG, "atlas truncation: %d of %d images dropped (atlas %dx%d, %d quads max)",
truncated, n, atlasMaxW, atlasMaxH, maxQuads);
}
if (accepted == 0) {
free(items);
free(slotX);
free(slotY);
return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, truncated);
}
memset(atlasPixels, 0, (size_t)atlasMaxW * packedH);
// Pass 2: walk the original list (= libass's painter/blend order), copying each
// accepted image into its assigned slot and emitting its quad.
int qi = 0;
int k = 0;
for (ASS_Image* img = image; img != NULL; img = img->next) {
if (img->w <= 0 || img->h <= 0) continue;
const int px = slotX[k];
const int py = slotY[k];
k++;
if (px < 0) continue;
for (int y = 0; y < img->h; y++) {
uint8_t* dst = atlasPixels + (size_t)(py + y) * atlasMaxW + px;
const uint8_t* src = img->bitmap + (size_t)y * img->stride;
memcpy(dst, src, (size_t)img->w);
}
const float x0 = (float)img->dst_x;
const float y0 = (float)img->dst_y;
const float x1 = x0 + (float)img->w;
const float y1 = y0 + (float)img->h;
const float u0 = (float)px / (float)atlasMaxW;
const float v0 = (float)py / (float)atlasMaxH;
const float u1 = (float)(px + img->w) / (float)atlasMaxW;
const float v1 = (float)(py + img->h) / (float)atlasMaxH;
const unsigned int c = img->color;
const float r = (float)((c >> 24) & 0xFFu) / 255.0f;
const float g = (float)((c >> 16) & 0xFFu) / 255.0f;
const float b = (float)((c >> 8) & 0xFFu) / 255.0f;
const float a = (float)(0xFFu - (c & 0xFFu)) / 255.0f;
float* vx = vertices + (size_t)qi * 48;
// 8 floats per vertex: x, y, u, v, r, g, b, a.
// Triangle 1: (x0,y0) (x1,y0) (x0,y1)
vx[0] = x0;
vx[1] = y0;
vx[2] = u0;
vx[3] = v0;
vx[4] = r;
vx[5] = g;
vx[6] = b;
vx[7] = a;
vx[8] = x1;
vx[9] = y0;
vx[10] = u1;
vx[11] = v0;
vx[12] = r;
vx[13] = g;
vx[14] = b;
vx[15] = a;
vx[16] = x0;
vx[17] = y1;
vx[18] = u0;
vx[19] = v1;
vx[20] = r;
vx[21] = g;
vx[22] = b;
vx[23] = a;
// Triangle 2: (x1,y0) (x1,y1) (x0,y1)
vx[24] = x1;
vx[25] = y0;
vx[26] = u1;
vx[27] = v0;
vx[28] = r;
vx[29] = g;
vx[30] = b;
vx[31] = a;
vx[32] = x1;
vx[33] = y1;
vx[34] = u1;
vx[35] = v1;
vx[36] = r;
vx[37] = g;
vx[38] = b;
vx[39] = a;
vx[40] = x0;
vx[41] = y1;
vx[42] = u0;
vx[43] = v1;
vx[44] = r;
vx[45] = g;
vx[46] = b;
vx[47] = a;
qi++;
}
free(items);
free(slotX);
free(slotY);
// Slow-render breakdown: separates libass's own cost (rasterize/blur/shape)
// from this function's packing + memcpy, so device logs attribute the time.
const long long tEnd = nowMs();
if (tEnd - t0 > 40) {
__android_log_print(
ANDROID_LOG_WARN, LOG_TAG,
"slow render t=%lldms: total=%lldms ass=%lldms pack+copy=%lldms images=%d srcPx=%lldk atlas=%dx%d quads=%d",
(long long)time, tEnd - t0, tAss - t0, tEnd - tAss, n, srcPixels / 1000, atlasMaxW, packedH, qi);
}
// atlasWidth is the full row stride (GLES2 can't upload with stride ≠ width);
// atlasHeight is the packed height — the only rows worth uploading.
return (*env)->NewObject(env, atlasFrameClass, ctor, atlasMaxW, packedH, qi, changed, truncated);
}
@@ -0,0 +1,10 @@
cmake_minimum_required(VERSION 3.22.1)
project("asskt")
# libass headers + libass.so come from the io.github.peerless2012:ass AAR via prefab.
add_library(${CMAKE_PROJECT_NAME} SHARED AssKt.c)
find_package(lib_ass REQUIRED CONFIG)
target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE
lib_ass::ass
log)
@@ -0,0 +1,62 @@
package com.edde746.plezy.libass
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
class Ass {
companion object {
init {
System.loadLibrary("asskt")
}
@JvmStatic
external fun nativeAssInit(): Long
@JvmStatic
external fun nativeAssAddFont(ptr: Long, name: String, buffer: ByteArray)
@JvmStatic
external fun nativeAssDeinit(ptr: Long)
}
/** Single lock for all libass calls on this library instance. */
val lock = ReentrantLock()
private var nativeAss: Long = nativeAssInit()
@Volatile
var released = false
private set
private fun <T> create(block: (Long) -> T): T = lock.withLock {
check(!released && nativeAss != 0L) { "Ass already released" }
block(nativeAss)
}
fun createTrack(): AssTrack = create { AssTrack(it, lock) }
fun createRender(): AssRender = create { AssRender(it, lock) }
fun addFont(name: String, buffer: ByteArray) {
lock.withLock {
if (!released && nativeAss != 0L) nativeAssAddFont(nativeAss, name, buffer)
}
}
fun release() {
lock.withLock {
if (released) return
released = true
if (nativeAss != 0L) {
nativeAssDeinit(nativeAss)
nativeAss = 0
}
}
}
protected fun finalize() {
release()
}
}
@@ -0,0 +1,21 @@
package com.edde746.plezy.libass
/**
* Result of a packed-atlas render. The atlas pixel data is stored in the direct ByteBuffer
* that was passed into [AssRender.renderFrameAtlas]; the vertex stream is in the other.
*
* @param atlasWidth atlas row stride in pixels (= the allocated width; 0 when [changed] == 0)
* @param atlasHeight packed atlas height in pixels — the rows worth uploading
* @param quadCount number of quads; the vertex buffer holds [quadCount] * 6 vertices
* @param changed libass change flag (0 = no change, 1 = positions, 2 = content)
* @param truncated images dropped because they exceeded the atlas/vertex capacity;
* the frame is incomplete but never stale (> 0 should be rare —
* it means even the GL-max-sized atlas couldn't fit the frame)
*/
class AssAtlasFrame(
val atlasWidth: Int,
val atlasHeight: Int,
val quadCount: Int,
val changed: Int,
val truncated: Int
)
@@ -0,0 +1,164 @@
package com.edde746.plezy.libass
import java.nio.ByteBuffer
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
class AssRender(nativeAss: Long, private val lock: ReentrantLock) {
companion object {
@JvmStatic
external fun nativeAssRenderInit(ass: Long): Long
@JvmStatic
external fun nativeAssRenderSetFontScale(render: Long, scale: Float)
@JvmStatic
external fun nativeAssRenderSetCacheLimit(render: Long, glyphMax: Int, bitmapMaxSize: Int)
@JvmStatic
external fun nativeAssRenderSetStorageSize(render: Long, width: Int, height: Int)
@JvmStatic
external fun nativeAssRenderSetFrameSize(render: Long, width: Int, height: Int)
@JvmStatic
external fun nativeAssRenderSetMargins(render: Long, top: Int, bottom: Int, left: Int, right: Int)
@JvmStatic
external fun nativeAssRenderSetUseMargins(render: Long, use: Boolean)
@JvmStatic
external fun nativeAssRenderFrameAtlas(
render: Long,
track: Long,
time: Long,
atlasBuf: ByteBuffer,
atlasMaxWidth: Int,
atlasMaxHeight: Int,
vertexBuf: ByteBuffer
): AssAtlasFrame?
@JvmStatic
external fun nativeAssRenderDeinit(render: Long)
}
private var nativeRender: Long = nativeAssRenderInit(nativeAss)
@Volatile
var released = false
private set
private var track: AssTrack? = null
/**
* Bumped on every renderer-state mutation (track, sizes, margins, font scale).
* Lets the render-ahead pipeline detect that a speculatively rendered frame was
* produced against stale state and must not be presented.
*/
private val generation = java.util.concurrent.atomic.AtomicInteger(0)
/** Current renderer-state generation; see [generation]. */
val stateGeneration: Int get() = generation.get()
/** Runs [block] with the native handle under the shared libass lock; no-op once released. */
private inline fun withNative(block: (Long) -> Unit) {
lock.withLock {
if (!released && nativeRender != 0L) block(nativeRender)
}
}
fun setTrack(track: AssTrack?) {
generation.incrementAndGet()
lock.withLock { this.track = track }
}
fun setFontScale(scale: Float) {
generation.incrementAndGet()
withNative { nativeAssRenderSetFontScale(it, scale) }
}
fun setCacheLimit(glyphMax: Int, bitmapMaxSize: Int) = withNative { nativeAssRenderSetCacheLimit(it, glyphMax, bitmapMaxSize) }
fun setStorageSize(width: Int, height: Int) {
generation.incrementAndGet()
withNative { nativeAssRenderSetStorageSize(it, width, height) }
}
fun setFrameSize(width: Int, height: Int) {
generation.incrementAndGet()
withNative { nativeAssRenderSetFrameSize(it, width, height) }
}
/**
* mpv-style frame margins: offsets of the video dst rect within the frame set by
* [setFrameSize]. Negative when the video extends beyond the frame (zoomed in / cover).
*/
fun setMargins(top: Int, bottom: Int, left: Int, right: Int) {
generation.incrementAndGet()
withNative { nativeAssRenderSetMargins(it, top, bottom, left, right) }
}
/**
* mpv's sub-ass-force-margins: lay out non-positioned events against the full frame
* (kept on the visible screen) instead of the video rect between the margins.
*/
fun setUseMargins(use: Boolean) {
generation.incrementAndGet()
withNative { nativeAssRenderSetUseMargins(it, use) }
}
/**
* Renders a frame into a packed ALPHA_8 texture atlas plus a single vertex stream
* ready for `glDrawArrays(GL_TRIANGLES, 0, quadCount * 6)`.
*
* UVs are normalized against ([atlasMaxW], [atlasMaxH]) — the allocated texture
* dims — so the caller can allocate the texture once and `glTexSubImage2D` only
* the packed rows. Images that exceed the capacity are dropped and counted in
* [AssAtlasFrame.truncated]; the render never fails on content size.
*
* @param atlasBuf direct ByteBuffer receiving the packed pixels (≥ atlasMaxW × atlasMaxH)
* @param atlasMaxW atlas row stride in pixels (bound by `GL_MAX_TEXTURE_SIZE`)
* @param atlasMaxH atlas height in pixels (bound by `GL_MAX_TEXTURE_SIZE`)
* @param vertexBuf direct ByteBuffer receiving the vertex stream (192 bytes per quad)
*/
/** How long the most recent [renderFrameAtlas] waited to acquire the shared
* libass lock (contended by track dialogue/font feeding), in milliseconds. */
@Volatile
var lastLockWaitMs: Long = 0
private set
fun renderFrameAtlas(
time: Long,
atlasBuf: ByteBuffer,
atlasMaxW: Int,
atlasMaxH: Int,
vertexBuf: ByteBuffer
): AssAtlasFrame? {
val tQueue = System.nanoTime()
lock.withLock {
lastLockWaitMs = (System.nanoTime() - tQueue) / 1_000_000
if (released || nativeRender == 0L) return null
val t = track ?: return null
if (t.released || t.nativeAssTrack == 0L) return null
return nativeAssRenderFrameAtlas(nativeRender, t.nativeAssTrack, time, atlasBuf, atlasMaxW, atlasMaxH, vertexBuf)
}
}
fun release() {
lock.withLock {
if (released) return
released = true
track = null
if (nativeRender != 0L) {
nativeAssRenderDeinit(nativeRender)
nativeRender = 0
}
}
}
protected fun finalize() {
release()
}
}
@@ -0,0 +1,77 @@
package com.edde746.plezy.libass
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
class AssTrack(private val ass: Long, private val lock: ReentrantLock) {
companion object {
@JvmStatic
external fun nativeAssTrackInit(track: Long): Long
@JvmStatic
external fun nativeAssTrackReadBuffer(track: Long, byteArray: ByteArray, offset: Int, length: Int)
@JvmStatic
external fun nativeAssTrackReadChunk(track: Long, start: Long, duration: Long, byteArray: ByteArray, offset: Int, length: Int)
@JvmStatic
external fun nativeAssTrackDeinit(track: Long)
@JvmStatic
external fun nativeAssTrackNextEventStart(track: Long, afterMs: Long): Long
@JvmStatic
external fun nativeAssTrackNextEventChange(track: Long, afterMs: Long): Long
}
var nativeAssTrack = nativeAssTrackInit(ass)
private set
@Volatile
var released = false
private set
/** Runs [block] with the native handle under the shared libass lock; no-op once released. */
private inline fun withNative(block: (Long) -> Unit) {
lock.withLock {
if (!released && nativeAssTrack != 0L) block(nativeAssTrack)
}
}
fun readBuffer(array: ByteArray, offset: Int = 0, length: Int = array.size) = withNative { nativeAssTrackReadBuffer(it, array, offset, length) }
fun readChunk(start: Long, duration: Long, array: ByteArray, offset: Int = 0, length: Int = array.size) = withNative { nativeAssTrackReadChunk(it, start, duration, array, offset, length) }
/** Earliest event start strictly after [afterMs], or -1 if none (yet). */
fun nextEventStartMs(afterMs: Long): Long {
lock.withLock {
if (released || nativeAssTrack == 0L) return -1
return nativeAssTrackNextEventStart(nativeAssTrack, afterMs)
}
}
/** Earliest event start OR end strictly after [afterMs], or -1 if none (yet). */
fun nextEventChangeMs(afterMs: Long): Long {
lock.withLock {
if (released || nativeAssTrack == 0L) return -1
return nativeAssTrackNextEventChange(nativeAssTrack, afterMs)
}
}
fun release() {
lock.withLock {
if (released) return
released = true
if (nativeAssTrack != 0L) {
nativeAssTrackDeinit(nativeAssTrack)
nativeAssTrack = 0
}
}
}
protected fun finalize() {
release()
}
}
@@ -0,0 +1,378 @@
package com.edde746.plezy.libass.media
import android.util.Log
import androidx.annotation.OptIn
import androidx.media3.common.Format
import androidx.media3.common.MediaItem
import androidx.media3.common.MimeTypes
import androidx.media3.common.MimeTypes.TEXT_SSA
import androidx.media3.common.Player.Listener
import androidx.media3.common.Tracks
import androidx.media3.common.VideoSize
import androidx.media3.common.util.Size
import androidx.media3.common.util.UnstableApi
import androidx.media3.exoplayer.ExoPlayer
import androidx.media3.exoplayer.video.VideoFrameMetadataListener
import com.edde746.plezy.libass.Ass
import com.edde746.plezy.libass.AssRender
import com.edde746.plezy.libass.AssTrack
import com.edde746.plezy.libass.media.parser.AssHeaderParser
/**
* Handles ASS subtitle rendering and integration with ExoPlayer.
*
* This class listens to ExoPlayer events and manages the creation, selection, and rendering of ASS
* subtitle tracks. Rendering always happens on a GL overlay surface (atlas pipeline) — the
* libass frame is the overlay surface, not the video.
*/
@OptIn(UnstableApi::class)
class AssHandler(
val config: AssHandlerConfig = AssHandlerConfig()
) : Listener {
/** The ASS instance used for creating tracks and renderers. This is lazy to avoid loading
* libass if the played media does not have ASS tracks. */
private val assDelegate = lazy { Ass() }
val ass by assDelegate
/** The current ASS renderer. It's created as soon as a ASS track is detected. */
var render: AssRender? = null
private set
/**
* AssRender changed callback
*/
var renderCallback: ((AssRender?) -> Unit)? = null
/** The currently selected ASS track. */
var track: AssTrack? = null
private set
/** The available ASS tracks in the current media. */
private val availableTracks = mutableMapOf<String, AssTrack>()
/** Fonts encountered before any ASS track was created. Flushed in [createTrack]. */
private val pendingFonts = mutableListOf<Pair<String, ByteArray>>()
/** The size of the video track. */
var videoSize = Size.ZERO
private set
/** The size of the surface on which subtitles are rendered. */
var surfaceSize = Size.ZERO
private set
/**
* True once an overlay widget reported its surface size via [setOverlaySurfaceSize].
* From then on the Player.Listener surface size (= the video output surface, which
* may differ from the subtitle overlay once the overlay is parented elsewhere) is
* ignored as a frame-size source.
*/
private var overlaySizeFromWidget = false
/** mpv-style margins of the video rect within the frame: top, bottom, left, right. */
private var margins: IntArray? = null
/** mpv's sub-ass-force-margins: anchor non-positioned events to the visible frame. */
private var useMargins = false
/**
* Per-video-frame callback. Fired by ExoPlayer just before MediaCodec releases the frame
* to the output surface. Carries the exact PTS of the frame and the System.nanoTime()
* domain target release time, so subtitle renderers can align composition to the same
* display vsync as the video.
*
* - [presentationTimeUs] is track-relative microseconds matching the subtitle track's PTS.
* - [releaseTimeNs] may be [androidx.media3.common.C.TIME_UNSET] in rare paths.
*
* Invoked on the playback thread.
*/
var videoFrameCallback: ((presentationTimeUs: Long, releaseTimeNs: Long) -> Unit)? = null
private val videoFrameMetadataListener = VideoFrameMetadataListener { pts, releaseNs, _, _ ->
videoFrameCallback?.invoke(pts, releaseNs)
}
private var player: ExoPlayer? = null
/** The current selected ass format. */
private var format: Format? = null
/**
* Initializes the handler with the provided ExoPlayer instance.
* @param player The ExoPlayer instance to attach to.
*/
fun init(player: ExoPlayer) {
this.player = player
player.addListener(this)
player.setVideoFrameMetadataListener(videoFrameMetadataListener)
}
/**
* Handles transitions between media items in the player and resets everything to the initial
* state.
*/
override fun onMediaItemTransition(mediaItem: MediaItem?, reason: Int) {
super.onMediaItemTransition(mediaItem, reason)
Log.i("AssHandler", "onMediaItemTransition: item = $mediaItem, reason = $reason")
render = null
track = null
availableTracks.clear()
pendingFonts.clear()
videoSize = Size.ZERO
renderCallback?.invoke(null)
}
/**
* Handles changes to the tracks available in the current media.
* Configures the selected ASS track if available.
* @param tracks The selected tracks.
*/
override fun onTracksChanged(tracks: Tracks) {
Log.i("AssHandler", "onTracksChanged $tracks")
val selectedVideoTrack = getSelectedVideoTrack(tracks)
if (selectedVideoTrack != null) {
setVideoSize(selectedVideoTrack.width, selectedVideoTrack.height)
}
format = getSelectedAssTrack(tracks)
if (format == null) {
Log.i("AssHandler", "subtitle track disabled")
track = null
render?.setTrack(null)
return
}
updateTrack()
}
private fun updateTrack() {
val track = availableTracks.firstNotNullOfOrNull {
// When media without external subtitles, format id will not change.
// When media with external subtitles, format will become like 1:1 .
// So to compat both situation, we extract the actual id after the colon.
if (format?.id?.substringAfter(":") == it.key) {
it.value
} else {
null
}
}
if (track == null || this.track == track) return
Log.i("AssHandler", "subtitle track changed to $format")
this.track = track
val render = requireNotNull(render)
applyRenderState(render)
render.setTrack(track)
}
/**
* (Re)applies sizing and margin state to [render]. Called whenever a render is
* (re)created or the selected track changes, so renderer state survives both.
*/
private fun applyRenderState(render: AssRender) {
if (videoSize.isValid) render.setStorageSize(videoSize.width, videoSize.height)
when {
surfaceSize.isValid -> render.setFrameSize(surfaceSize.width, surfaceSize.height)
// Fallback frame until the overlay surface reports its size.
videoSize.isValid -> render.setFrameSize(videoSize.width, videoSize.height)
}
margins?.let { m -> render.setMargins(m[0], m[1], m[2], m[3]) }
render.setUseMargins(useMargins)
}
/**
* Handles changes to the surface size for video playback.
* Notifies the callback if the size has changed.
* @param width The new width of the surface.
* @param height The new height of the surface.
*/
override fun onSurfaceSizeChanged(width: Int, height: Int) {
super.onSurfaceSizeChanged(width, height)
Log.i("AssHandler", "onSurfaceSizeChanged: width = $width, height = $height")
// The video output surface is only a frame-size proxy until an overlay widget
// reports its own size — they diverge when the overlay isn't video-rect-sized.
if (overlaySizeFromWidget) return
if (surfaceSize.width == width && surfaceSize.height == height) return
surfaceSize = Size(width, height)
}
/**
* Reports the subtitle overlay widget's actual surface size — the authoritative
* libass frame size for OVERLAY render types.
*/
fun setOverlaySurfaceSize(width: Int, height: Int) {
overlaySizeFromWidget = true
if (surfaceSize.width == width && surfaceSize.height == height) return
Log.i("AssHandler", "setOverlaySurfaceSize: width = $width, height = $height")
surfaceSize = Size(width, height)
render?.setFrameSize(width, height)
}
/**
* Sets mpv-style frame margins: the offsets of the video dst rect within the libass
* frame ([setOverlaySurfaceSize]); negative when the video extends beyond the frame.
* Applied to the current render and re-applied whenever a render is (re)created.
*/
fun setMargins(top: Int, bottom: Int, left: Int, right: Int) {
margins = intArrayOf(top, bottom, left, right)
render?.setMargins(top, bottom, left, right)
}
/** mpv's sub-ass-force-margins: anchor non-positioned events to the visible frame. */
fun setUseMargins(use: Boolean) {
useMargins = use
render?.setUseMargins(use)
}
override fun onVideoSizeChanged(videoSize: VideoSize) {
super.onVideoSizeChanged(videoSize)
this.videoSize = Size(videoSize.width, videoSize.height)
Log.i("AssHandler", "onVideoSizeChanged: width = ${videoSize.width}, height = ${videoSize.height}")
}
/**
* Updates the video size for the ASS renderer. Called as soon as the video size is known in
* order to properly render subtitles.
* @param width The width of the video.
* @param height The height of the video.
*/
fun setVideoSize(width: Int, height: Int) {
Log.i("AssHandler", "setVideoSize: width = $width, height = $height")
videoSize = Size(width, height)
}
/**
* Returns true if the current media has ASS tracks, false otherwise.
*/
fun hasTracks(): Boolean = availableTracks.isNotEmpty()
/**
* Adds a font to the ASS library. If no tracks have been created yet, the font is buffered
* and will be added when the first track is created via [createTrack].
*/
@Synchronized
fun addFont(name: String, data: ByteArray) {
if (hasTracks()) {
ass.addFont(name, data)
} else {
pendingFonts.add(name to data)
}
}
/**
* Creates a new ASS track from the given format and saves it in the [availableTracks].
* The renderer and libass are also created if needed.
* @param format The format of the ASS track.
* @return The created ASS track.
*/
@Synchronized
fun createTrack(format: Format): AssTrack {
Log.i("AssHandler", "createTrack: format = $format")
// Ensure the renderer is created before creating tracks.
createRenderIfNeeded()
// Flush any fonts that were buffered before the first track was created.
if (pendingFonts.isNotEmpty()) {
for ((name, data) in pendingFonts) {
ass.addFont(name, data)
}
pendingFonts.clear()
}
val track = ass.createTrack()
if (format.initializationData.size > 0) {
val header = AssHeaderParser.parse(format)
track.readBuffer(header)
}
availableTracks[format.id!!] = track
updateTrack()
return track
}
/**
* Ensures the ASS renderer is created if it does not already exist.
*/
private fun createRenderIfNeeded() {
if (render != null) return
Log.i("AssHandler", "createRender (cacheSize: ${config.cacheSize}MB, glyphSize: ${config.glyphSize})")
render = ass.createRender().also { render ->
render.setCacheLimit(config.glyphSize, config.cacheSize)
applyRenderState(render)
}
renderCallback?.invoke(render)
}
/**
* Reads a dialogue into the track of the given [trackId].
* Thread-safe: AssTrack.readChunk internally acquires the shared libass lock.
*/
fun readTrackDialogue(
trackId: String?,
start: Long,
duration: Long,
data: ByteArray,
offset: Int = 0,
length: Int = data.size
) {
val t = availableTracks[trackId] ?: return
t.readChunk(start, duration, data, offset, length)
}
/**
* Retrieves the selected video track, if any.
*/
private fun getSelectedVideoTrack(tracks: Tracks): Format? = tracks.groups.find { group ->
if (group.isSelected) {
(0 until group.length).any { index ->
val track = group.getTrackFormat(index)
MimeTypes.isVideo(track.sampleMimeType)
}
} else {
false
}
}?.getTrackFormat(0)
/**
* Retrieves the ID of the selected ASS track, if any.
* @param tracks The selected tracks.
* @return The ID of the selected ASS track, or null if none.
*/
private fun getSelectedAssTrack(tracks: Tracks): Format? = tracks.groups.find { group ->
if (group.isSelected) {
(0 until group.length).any { index ->
val track = group.getTrackFormat(index)
track.sampleMimeType == TEXT_SSA || track.codecs == TEXT_SSA
}
} else {
false
}
}?.getTrackFormat(0)
/**
* Releases all native resources held by this handler.
*/
fun release() {
videoFrameCallback = null
player?.clearVideoFrameMetadataListener(videoFrameMetadataListener)
player = null
render?.release()
render = null
availableTracks.values.forEach { it.release() }
availableTracks.clear()
track = null
pendingFonts.clear()
if (assDelegate.isInitialized()) {
ass.release()
}
}
/**
* Checks if the size is valid (both width and height are greater than 0).
*/
private val Size.isValid
get() = width > 0 && height > 0
}
@@ -0,0 +1,6 @@
package com.edde746.plezy.libass.media
data class AssHandlerConfig(
val glyphSize: Int = 10000,
val cacheSize: Int = 128
)
@@ -0,0 +1,127 @@
package com.edde746.plezy.libass.media.extractor
import androidx.annotation.OptIn
import androidx.media3.common.util.ParsableByteArray
import androidx.media3.common.util.UnstableApi
import androidx.media3.extractor.ExtractorInput
import androidx.media3.extractor.ExtractorOutput
import androidx.media3.extractor.mkv.EbmlProcessor
import androidx.media3.extractor.mkv.MatroskaExtractor
import androidx.media3.extractor.text.SubtitleParser
import com.edde746.plezy.libass.media.AssHandler
import com.edde746.plezy.libass.media.text.AssSubtitleExtractorOutput
@OptIn(UnstableApi::class)
open class AssMatroskaExtractor(
subtitleParserFactory: SubtitleParser.Factory,
private val assHandler: AssHandler,
flags: Int = 0
) : MatroskaExtractor(subtitleParserFactory, flags) {
private var currentAttachmentName: String? = null
private var currentAttachmentMime: String? = null
internal val subtitleSample = subtitleSampleField.get(this) as ParsableByteArray
override fun getElementType(id: Int): Int = when (id) {
ID_ATTACHMENTS -> EbmlProcessor.ELEMENT_TYPE_MASTER
ID_ATTACHED_FILE -> EbmlProcessor.ELEMENT_TYPE_MASTER
ID_FILE_NAME -> EbmlProcessor.ELEMENT_TYPE_STRING
ID_FILE_MIME_TYPE -> EbmlProcessor.ELEMENT_TYPE_STRING
ID_FILE_DATA -> EbmlProcessor.ELEMENT_TYPE_BINARY
else -> super.getElementType(id)
}
override fun isLevel1Element(id: Int): Boolean = super.isLevel1Element(id) || id == ID_ATTACHMENTS
override fun startMasterElement(id: Int, contentPosition: Long, contentSize: Long) {
when (id) {
ID_EBML -> {
val currentExtractor = extractorOutput.get(this) as ExtractorOutput
if (currentExtractor !is AssSubtitleExtractorOutput) {
extractorOutput.set(
this,
AssSubtitleExtractorOutput(currentExtractor, assHandler, this)
)
}
super.startMasterElement(id, contentPosition, contentSize)
}
ID_ATTACHED_FILE -> clearAttachment()
else -> super.startMasterElement(id, contentPosition, contentSize)
}
}
override fun endMasterElement(id: Int) {
when (id) {
ID_VIDEO -> {
// We need to get the video dimensions very early
val track = getCurrentTrack(id)
assHandler.setVideoSize(track.width, track.height)
super.endMasterElement(id)
}
ID_ATTACHED_FILE -> clearAttachment()
else -> super.endMasterElement(id)
}
}
override fun stringElement(id: Int, value: String) {
when (id) {
ID_FILE_NAME -> currentAttachmentName = value
ID_FILE_MIME_TYPE -> currentAttachmentMime = value
else -> super.stringElement(id, value)
}
}
override fun binaryElement(id: Int, contentSize: Int, input: ExtractorInput) {
when (id) {
ID_FILE_DATA -> {
val attachmentName = requireNotNull(currentAttachmentName)
val attachmentMime = requireNotNull(currentAttachmentMime)
if (attachmentMime in fontMimeTypes) {
val data = ByteArray(contentSize)
input.readFully(data, 0, contentSize)
assHandler.addFont(attachmentName, data)
} else {
input.skipFully(contentSize)
}
}
else -> super.binaryElement(id, contentSize, input)
}
}
private fun clearAttachment() {
currentAttachmentName = null
currentAttachmentMime = null
}
companion object {
const val ID_EBML = 0x1A45DFA3
const val ID_VIDEO = 0xE0
const val ID_ATTACHMENTS = 0x1941A469
const val ID_ATTACHED_FILE = 0x61A7
const val ID_FILE_NAME = 0x466E
const val ID_FILE_MIME_TYPE = 0x4660
const val ID_FILE_DATA = 0x465C
val fontMimeTypes = listOf(
"font/ttf",
"font/otf",
"font/sfnt",
"font/woff",
"font/woff2",
"application/font-sfnt",
"application/font-woff",
"application/x-truetype-font",
"application/vnd.ms-opentype",
"application/x-font-ttf"
)
val extractorOutput = MatroskaExtractor::class.java.getDeclaredField("extractorOutput").apply {
isAccessible = true
}
val subtitleSampleField = MatroskaExtractor::class.java.getDeclaredField("subtitleSample").apply {
isAccessible = true
}
}
}
@@ -0,0 +1,29 @@
package com.edde746.plezy.libass.media.parser
import androidx.media3.common.Format
import androidx.media3.common.util.Consumer
import androidx.media3.common.util.UnstableApi
import androidx.media3.extractor.text.CuesWithTiming
import androidx.media3.extractor.text.SubtitleParser
import com.edde746.plezy.libass.AssTrack
/**
* Parser for full (non-embedded) ASS documents, e.g. external sidecar files.
* Feeds the whole document into the libass track; rendering happens on the
* overlay surface, so no Media3 cues are emitted.
*/
@UnstableApi
class AssFullSubtitleParser(private val track: AssTrack) : SubtitleParser {
override fun parse(
data: ByteArray,
offset: Int,
length: Int,
outputOptions: SubtitleParser.OutputOptions,
output: Consumer<CuesWithTiming>
) {
track.readBuffer(data, offset, length)
}
override fun getCueReplacementBehavior(): Int = Format.CUE_REPLACEMENT_BEHAVIOR_REPLACE
}
@@ -0,0 +1,30 @@
package com.edde746.plezy.libass.media.parser
import androidx.annotation.OptIn
import androidx.media3.common.Format
import androidx.media3.common.util.UnstableApi
@OptIn(UnstableApi::class)
object AssHeaderParser {
private const val ASS_EVENTS = "[Events]\n" +
"Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text"
/**
* Fix some ass header with error end.
* https://github.com/jellyfin/jellyfin-ffmpeg/issues/506
*/
private fun fixAssHeaderIfNeed(buffer: ByteArray): ByteArray = if (buffer[buffer.size - 1] != 0.toByte()) {
// validate ass header
buffer
} else {
// remote the last null character and append the events tag
(String(buffer, 0, buffer.size - 1) + "\n" + ASS_EVENTS).toByteArray()
}
/**
* Parses the headers from the initialization data of the given [format]. The original
* headers are preserved (duplication checks are handled by libass).
*/
fun parse(format: Format): ByteArray = fixAssHeaderIfNeed(format.initializationData[1])
}
@@ -0,0 +1,24 @@
package com.edde746.plezy.libass.media.parser
import androidx.media3.common.Format
import androidx.media3.common.util.Consumer
import androidx.media3.common.util.UnstableApi
import androidx.media3.extractor.text.CuesWithTiming
import androidx.media3.extractor.text.SubtitleParser
/**
* No operation subtitle parser.
*/
@UnstableApi
class AssNoOpSubtitleParser : SubtitleParser {
override fun parse(
p0: ByteArray,
p1: Int,
p2: Int,
p3: SubtitleParser.OutputOptions,
p4: Consumer<CuesWithTiming>
) {
}
override fun getCueReplacementBehavior(): Int = Format.CUE_REPLACEMENT_BEHAVIOR_REPLACE
}
@@ -0,0 +1,32 @@
package com.edde746.plezy.libass.media.parser
import androidx.media3.common.Format
import androidx.media3.common.MimeTypes
import androidx.media3.common.util.UnstableApi
import androidx.media3.extractor.text.DefaultSubtitleParserFactory
import androidx.media3.extractor.text.SubtitleParser
import com.edde746.plezy.libass.media.AssHandler
@UnstableApi
class AssSubtitleParserFactory(private val assHandler: AssHandler) : SubtitleParser.Factory {
private val defaultSubtitleParserFactory = DefaultSubtitleParserFactory()
override fun supportsFormat(format: Format): Boolean = defaultSubtitleParserFactory.supportsFormat(format)
override fun getCueReplacementBehavior(format: Format): Int = defaultSubtitleParserFactory.getCueReplacementBehavior(format)
override fun create(format: Format): SubtitleParser = if (format.sampleMimeType == MimeTypes.TEXT_SSA) {
val embeddedSubtitles = MimeTypes.VIDEO_MATROSKA
.contentEquals(format.containerMimeType)
val track = assHandler.createTrack(format)
if (embeddedSubtitles) {
// Embedded dialogue lines reach libass via AssTrackOutput; nothing to parse here.
AssNoOpSubtitleParser()
} else {
AssFullSubtitleParser(track)
}
} else {
defaultSubtitleParserFactory.create(format)
}
}
@@ -0,0 +1,26 @@
package com.edde746.plezy.libass.media.text
import androidx.media3.common.C
import androidx.media3.common.util.UnstableApi
import androidx.media3.extractor.ExtractorOutput
import androidx.media3.extractor.TrackOutput
import com.edde746.plezy.libass.media.AssHandler
import com.edde746.plezy.libass.media.extractor.AssMatroskaExtractor
/**
* This class is only used by the overlay renderer. It's needed to get the start time of the subtitles.
*/
@UnstableApi
class AssSubtitleExtractorOutput(
private val delegate: ExtractorOutput,
private val assHandler: AssHandler,
private val extractor: AssMatroskaExtractor
) : ExtractorOutput by delegate {
override fun track(id: Int, type: Int): TrackOutput = if (type == C.TRACK_TYPE_TEXT) {
// We can't know at this time if the subtitle track is ASS or other format, so we wrap
// every subtitle track
AssTrackOutput(delegate.track(id, type), assHandler, extractor)
} else {
delegate.track(id, type)
}
}
@@ -0,0 +1,95 @@
package com.edde746.plezy.libass.media.text
import androidx.media3.common.C
import androidx.media3.common.Format
import androidx.media3.common.MimeTypes
import androidx.media3.common.util.UnstableApi
import androidx.media3.common.util.Util
import androidx.media3.extractor.TrackOutput
import com.edde746.plezy.libass.media.AssHandler
import com.edde746.plezy.libass.media.extractor.AssMatroskaExtractor
import java.util.regex.Pattern
/**
* This class is only used by the overlay renderer. It's needed to get the start time of the subtitles.
*/
@UnstableApi
class AssTrackOutput(
private val delegate: TrackOutput,
private val assHandler: AssHandler,
private val extractor: AssMatroskaExtractor
) : TrackOutput by delegate {
private var isAss = false
private var trackId: String? = null
override fun format(format: Format) {
if (format.sampleMimeType == MimeTypes.TEXT_SSA || format.codecs == MimeTypes.TEXT_SSA) {
isAss = true
trackId = format.id
}
delegate.format(format)
}
override fun sampleMetadata(
timeUs: Long,
flags: Int,
size: Int,
offset: Int,
cryptoData: TrackOutput.CryptoData?
) {
if (isAss && timeUs.isValidTs) {
val sample = extractor.subtitleSample
val endIndex = findTokenIndex(sample.data, 1)
val lineIndex = findTokenIndex(sample.data, 2)
val rawDuration = sample.data.decodeToString(endIndex, lineIndex - 1)
val durationUs = parseTimecodeUs(rawDuration)
assHandler.readTrackDialogue(
trackId = trackId,
start = timeUs / 1000,
duration = durationUs / 1000,
data = sample.data,
offset = lineIndex,
length = sample.limit() - lineIndex
)
}
delegate.sampleMetadata(timeUs, flags, size, offset, cryptoData)
}
private fun parseTimecodeUs(timeString: String): Long {
val matcher = SSA_TIMECODE_PATTERN.matcher(timeString.trim { it <= ' ' })
if (!matcher.matches()) {
return C.TIME_UNSET
}
var timestampUs =
Util.castNonNull(matcher.group(1)).toLong() * 60 * 60 * C.MICROS_PER_SECOND
timestampUs += Util.castNonNull(matcher.group(2)).toLong() * 60 * C.MICROS_PER_SECOND
timestampUs += Util.castNonNull(matcher.group(3)).toLong() * C.MICROS_PER_SECOND
timestampUs += Util.castNonNull(matcher.group(4)).toLong() * 10000
return timestampUs
}
private fun findTokenIndex(array: ByteArray, tokenNumber: Int): Int {
if (tokenNumber == 0) return 0
var tokensFound = 0
array.forEachIndexed { index, byte ->
if (byte == COMMA && ++tokensFound == tokenNumber) {
return index + 1
}
}
return 0
}
private val Long.isValidTs
get() = this != C.TIME_UNSET
private companion object {
val SSA_TIMECODE_PATTERN: Pattern =
Pattern.compile("""(?:(\d+):)?(\d+):(\d+)[:.](\d+)""")
const val COMMA = ','.code.toByte()
}
}
@@ -0,0 +1,991 @@
package com.edde746.plezy.libass.media.widget
import android.opengl.EGL14
import android.opengl.EGLContext
import android.opengl.EGLDisplay
import android.opengl.EGLExt
import android.opengl.EGLSurface
import android.opengl.GLES20
import android.os.Handler
import android.os.HandlerThread
import android.os.Process
import android.util.Log
import android.view.Surface
import androidx.media3.common.C
import androidx.media3.common.util.GlProgram
import androidx.media3.common.util.GlUtil
import androidx.media3.common.util.Size
import androidx.media3.common.util.UnstableApi
import com.edde746.plezy.libass.AssAtlasFrame
import com.edde746.plezy.libass.media.AssHandler
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.util.concurrent.atomic.AtomicReference
/**
* Atlas-rendering pipeline behind [AssSubtitleSurfaceView].
* Runs libass on its own [HandlerThread] into a packed
* ALPHA_8 texture atlas plus a single vertex stream, and a GL thread that uploads
* both and issues one `glDrawArrays` per frame. Each swap is pinned to the video's
* target release time via [EGLExt.eglPresentationTimeANDROID]: SurfaceFlinger holds
* the buffer and composes it on the same vsync as the corresponding video frame.
*/
@UnstableApi
internal object AssAtlasPipelineConfig {
/** Flip to `true` for `adb logcat -s AssSurfaceGlThread:D AssLibassThread:D` traces. */
internal const val TIMING_LOGS = false
/** Fallback atlas row width when GL caps are unknown (EGL init failed/slow). */
internal const val FALLBACK_ATLAS_W = 2048
/** Fallback atlas height; 2048 × 4096 = 8 MB matches the long-proven default. */
internal const val FALLBACK_ATLAS_H = 4096
/**
* Pixel budget per atlas slot once GL caps are known: 4096×4096 or 2048×8192
* (16 MB ALPHA_8). Sized so heavy typesetting (full-screen gradients/blurs)
* fits where the old fixed 2048×4096 overflowed and dropped frames.
*/
internal const val ATLAS_PIXEL_BUDGET = 16 * 1024 * 1024
/** Preallocated vertex-stream capacity (192 bytes × 16384 = 3 MB per buffer). */
internal const val MAX_QUADS = 16384
/** Must match the byte layout produced by `nativeAssRenderFrameAtlas` in AssKt.c. */
internal const val BYTES_PER_VERTEX = 32
internal const val BYTES_PER_QUAD = BYTES_PER_VERTEX * 6
/**
* Kill switch for speculative render-ahead (see [SpecRenderEngine]) and the
* event prefetch. With it off (or on low-RAM devices, which stay at 2 slots)
* every request renders on-demand inside the video frame's release deadline —
* the pre-speculation behavior.
*
* Only pays off when typical changed renders fit the speculation coverage
* window (~frame interval + release budget): with the un-optimized (-O0)
* native core's 90ms+ renders it could only add overhead; with the -O3/NEON
* core's ~35-50ms renders it converts near-misses into on-time latches.
*/
internal const val SPECULATION_ENABLED = true
}
/** Payload handed from the libass worker to the GL thread. */
internal class AtlasPayload(
val slotIndex: Int,
val atlasBuf: ByteBuffer,
val vertexBuf: ByteBuffer,
var frame: AssAtlasFrame,
var presentationTimeUs: Long,
var releaseTimeNs: Long,
/** Bumped on every content-changing render; lets the GL thread tell "same slot,
* new content" apart from "same slot, same content" when deciding to re-upload. */
var contentSeq: Long = 0L
)
/** Payload slots plus the atlas dims their buffers were sized for. */
internal class AtlasSlots(
val payloads: Array<AtlasPayload>,
val atlasW: Int,
val atlasH: Int
)
/**
* Owns both the libass worker and the GL thread. The two talk via a single-slot
* atomic — a newer payload always replaces a pending one so the GL thread never
* falls behind.
*/
@UnstableApi
internal class AssAtlasPipeline(
surface: Surface,
width: Int,
height: Int,
assHandler: AssHandler,
lowRamDevice: Boolean = false
) {
private val surfaceWidth = width
private val surfaceHeight = height
// 3 slots give the render-ahead engine a writable target while one slot is
// posted and another is in GL's hands; low-RAM devices stay at 2 slots with
// speculation off (the legacy on-demand behavior, ~19 MB less in buffers).
private val slotCount = if (lowRamDevice) 2 else 3
private val speculationEnabled = AssAtlasPipelineConfig.SPECULATION_ENABLED && slotCount >= 3
// Atlas dims, resolved exactly once (first-wins): normally by the GL thread from
// GL_MAX_TEXTURE_SIZE right after EGL init; by the libass thread's 1 s fallback
// if GL never comes up. Both threads then agree on the dims, which matters
// because the C side bakes UV denominators = these dims into the vertex stream
// and the GL side allocates the texture once at these dims.
private val dimsResolved = java.util.concurrent.atomic.AtomicBoolean(false)
private val dimsLatch = java.util.concurrent.CountDownLatch(1)
@Volatile private var atlasW = 0
@Volatile private var atlasH = 0
private fun resolveAtlasDims(maxTextureSize: Int): Pair<Int, Int> {
if (dimsResolved.compareAndSet(false, true)) {
if (maxTextureSize < 4096) {
// Query failed (or an ancient GPU): keep the long-proven fixed size.
atlasW = AssAtlasPipelineConfig.FALLBACK_ATLAS_W
atlasH = AssAtlasPipelineConfig.FALLBACK_ATLAS_H
} else {
val w = if (surfaceWidth > 2048) 4096 else 2048
atlasW = w
atlasH = minOf(maxTextureSize, AssAtlasPipelineConfig.ATLAS_PIXEL_BUDGET / w)
}
if (AssAtlasPipelineConfig.TIMING_LOGS) {
Log.d("AssAtlasPipeline", "atlas dims ${atlasW}x$atlasH (glMaxTexture=$maxTextureSize)")
}
dimsLatch.countDown()
}
return atlasW to atlasH
}
// Lazily allocated on the libass thread once atlas dims are known; ~19 MB of
// direct buffers per slot at the full budget, so don't pay it before the
// first actual render. Confined to the libass thread after creation.
private var slots: AtlasSlots? = null
private fun acquireSlots(): AtlasSlots {
slots?.let { return it }
if (!dimsLatch.await(1, java.util.concurrent.TimeUnit.SECONDS)) {
resolveAtlasDims(0) // first-wins: no-op if the GL thread resolved meanwhile
}
val w = atlasW
val h = atlasH
val payloads = Array(slotCount) { index ->
AtlasPayload(
slotIndex = index,
atlasBuf = ByteBuffer.allocateDirect(w * h).order(ByteOrder.nativeOrder()),
vertexBuf = ByteBuffer.allocateDirect(
AssAtlasPipelineConfig.MAX_QUADS * AssAtlasPipelineConfig.BYTES_PER_QUAD
).order(ByteOrder.nativeOrder()),
frame = AssAtlasFrame(0, 0, 0, 0, 0),
presentationTimeUs = 0L,
releaseTimeNs = C.TIME_UNSET
)
}
return AtlasSlots(payloads, w, h).also { slots = it }
}
/** Slot index the GL thread most recently took for drawing; the libass side
* never writes it. Written only inside [takePending] on the GL thread. */
@Volatile private var glLastTakenSlot = -1
private val pendingPayload = AtomicReference<AtlasPayload?>(null)
private val glThread = AtlasGlThread(
surface,
width,
height,
assHandler,
takePending = {
pendingPayload.getAndSet(null)?.also { glLastTakenSlot = it.slotIndex }
},
resolveAtlasDims = ::resolveAtlasDims
)
private val libassThread = AtlasLibassThread(
assHandler,
acquireSlots = ::acquireSlots,
speculationEnabled = speculationEnabled,
glTakenSlot = { glLastTakenSlot },
onFrameReady = { payload ->
pendingPayload.set(payload)
glThread.triggerDraw()
}
)
fun start() {
if (AssAtlasPipelineConfig.TIMING_LOGS) {
Log.d(
"AssAtlasPipeline",
"start surface=${surfaceWidth}x$surfaceHeight slots=$slotCount speculation=$speculationEnabled"
)
}
glThread.start()
libassThread.start()
}
fun requestRender(presentationTimeUs: Long, releaseTimeNs: Long) {
libassThread.enqueue(presentationTimeUs, releaseTimeNs)
}
/**
* Re-renders the last requested position — for renderer state changes (margins,
* use-margins) that must become visible while playback is paused. Safe during
* playback: the next video frame's [requestRender] supersedes it (latest-wins).
*/
fun invalidate() {
libassThread.invalidate()
}
fun onSurfaceSizeChanged(width: Int, height: Int) {
glThread.onSurfaceSizeChanged(width, height)
}
/** Vsync-pinned swaps performed (excludes untimed invalidate repaints). */
val swapCount: Long get() = glThread.swapCount
/** Pinned swaps that finished past the swap-time budget (possible missed vsync). */
val lateSwapCount: Long get() = glThread.lateSwapCount
/** Worst observed swap lateness past the target release time, in milliseconds. */
val maxLateMs: Long get() = glThread.maxLateMs
/** Total libass renders performed (one per serviced request). */
val renderCount: Long get() = libassThread.renderCount
/** Renders where libass reported changed content (atlas/vertex rewritten). */
val changedRenderCount: Long get() = libassThread.changedRenderCount
/** Renders that overflowed the atlas/vertex capacity (frame content incomplete). */
val overflowCount: Long get() = libassThread.overflowCount
/** Duration of the most recent libass render, in milliseconds. */
val lastLibassMs: Long get() = libassThread.lastLibassMs
/** Worst observed libass render duration, in milliseconds. */
val maxLibassMs: Long get() = libassThread.maxLibassMs
/** Changed-render duration histogram: [≤10ms, ≤25ms, ≤42ms, ≤84ms, >84ms]. */
val libassMsHistogram: List<Long> get() = libassThread.histogramSnapshot()
/** Requests served from a pre-rendered (speculative) frame — GL-only hot path. */
val specHits: Long get() = libassThread.specHits
/** Requests where a speculative frame existed but didn't match (seek, state change). */
val specMisses: Long get() = libassThread.specMisses
/** Speculation rounds skipped (paused, pending request, no confident cadence). */
val specSkips: Long get() = libassThread.specSkips
/** Cache-warming prefetch renders of upcoming events. */
val prefetchCount: Long get() = libassThread.prefetchCount
/** Worst (minimum) lead of a changed-content pinned swap vs its target release
* time, in ms; negative = the new content was queued after the video frame's
* vsync. Long.MAX_VALUE until a changed pinned swap happened. */
val minLeadChangedMs: Long get() = glThread.minLeadChangedMs
fun releaseAndWait() {
libassThread.releaseAndWait()
glThread.releaseAndWait()
}
}
/**
* Stops a [Handler] synchronously: posts [releaseWhat] with an [Ack], waits up to 1 s
* for the handler to invoke [onReleased] (on its own looper) and signal the latch.
*/
private fun postShutdownAndWait(
handler: Handler,
releaseWhat: Int,
onReleased: () -> Unit
) {
val latch = Object()
synchronized(latch) {
handler.obtainMessage(releaseWhat, Ack(latch, onReleased)).sendToTarget()
try {
latch.wait(1_000)
} catch (_: InterruptedException) {
Thread.currentThread().interrupt()
}
}
}
/** Transport for [postShutdownAndWait] — the handler callback calls [release] then notifies. */
private class Ack(val latch: Any, val release: () -> Unit)
/**
* Runs libass off the GL thread into a packed atlas + vertex stream. Latest-wins:
* older pending renders are dropped when a newer one arrives. Slot choice, the
* changed-flag bookkeeping and speculative render-ahead live in [SpecRenderEngine];
* this thread owns the buffers, the timing/stat accounting and the GL handoff.
*/
@UnstableApi
private class AtlasLibassThread(
private val assHandler: AssHandler,
private val acquireSlots: () -> AtlasSlots,
private val speculationEnabled: Boolean,
private val glTakenSlot: () -> Int,
private val onFrameReady: (AtlasPayload) -> Unit
) : HandlerThread(TAG, Process.THREAD_PRIORITY_DISPLAY) {
/** Immutable (pts, release) request — handed off through a single atomic so a
* concurrent enqueue can neither be lost by drain's consume nor torn in half.
* [enqueueNs] timestamps the handoff so drain can report how long the request
* sat behind an in-flight render (the queue-wait component of subtitle lag). */
private class PendingFrame(val ptsUs: Long, val releaseNs: Long, val enqueueNs: Long = System.nanoTime())
private lateinit var handler: Handler
private val pending = AtomicReference<PendingFrame?>(null)
@Volatile private var lastRequestedPtsUs = UNSET
private var contentSeqCounter = 0L
// Thread-confined; created on first render so non-ASS playback never allocates.
private var engine: SpecRenderEngine? = null
val specHits: Long get() = engine?.specHits ?: 0L
val specMisses: Long get() = engine?.specMisses ?: 0L
val specSkips: Long get() = engine?.specSkips ?: 0L
val prefetchCount: Long get() = engine?.prefetchCount ?: 0L
// Telemetry; single-writer (this thread), read from the stats path.
@Volatile var renderCount = 0L
private set
@Volatile var changedRenderCount = 0L
private set
@Volatile var overflowCount = 0L
private set
@Volatile var lastLibassMs = 0L
private set
@Volatile var maxLibassMs = 0L
private set
/** Changed-render durations bucketed at ≤10 / ≤25 / ≤42 / ≤84 / >84 ms. */
private val histogram = java.util.concurrent.atomic.AtomicLongArray(5)
fun histogramSnapshot(): List<Long> = List(histogram.length()) { histogram.get(it) }
private fun recordChangedRenderMs(ms: Long) {
val bucket = when {
ms <= 10 -> 0
ms <= 25 -> 1
ms <= 42 -> 2
ms <= 84 -> 3
else -> 4
}
histogram.incrementAndGet(bucket)
}
override fun start() {
super.start()
handler = Handler(looper) { msg ->
when (msg.what) {
MSG_RENDER -> drainAndRender()
MSG_RELEASE -> {
val ack = msg.obj as Ack
ack.release()
quit()
synchronized(ack.latch) { (ack.latch as Object).notifyAll() }
}
}
true
}
}
fun enqueue(presentationTimeUs: Long, releaseTimeNs: Long) {
if (!::handler.isInitialized) return
val dropped = pending.getAndSet(PendingFrame(presentationTimeUs, releaseTimeNs))
if (dropped != null && AssAtlasPipelineConfig.TIMING_LOGS) {
// A request was coalesced away — the renderer is behind by at least one
// frame. agedMs = how long the dropped request had been waiting.
Log.d(
TAG,
"drop pts=${dropped.ptsUs / 1000}ms agedMs=${(System.nanoTime() - dropped.enqueueNs) / 1_000_000} " +
"replacedBy=${presentationTimeUs / 1000}ms"
)
}
handler.removeMessages(MSG_RENDER)
handler.sendEmptyMessage(MSG_RENDER)
}
/** Re-enqueues the last requested PTS (renderer state changed, possibly while paused). */
fun invalidate() {
val pts = lastRequestedPtsUs
if (pts == UNSET) return
// TIME_UNSET release time => the GL thread swaps immediately instead of
// vsync-pinning to a video frame that may never come while paused.
enqueue(pts, C.TIME_UNSET)
}
private fun ensureEngine(slots: AtlasSlots): SpecRenderEngine {
engine?.let { return it }
return SpecRenderEngine(
slotCount = slots.payloads.size,
speculationEnabled = speculationEnabled,
renderAt = { timeMs, slot -> timedRender(timeMs, slots, slot) },
// Renderer identity in the high bits + its state generation in the low bits:
// a recreated renderer (media item transition) can never alias a stale
// speculation, even if the new generation counter happens to match.
stateGeneration = {
assHandler.render?.let {
(System.identityHashCode(it).toLong() shl 32) or (it.stateGeneration.toLong() and 0xffffffffL)
} ?: -1L
},
glTakenSlot = glTakenSlot,
debugLog = if (AssAtlasPipelineConfig.TIMING_LOGS) ({ msg -> Log.d(TAG, msg) }) else null
).also { engine = it }
}
/** Consecutive renders that reported no content change — a static screen.
* Reset by any changed render (dialogue flips, karaoke, animated signs). */
private var unchangedStreak = 0
/** Renders into [slot]'s buffers, owning the per-render timing/stat accounting
* for both on-demand and speculative renders. */
private fun timedRender(timeMs: Long, slots: AtlasSlots, slot: Int): AssAtlasFrame? {
val render = assHandler.render ?: return null
val payload = slots.payloads[slot]
val t0 = System.nanoTime()
val frame = render.renderFrameAtlas(timeMs, payload.atlasBuf, slots.atlasW, slots.atlasH, payload.vertexBuf)
?: return null
val libassMs = (System.nanoTime() - t0) / 1_000_000
renderCount++
lastLibassMs = libassMs
if (libassMs > maxLibassMs) maxLibassMs = libassMs
if (frame.truncated > 0) overflowCount++
if (frame.changed != 0) {
changedRenderCount++
recordChangedRenderMs(libassMs)
unchangedStreak = 0
} else {
unchangedStreak++
}
return frame
}
private fun drainAndRender() {
val request = pending.getAndSet(null) ?: return
val pts = request.ptsUs
val releaseNs = request.releaseNs
lastRequestedPtsUs = pts
val tDrain = System.nanoTime()
// How long the request sat in the handoff (behind an in-flight on-demand or
// speculative render) — the queue-wait component of any subtitle lag.
val waitMs = (tDrain - request.enqueueNs) / 1_000_000
// Before any ASS render exists (SRT/VTT or no subs) do nothing — this also
// keeps the slot buffers unallocated for non-ASS playback.
if (assHandler.render == null) return
val slots = acquireSlots()
val engine = ensureEngine(slots)
val pinned = releaseNs != C.TIME_UNSET
// Budget left until the video frame's vsync when we START servicing.
val budgetMs = if (pinned) (releaseNs - tDrain) / 1_000_000 else -1L
when (val outcome = engine.service(pts, pinned)) {
is SpecRenderEngine.Outcome.Post -> {
val payload = slots.payloads[outcome.slot]
if (outcome.newContent) {
payload.frame = outcome.frame
payload.contentSeq = ++contentSeqCounter
}
payload.presentationTimeUs = pts
payload.releaseTimeNs = releaseNs
onFrameReady(payload)
if (AssAtlasPipelineConfig.TIMING_LOGS) {
Log.d(
TAG,
"render pts=${pts / 1000}ms seq=${payload.contentSeq} waitMs=$waitMs budgetMs=$budgetMs " +
"libassMs=$lastLibassMs lockWaitMs=${assHandler.render?.lastLockWaitMs} " +
"specHit=${outcome.specHit} changed=${payload.frame.changed} quads=${payload.frame.quadCount} " +
"atlas=${payload.frame.atlasWidth}x${payload.frame.atlasHeight} truncated=${payload.frame.truncated}"
)
}
}
SpecRenderEngine.Outcome.Skip -> {
if (AssAtlasPipelineConfig.TIMING_LOGS) {
Log.d(TAG, "skip pts=${pts / 1000}ms waitMs=$waitMs budgetMs=$budgetMs (no content)")
}
}
}
// Pre-render the predicted next frame in the dead time between requests so the
// next service is (usually) a GL-only hit. Never delays a waiting request.
engine.speculateAfter(pts, pinned, hasPending = pending.get() != null)?.let { write ->
val payload = slots.payloads[write.slot]
payload.frame = write.frame
payload.contentSeq = ++contentSeqCounter
if (AssAtlasPipelineConfig.TIMING_LOGS) {
Log.d(
TAG,
"spec after=${pts / 1000}ms seq=${payload.contentSeq} libassMs=$lastLibassMs " +
"lockWaitMs=${assHandler.render?.lastLockWaitMs} slot=${write.slot} quads=${write.frame.quadCount}"
)
}
}
maybePrefetch(engine, slots, pts, pinned)
}
/** Start time of the last event boundary we cache-warmed (libass/track ms). */
private var lastPrefetchedStartMs = Long.MIN_VALUE
/** Wall time of the last prefetch render, for the cooldown. */
private var lastPrefetchNs = Long.MIN_VALUE / 2
/**
* Cache-warms the next upcoming subtitle event so heavy typesetting pays its
* cache-cold rasterization (seconds on weak devices) before the sign appears
* instead of at appearance.
*
* The render blocks this thread, and during playback a new request is never
* more than one frame interval away — so a prefetch is only allowed when its
* delay cannot be SEEN, not merely when the queue is momentarily empty
* (the v1 mistake, which thrashed on densely-authored per-frame events and
* stalled visible dialogue):
* - the screen must be static ([unchangedStreak]): requests delayed behind
* the prefetch re-render identical content, so their lateness is invisible;
* - the warmed event must be the NEXT on-screen change (no other event start
* or end before it) — this also kills dense per-frame event sections,
* where the next change is always ≤ one frame away;
* - a cooldown bounds the worst-case overhead to one render per window.
*/
private fun maybePrefetch(engine: SpecRenderEngine, slots: AtlasSlots, ptsUs: Long, pinned: Boolean) {
if (!speculationEnabled) return
if (!pinned) return
if (pending.get() != null) return
if (unchangedStreak < PREFETCH_STATIC_STREAK) return
val now = System.nanoTime()
if (now - lastPrefetchNs < PREFETCH_COOLDOWN_NS) return
val track = assHandler.track ?: return
val nowMs = ptsUs / 1000
// Events closer than MIN_AHEAD are the regular per-frame path's business;
// beyond HORIZON the warmed bitmaps may be evicted before they're needed.
val targetMs = track.nextEventStartMs(nowMs + PREFETCH_MIN_AHEAD_MS)
if (targetMs < 0 || targetMs == lastPrefetchedStartMs) return
if (targetMs > nowMs + PREFETCH_HORIZON_MS) return
// Invisibility gate, time-budgeted: a prefetch is invisible as long as it
// finishes before anything on screen is due to change (the screen is static
// per the streak gate, so requests delayed behind it re-render identical
// content). Estimate the cost from the worst render seen this session —
// an overestimate only skips warming; an underestimate delays one boundary
// by the shortfall. nextEventChangeMs also sees ends, so a dialogue line
// due to disappear inside the budget skips the prefetch.
val nextChangeMs = track.nextEventChangeMs(nowMs)
if (nextChangeMs in 0 until targetMs) {
val runwayMs = nextChangeMs - nowMs
val costEstimateMs = (maxLibassMs * 5 / 4).coerceIn(PREFETCH_COST_FLOOR_MS, PREFETCH_COST_CEIL_MS)
if (runwayMs < costEstimateMs + PREFETCH_SAFETY_MS) return
}
lastPrefetchedStartMs = targetMs
lastPrefetchNs = now
engine.prefetch(targetMs * 1000)?.let { write ->
val payload = slots.payloads[write.slot]
payload.frame = write.frame
payload.contentSeq = ++contentSeqCounter
}
if (AssAtlasPipelineConfig.TIMING_LOGS) {
Log.d(
TAG,
"prefetch evt=${targetMs}ms aheadMs=${targetMs - nowMs} " +
"tookMs=${(System.nanoTime() - now) / 1_000_000} libassMs=$lastLibassMs"
)
}
}
fun releaseAndWait() {
if (!::handler.isInitialized) {
quit()
return
}
postShutdownAndWait(handler, MSG_RELEASE) { /* nothing thread-local to tear down */ }
}
companion object {
private const val TAG = "AssLibassThread"
private const val MSG_RENDER = 1
private const val MSG_RELEASE = 2
private const val UNSET = Long.MIN_VALUE
/** Don't prefetch events the per-frame speculation will reach imminently. */
private const val PREFETCH_MIN_AHEAD_MS = 1_000L
/** Don't warm caches so early that the bitmaps could be evicted again. */
private const val PREFETCH_HORIZON_MS = 15_000L
/** Static-screen requirement before a prefetch may block this thread. */
private const val PREFETCH_STATIC_STREAK = 3
/** Minimum spacing between prefetch renders. */
private const val PREFETCH_COOLDOWN_NS = 2_000_000_000L
/** Cost-estimate clamp for the time-budgeted invisibility gate: never
* assume a prefetch cheaper than the floor (estimator may not have seen a
* heavy frame yet) nor pointlessly demand more runway than the ceiling. */
private const val PREFETCH_COST_FLOOR_MS = 250L
private const val PREFETCH_COST_CEIL_MS = 1_500L
/** Slack added to the cost estimate when checking the static runway. */
private const val PREFETCH_SAFETY_MS = 150L
}
}
/**
* Owns the EGL surface, uploads the atlas + vertex stream and issues a single
* `glDrawArrays` per frame. Swaps immediately with the swap pinned to the video's
* target release time via [EGLExt.eglPresentationTimeANDROID]; SurfaceFlinger
* holds the buffer until then, so the thread is never blocked waiting for a vsync.
*/
@UnstableApi
private class AtlasGlThread(
private val surface: Surface,
@Volatile private var width: Int,
@Volatile private var height: Int,
private val assHandler: AssHandler,
private val takePending: () -> AtlasPayload?,
private val resolveAtlasDims: (maxTextureSize: Int) -> Pair<Int, Int>
) : HandlerThread(TAG, Process.THREAD_PRIORITY_DISPLAY) {
private lateinit var handler: Handler
private var eglDisplay: EGLDisplay = EGL14.EGL_NO_DISPLAY
private var eglContext: EGLContext = EGL14.EGL_NO_CONTEXT
private var eglSurface: EGLSurface = EGL14.EGL_NO_SURFACE
private val renderer = AtlasRenderer(assHandler)
private var lastUploadedPayload: AtlasPayload? = null
private var lastUploadedSeq = -1L
private var lastSwappedSeq = -1L
// Lateness telemetry; single-writer (this thread), read from the stats path.
@Volatile var swapCount = 0L
private set
@Volatile var lateSwapCount = 0L
private set
@Volatile var maxLateMs = 0L
private set
/** Minimum lead (release target swap completion) over changed-content pinned
* swaps; negative = content queued after the video frame's vsync. */
@Volatile var minLeadChangedMs = Long.MAX_VALUE
private set
override fun start() {
super.start()
handler = Handler(looper) { msg ->
try {
when (msg.what) {
MSG_INIT -> initEgl()
MSG_DRAW -> drawAndSwap()
MSG_SIZE_CHANGED -> sizeChanged(width, height)
MSG_RELEASE -> {
val ack = msg.obj as Ack
ack.release()
quit()
synchronized(ack.latch) { (ack.latch as Object).notifyAll() }
}
}
} catch (e: Exception) {
Log.e(TAG, "GL thread error", e)
releaseEgl()
}
true
}
handler.sendEmptyMessage(MSG_INIT)
}
fun onSurfaceSizeChanged(width: Int, height: Int) {
this.width = width
this.height = height
handler.sendEmptyMessage(MSG_SIZE_CHANGED)
}
fun triggerDraw() {
if (!::handler.isInitialized) return
handler.removeMessages(MSG_DRAW)
handler.sendEmptyMessage(MSG_DRAW)
}
fun releaseAndWait() {
if (!::handler.isInitialized) {
quit()
return
}
postShutdownAndWait(handler, MSG_RELEASE) { releaseEgl() }
}
private fun initEgl() {
try {
eglDisplay = GlUtil.getDefaultEglDisplay()
eglContext = GlUtil.createEglContext(eglDisplay)
eglSurface = GlUtil.createEglSurface(eglDisplay, surface, C.COLOR_TRANSFER_SDR, false)
EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext)
renderer.onSurfaceCreated()
// Resolve atlas dims from real GL caps (first-wins against the libass
// thread's fallback) and allocate the texture once at those dims — uploads
// are glTexSubImage2D of the packed rows from then on.
val maxTexture = IntArray(1)
GLES20.glGetIntegerv(GLES20.GL_MAX_TEXTURE_SIZE, maxTexture, 0)
val (atlasW, atlasH) = resolveAtlasDims(maxTexture[0])
renderer.allocateAtlasTexture(atlasW, atlasH)
sizeChanged(width, height)
} catch (e: GlUtil.GlException) {
Log.e(TAG, "Failed to initialize EGL", e)
}
}
private fun sizeChanged(width: Int, height: Int) {
renderer.onSurfaceChanged(width, height)
if (eglDisplay != EGL14.EGL_NO_DISPLAY) {
GlUtil.clearFocusedBuffers()
EGL14.eglSwapBuffers(eglDisplay, eglSurface)
}
}
private fun drawAndSwap() {
if (eglDisplay == EGL14.EGL_NO_DISPLAY) return
val payload = takePending() ?: return
// Render immediately (GL commands queue on the GPU). Re-upload only when the
// slot's content actually changed — identity alone is not enough because the
// libass side rewrites slot buffers in place (contentSeq tracks the rewrites).
val t0 = System.nanoTime()
val reuse = payload === lastUploadedPayload && payload.contentSeq == lastUploadedSeq
renderer.onDrawFrame(payload, reuseUploads = reuse)
lastUploadedPayload = payload
lastUploadedSeq = payload.contentSeq
val t1 = System.nanoTime()
// Swap immediately with the presentation time set: SurfaceFlinger holds the
// queued buffer until the video frame's target release time, so the subtitle
// can never appear early, and the GL thread is free again within a couple of
// milliseconds. Sleeping here until near the target vsync (as a removed
// TextureView path once required) made this thread blind for almost a whole
// frame interval — the single-slot latest-wins handoff would then drop an
// intermediate subtitle state (e.g. the transition to blank), showing the
// previous state one frame too long.
if (payload.releaseTimeNs != C.TIME_UNSET) {
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, payload.releaseTimeNs)
}
EGL14.eglSwapBuffers(eglDisplay, eglSurface)
val t2 = System.nanoTime()
if (payload.releaseTimeNs != C.TIME_UNSET) {
swapCount++
val lateNs = t2 - payload.releaseTimeNs
if (lateNs > LATE_THRESHOLD_NS) {
lateSwapCount++
val lateMs = lateNs / 1_000_000
if (lateMs > maxLateMs) maxLateMs = lateMs
}
// Lead of changed-content swaps is the frame-perfection signal: ≥ 0 means
// the new subtitle content reached the queue before the video frame's vsync.
if (payload.contentSeq != lastSwappedSeq) {
val leadMs = -lateNs / 1_000_000
if (leadMs < minLeadChangedMs) minLeadChangedMs = leadMs
}
}
lastSwappedSeq = payload.contentSeq
if (AssAtlasPipelineConfig.TIMING_LOGS) {
val pinned = payload.releaseTimeNs != C.TIME_UNSET
// headroomMs: slack before the target vsync when GL STARTED; leadMs: slack
// when the buffer was actually queued (negative = queued after the vsync).
val headroomMs = if (pinned) (payload.releaseTimeNs - t0) / 1_000_000 else -1L
val leadMs = if (pinned) (payload.releaseTimeNs - t2) / 1_000_000 else -1L
Log.d(
TAG,
"swap pts=${payload.presentationTimeUs / 1000}ms seq=${payload.contentSeq} " +
"quads=${payload.frame.quadCount} reused=$reuse drawMs=${(t1 - t0) / 1_000_000} " +
"swapMs=${(t2 - t1) / 1_000_000} headroomMs=$headroomMs leadMs=$leadMs pinned=$pinned"
)
}
}
private fun releaseEgl() {
if (eglDisplay != EGL14.EGL_NO_DISPLAY) {
try {
renderer.onSurfaceDestroyed()
GlUtil.destroyEglSurface(eglDisplay, eglSurface)
GlUtil.destroyEglContext(eglDisplay, eglContext)
} catch (e: GlUtil.GlException) {
Log.e(TAG, "Failed to release EGL", e)
} finally {
eglDisplay = EGL14.EGL_NO_DISPLAY
eglContext = EGL14.EGL_NO_CONTEXT
eglSurface = EGL14.EGL_NO_SURFACE
}
}
}
companion object {
private const val TAG = "AssSurfaceGlThread"
private const val MSG_INIT = 1
private const val MSG_DRAW = 2
private const val MSG_SIZE_CHANGED = 3
private const val MSG_RELEASE = 4
/**
* Swaps finishing this far past the target release time arrived after the
* buffer should already have been queued and may miss the frame's vsync —
* actual slack depends on the display's vsync offset from the release time.
*/
private const val LATE_THRESHOLD_NS = 4_000_000L
}
}
/**
* GL-side work for the atlas-based path. Maintains a single atlas texture and a
* single vertex buffer; uploads them per frame (unless the payload identity
* matches the last upload) and issues one `glDrawArrays` for the whole frame.
*/
@UnstableApi
private class AtlasRenderer(private val assHandler: AssHandler) {
private val vertexShaderCode = """
attribute vec2 a_Position;
attribute vec2 a_TexCoord;
attribute vec4 a_Color;
uniform vec2 u_SurfaceSize;
varying vec2 v_TexCoord;
varying vec4 v_Color;
void main() {
vec2 clip = (a_Position / u_SurfaceSize) * 2.0 - 1.0;
clip.y = -clip.y;
gl_Position = vec4(clip, 0.0, 1.0);
v_TexCoord = a_TexCoord;
v_Color = a_Color;
}
""".trimIndent()
private val fragmentShaderCode = """
precision mediump float;
varying vec2 v_TexCoord;
varying vec4 v_Color;
uniform sampler2D u_Texture;
void main() {
float alpha = texture2D(u_Texture, v_TexCoord).a;
gl_FragColor = v_Color * alpha;
}
""".trimIndent()
private var surfaceSize = Size.ZERO
private lateinit var glProgram: GlProgram
private var atlasTexId = 0
private var vertexBufferId = 0
private var aPosition = 0
private var aTexCoord = 0
private var aColor = 0
private var uTexture = 0
private var uSurfaceSize = 0
private var atlasAllocatedW = 0
private var atlasAllocatedH = 0
/**
* Allocates the atlas texture once at the resolved dims. The C side bakes UV
* denominators = these dims into the vertex stream, so per-frame uploads can be
* partial ([uploadAtlas]) without ever reallocating — drivers keep one stable
* texture allocation instead of churning on packed-height changes.
*/
fun allocateAtlasTexture(width: Int, height: Int) {
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, atlasTexId)
GLES20.glTexImage2D(
GLES20.GL_TEXTURE_2D, 0, GLES20.GL_ALPHA,
width, height, 0,
GLES20.GL_ALPHA, GLES20.GL_UNSIGNED_BYTE, null
)
atlasAllocatedW = width
atlasAllocatedH = height
}
fun onSurfaceCreated() {
glProgram = GlProgram(vertexShaderCode, fragmentShaderCode)
GlUtil.checkGlError()
glProgram.use()
aPosition = glProgram.getAttributeArrayLocationAndEnable("a_Position")
aTexCoord = glProgram.getAttributeArrayLocationAndEnable("a_TexCoord")
aColor = glProgram.getAttributeArrayLocationAndEnable("a_Color")
uTexture = glProgram.getUniformLocation("u_Texture")
uSurfaceSize = glProgram.getUniformLocation("u_SurfaceSize")
val tex = IntArray(1)
GLES20.glGenTextures(1, tex, 0)
atlasTexId = tex[0]
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, atlasTexId)
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE)
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE)
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR)
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR)
GLES20.glUniform1i(uTexture, 0)
val buf = IntArray(1)
GLES20.glGenBuffers(1, buf, 0)
vertexBufferId = buf[0]
GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1)
GLES20.glEnable(GLES20.GL_BLEND)
GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA)
}
fun onSurfaceChanged(width: Int, height: Int) {
surfaceSize = Size(width, height)
assHandler.render?.setFrameSize(width, height)
GLES20.glViewport(0, 0, width, height)
GLES20.glUniform2f(uSurfaceSize, width.toFloat(), height.toFloat())
}
fun onDrawFrame(payload: AtlasPayload, reuseUploads: Boolean) {
GlUtil.clearFocusedBuffers()
val frame = payload.frame
val quadCount = frame.quadCount
if (quadCount == 0) return
if (!reuseUploads) {
uploadAtlas(payload.atlasBuf, frame.atlasWidth, frame.atlasHeight)
uploadVertices(payload.vertexBuf, quadCount)
}
GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vertexBufferId)
val stride = AssAtlasPipelineConfig.BYTES_PER_VERTEX
GLES20.glVertexAttribPointer(aPosition, 2, GLES20.GL_FLOAT, false, stride, 0)
GLES20.glVertexAttribPointer(aTexCoord, 2, GLES20.GL_FLOAT, false, stride, 8)
GLES20.glVertexAttribPointer(aColor, 4, GLES20.GL_FLOAT, false, stride, 16)
GLES20.glDrawArrays(GLES20.GL_TRIANGLES, 0, quadCount * 6)
}
private fun uploadAtlas(atlasBuf: ByteBuffer, atlasW: Int, atlasH: Int) {
atlasBuf.position(0).limit(atlasW * atlasH)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, atlasTexId)
if (atlasW == atlasAllocatedW && atlasH <= atlasAllocatedH) {
// Steady state: packed rows into the once-allocated texture.
GLES20.glTexSubImage2D(
GLES20.GL_TEXTURE_2D, 0, 0, 0, atlasW, atlasH,
GLES20.GL_ALPHA, GLES20.GL_UNSIGNED_BYTE, atlasBuf
)
} else {
// Defensive: dims disagree with the allocation (shouldn't happen — both
// sides resolve dims through the same first-wins gate).
Log.w("AssAtlasRenderer", "atlas upload ${atlasW}x$atlasH outside allocation ${atlasAllocatedW}x$atlasAllocatedH")
GLES20.glTexImage2D(
GLES20.GL_TEXTURE_2D, 0, GLES20.GL_ALPHA,
atlasW, atlasH, 0,
GLES20.GL_ALPHA, GLES20.GL_UNSIGNED_BYTE, atlasBuf
)
atlasAllocatedW = atlasW
atlasAllocatedH = atlasH
}
}
private fun uploadVertices(vertexBuf: ByteBuffer, quadCount: Int) {
val size = quadCount * AssAtlasPipelineConfig.BYTES_PER_QUAD
vertexBuf.position(0).limit(size)
GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vertexBufferId)
GLES20.glBufferData(GLES20.GL_ARRAY_BUFFER, size, vertexBuf, GLES20.GL_STREAM_DRAW)
}
fun onSurfaceDestroyed() {
if (atlasTexId != 0) {
val tex = intArrayOf(atlasTexId)
GLES20.glDeleteTextures(1, tex, 0)
atlasTexId = 0
}
if (vertexBufferId != 0) {
val buf = intArrayOf(vertexBufferId)
GLES20.glDeleteBuffers(1, buf, 0)
vertexBufferId = 0
}
if (::glProgram.isInitialized) glProgram.delete()
}
}
@@ -0,0 +1,103 @@
package com.edde746.plezy.libass.media.widget
import android.app.ActivityManager
import android.content.Context
import android.graphics.PixelFormat
import android.view.SurfaceHolder
import android.view.SurfaceView
import androidx.media3.common.util.UnstableApi
import com.edde746.plezy.libass.media.AssHandler
/**
* Subtitle overlay rendered through a dedicated [SurfaceView] layer.
*
* Uses a SurfaceFlinger layer directly, which lets the atlas-based pipeline
* vsync-align its swap with the corresponding video frame via
* `eglPresentationTimeANDROID`.
*/
@UnstableApi
class AssSubtitleSurfaceView(
context: Context,
private val assHandler: AssHandler
) : SurfaceView(context),
SurfaceHolder.Callback {
private var pipeline: AssAtlasPipeline? = null
init {
setZOrderMediaOverlay(true)
holder.setFormat(PixelFormat.TRANSLUCENT)
holder.addCallback(this)
}
fun requestRender(presentationTimeUs: Long, releaseTimeNs: Long) {
pipeline?.requestRender(presentationTimeUs, releaseTimeNs)
}
/** Re-renders the last position, e.g. after margin changes while paused. */
fun invalidateSubtitles() {
pipeline?.invalidate()
}
/** Vsync-pinned swaps performed by the current pipeline. */
val swapCount: Long get() = pipeline?.swapCount ?: 0L
/** Pinned swaps that finished past the swap-time budget (possible missed vsync). */
val lateSwapCount: Long get() = pipeline?.lateSwapCount ?: 0L
/** Worst observed swap lateness past the target release time, in milliseconds. */
val maxLateMs: Long get() = pipeline?.maxLateMs ?: 0L
/** Total libass renders performed by the current pipeline. */
val renderCount: Long get() = pipeline?.renderCount ?: 0L
/** Renders where libass reported changed content. */
val changedRenderCount: Long get() = pipeline?.changedRenderCount ?: 0L
/** Renders that overflowed the atlas/vertex capacity. */
val overflowCount: Long get() = pipeline?.overflowCount ?: 0L
/** Duration of the most recent libass render, in milliseconds. */
val lastLibassMs: Long get() = pipeline?.lastLibassMs ?: 0L
/** Worst observed libass render duration, in milliseconds. */
val maxLibassMs: Long get() = pipeline?.maxLibassMs ?: 0L
/** Changed-render duration histogram: [≤10ms, ≤25ms, ≤42ms, ≤84ms, >84ms]. */
val libassMsHistogram: List<Long> get() = pipeline?.libassMsHistogram ?: emptyList()
/** Requests served from a pre-rendered (speculative) frame. */
val specHits: Long get() = pipeline?.specHits ?: 0L
/** Requests where speculation existed but didn't match (seek, state change). */
val specMisses: Long get() = pipeline?.specMisses ?: 0L
/** Speculation rounds skipped (paused, pending request, no confident cadence). */
val specSkips: Long get() = pipeline?.specSkips ?: 0L
/** Cache-warming prefetch renders of upcoming events. */
val prefetchCount: Long get() = pipeline?.prefetchCount ?: 0L
/** Minimum lead of changed-content pinned swaps vs the video frame's release
* time, in ms (negative = late); null until one happened. */
val minLeadChangedMs: Long? get() = pipeline?.minLeadChangedMs?.takeIf { it != Long.MAX_VALUE }
override fun surfaceCreated(holder: SurfaceHolder) {
val rect = holder.surfaceFrame
assHandler.setOverlaySurfaceSize(rect.width(), rect.height())
val lowRam = (context.getSystemService(Context.ACTIVITY_SERVICE) as? ActivityManager)
?.isLowRamDevice ?: false
pipeline = AssAtlasPipeline(holder.surface, rect.width(), rect.height(), assHandler, lowRam)
.also { it.start() }
}
override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) {
assHandler.setOverlaySurfaceSize(width, height)
pipeline?.onSurfaceSizeChanged(width, height)
}
override fun surfaceDestroyed(holder: SurfaceHolder) {
pipeline?.releaseAndWait()
pipeline = null
}
}
@@ -0,0 +1,265 @@
package com.edde746.plezy.libass.media.widget
import com.edde746.plezy.libass.AssAtlasFrame
import kotlin.math.abs
/**
* Decision core of the speculative render-ahead pipeline.
*
* The problem it solves: ExoPlayer's frame-metadata callback gives only ~40 ms
* between "video frame scheduled" and that frame's vsync, while a changed-content
* libass render can take longer — so a render started at the callback lands a
* frame (or more) late. The engine renders the *predicted next* PTS right after
* servicing the current one, in the dead time between frames; when the next
* request matches the prediction, only the GL upload+swap (~611 ms) sits inside
* the deadline.
*
* Pure logic, single-threaded (the libass worker) — rendering, buffers and GL
* are behind injected closures, so the state machine is unit-testable. Only the
* stat fields are read from other threads.
*
* Slot ownership invariant: a slot handed to GL ([lastPostedSlot]) or currently
* read by GL ([glTakenSlot]) is never chosen as a render target. With 3 slots and
* at most 2 exclusions a target always exists; with 2 slots (low-RAM devices)
* speculation must be disabled and the legacy alternation behavior falls out.
*
* Key subtlety inherited from libass: `ass_render_frame`'s `changed` flag compares
* against libass's *previous render* — which, with speculation, may be content
* that never reached the screen. The engine therefore tracks [libassLastSlot]
* (the slot holding libass's most recent render output) rather than the last
* *posted* slot, and prefers it as the render target: on `changed == 0` the
* buffers were untouched and that slot already holds exactly the right content.
*/
internal class SpecRenderEngine(
private val slotCount: Int,
private val speculationEnabled: Boolean,
/**
* Renders the current track at [timeMs] into the slot's buffers. Returns null
* when the renderer/track is gone. The closure owns timing/stat accounting.
*/
private val renderAt: (timeMs: Long, slot: Int) -> AssAtlasFrame?,
/**
* Renderer identity + state generation as one comparable value. Must change
* whenever a speculatively rendered frame could be stale: margins, sizes,
* track switches, renderer recreation.
*/
private val stateGeneration: () -> Long,
/** Slot the GL thread most recently took for drawing, or -1. Never write it. */
private val glTakenSlot: () -> Int,
/** Diagnostic sink for hit/miss/skip decisions; null disables (no string cost). */
private val debugLog: ((String) -> Unit)? = null
) {
/** What the caller should do after [service]. */
sealed class Outcome {
/** Nothing to post (no renderer, or no rendered content exists yet). */
object Skip : Outcome()
/**
* Post [slot] to GL. [newContent] means this call wrote fresh content into
* the slot (the caller bumps the payload's content seq). [specHit] means the
* content was pre-rendered — no libass render ran inside the deadline.
*/
class Post(val slot: Int, val frame: AssAtlasFrame, val newContent: Boolean, val specHit: Boolean) : Outcome()
}
/** A speculative render that wrote new content into [slot] (bump its seq). */
class SpecWrite(val slot: Int, val frame: AssAtlasFrame)
// Speculation state: content for specPtsUs is pre-rendered and lives either in
// specSlot, or — when the spec render returned changed == 0 — is identical to
// libassLastSlot's content (specIsLibassLast).
private var specPtsUs = UNSET
private var specSlot = -1
private var specIsLibassLast = false
private var specGen = 0L
// The slot holding libass's most recent render output and its frame.
private var libassLastSlot = -1
private var libassLastFrame: AssAtlasFrame? = null
private var lastPostedSlot = -1
// Request-cadence estimator over pinned (playing) requests: median of the last
// 8 PTS deltas, valid after 4, reset on any non-monotonic or > 250 ms jump.
private val deltas = LongArray(DELTA_SAMPLES)
private var deltaCount = 0
private var deltaIndex = 0
private var lastPinnedPtsUs = UNSET
// Stats; single-writer (the libass thread), read from the stats path.
@Volatile
var specHits = 0L
private set
@Volatile
var specMisses = 0L
private set
@Volatile
var specSkips = 0L
private set
/**
* Services a render request for [ptsUs]. [pinned] is false for invalidate
* repaints (paused margin changes etc.), which never feed the cadence
* estimator and never count as speculation misses against playback.
*/
fun service(ptsUs: Long, pinned: Boolean): Outcome {
if (pinned) updateDeltaEstimator(ptsUs)
if (specPtsUs != UNSET) {
val eps = epsilonUs()
val genNow = stateGeneration()
val hit = genNow == specGen && eps > 0 && abs(ptsUs - specPtsUs) <= eps
val slot = if (specIsLibassLast) libassLastSlot else specSlot
val frame = libassLastFrame
val specPts = specPtsUs
specPtsUs = UNSET
if (hit && slot >= 0 && frame != null) {
specHits++
lastPostedSlot = slot
debugLog?.invoke("hit pts=${ptsUs / 1000}ms spec=${specPts / 1000}ms d=${(ptsUs - specPts) / 1000}ms slot=$slot")
return Outcome.Post(slot, frame, newContent = false, specHit = true)
}
specMisses++
debugLog?.invoke(
"miss pts=${ptsUs / 1000}ms spec=${specPts / 1000}ms d=${(ptsUs - specPts) / 1000}ms eps=${eps / 1000}ms " +
"gen=${if (genNow == specGen) "ok" else "CHANGED"} slot=$slot frame=${frame != null}"
)
} else {
debugLog?.invoke("no-spec pts=${ptsUs / 1000}ms")
}
// On-demand render. Preferring libassLastSlot as the target makes changed == 0
// unambiguous: the buffers were untouched and already hold the right content.
val target = renderTargetSlot() ?: return Outcome.Skip
val frame = renderAt(ptsUs / 1000, target) ?: return Outcome.Skip
if (frame.changed == 0) {
val lastSlot = libassLastSlot
val lastFrame = libassLastFrame ?: return Outcome.Skip
if (lastSlot < 0) return Outcome.Skip
lastPostedSlot = lastSlot
return Outcome.Post(lastSlot, lastFrame, newContent = false, specHit = false)
}
libassLastSlot = target
libassLastFrame = frame
lastPostedSlot = target
return Outcome.Post(target, frame, newContent = true, specHit = false)
}
/**
* Speculatively renders the predicted next request ([servicedPtsUs] + median
* delta) into a free slot. Call after posting the current frame; skipped while
* paused ([pinned] false), when a newer request is already waiting, or while
* the cadence estimator has no confident delta.
*/
fun speculateAfter(servicedPtsUs: Long, pinned: Boolean, hasPending: Boolean): SpecWrite? {
if (!speculationEnabled) return null
if (!pinned || hasPending || !deltaValid()) {
specSkips++
debugLog?.invoke(
"spec-skip after=${servicedPtsUs / 1000}ms pinned=$pinned pending=$hasPending cadence=${deltaValid()}"
)
return null
}
val target = renderTargetSlot() ?: run {
specSkips++
debugLog?.invoke("spec-skip after=${servicedPtsUs / 1000}ms no-free-slot")
return null
}
val gen = stateGeneration()
val specPts = servicedPtsUs + medianDeltaUs()
val frame = renderAt(specPts / 1000, target) ?: run {
specSkips++
return null
}
specGen = gen
specPtsUs = specPts
if (frame.changed == 0) {
// Content at specPts is identical to libass's last render — nothing was
// written; a hit will repost libassLastSlot (and GL will skip the upload).
specIsLibassLast = true
specSlot = -1
return null
}
libassLastSlot = target
libassLastFrame = frame
specIsLibassLast = false
specSlot = target
return SpecWrite(target, frame)
}
/**
* Pre-renders [ptsUs] (an upcoming event's start) purely to warm the
* renderer's glyph/bitmap caches before that content is actually needed —
* heavy typesetting otherwise pays its cache-cold rasterization (measured
* 0.82.6 s on weak devices) exactly when the sign appears. The result is
* never posted; like any render it becomes libass's last-rendered content,
* so any pending speculation is invalidated first (its slot/content
* references would go stale).
*
* Returns the slot written (caller bumps its content seq) or null when
* nothing was rendered.
*/
fun prefetch(ptsUs: Long): SpecWrite? {
specPtsUs = UNSET
val target = renderTargetSlot() ?: return null
val frame = renderAt(ptsUs / 1000, target) ?: return null
prefetchCount++
if (frame.changed == 0) return null
libassLastSlot = target
libassLastFrame = frame
return SpecWrite(target, frame)
}
/** Cache-warming prefetch renders performed; single-writer, read cross-thread. */
@Volatile
var prefetchCount = 0L
private set
private fun renderTargetSlot(): Int? {
// The GL exclusion only exists in ≥3-slot mode; with 2 slots this reduces to
// the legacy "don't write the posted slot" alternation.
val taken = if (slotCount > 2) glTakenSlot() else -1
val last = libassLastSlot
if (last >= 0 && last != lastPostedSlot && last != taken) return last
for (s in 0 until slotCount) {
if (s != lastPostedSlot && s != taken) return s
}
return null
}
private fun updateDeltaEstimator(ptsUs: Long) {
val prev = lastPinnedPtsUs
lastPinnedPtsUs = ptsUs
if (prev == UNSET) return
val d = ptsUs - prev
if (d <= 0 || d > MAX_DELTA_US) {
// Seek/discontinuity (or duplicate PTS): forget the cadence.
deltaCount = 0
deltaIndex = 0
return
}
deltas[deltaIndex] = d
deltaIndex = (deltaIndex + 1) % DELTA_SAMPLES
if (deltaCount < DELTA_SAMPLES) deltaCount++
}
private fun deltaValid() = deltaCount >= MIN_DELTA_SAMPLES
private fun medianDeltaUs(): Long {
val copy = deltas.copyOfRange(0, deltaCount)
copy.sort()
return copy[deltaCount / 2]
}
private fun epsilonUs(): Long = if (deltaValid()) minOf(medianDeltaUs() / 2, EPSILON_CAP_US) else 0L
private companion object {
const val UNSET = Long.MIN_VALUE
const val DELTA_SAMPLES = 8
const val MIN_DELTA_SAMPLES = 4
const val MAX_DELTA_US = 250_000L
const val EPSILON_CAP_US = 8_000L
}
}
@@ -0,0 +1,332 @@
package com.edde746.plezy.libass.media.widget
import com.edde746.plezy.libass.AssAtlasFrame
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Test
/**
* Tests for the speculative render-ahead state machine. The render function is
* scripted: each call records (timeMs, slot) and returns the next queued frame.
*/
class SpecRenderEngineTest {
private class RenderCall(val timeMs: Long, val slot: Int)
private class Harness(
slotCount: Int = 3,
speculationEnabled: Boolean = true
) {
val calls = mutableListOf<RenderCall>()
val script = ArrayDeque<AssAtlasFrame?>()
var generation = 0L
var glTaken = -1
val engine = SpecRenderEngine(
slotCount = slotCount,
speculationEnabled = speculationEnabled,
renderAt = { timeMs, slot ->
calls.add(RenderCall(timeMs, slot))
if (script.isEmpty()) changed() else script.removeFirst()
},
stateGeneration = { generation },
glTakenSlot = { glTaken }
)
/** Primes the cadence estimator with [n] steady requests; returns last pts. */
fun prime(n: Int = 5, startUs: Long = 0L, deltaUs: Long = 42_000L): Long {
var pts = startUs
repeat(n) {
engine.service(pts, pinned = true)
engine.speculateAfter(pts, pinned = true, hasPending = false)
pts += deltaUs
}
return pts - deltaUs
}
}
private companion object {
const val DELTA = 42_000L
fun changed(quads: Int = 5) = AssAtlasFrame(2048, 100, quads, 2, 0)
fun unchanged() = AssAtlasFrame(0, 0, 0, 0, 0)
}
@Test
fun `steady state hits serve without rendering`() {
val h = Harness()
val last = h.prime()
// Cadence is confident by now: the last speculateAfter pre-rendered last+Δ.
val before = h.calls.size
val outcome = h.engine.service(last + DELTA, pinned = true)
assertTrue(outcome is SpecRenderEngine.Outcome.Post)
outcome as SpecRenderEngine.Outcome.Post
assertTrue(outcome.specHit)
assertFalse(outcome.newContent) // content was written (and seq-bumped) at spec time
assertEquals("service must not render on a hit", before, h.calls.size)
assertEquals(1L, h.engine.specHits)
}
@Test
fun `hit tolerates jitter within epsilon`() {
val h = Harness()
val last = h.prime()
val outcome = h.engine.service(last + DELTA + 3_000, pinned = true)
assertTrue((outcome as SpecRenderEngine.Outcome.Post).specHit)
}
@Test
fun `seek misses and renders on demand into the spec slot`() {
val h = Harness()
val last = h.prime()
val specSlot = h.calls.last().slot // where the speculative content went
val before = h.calls.size
h.script.add(changed())
val outcome = h.engine.service(last + 1_000_000, pinned = true)
assertTrue(outcome is SpecRenderEngine.Outcome.Post)
outcome as SpecRenderEngine.Outcome.Post
assertFalse(outcome.specHit)
assertTrue(outcome.newContent)
assertEquals(before + 1, h.calls.size)
assertEquals("miss must render into the slot holding libass's last content", specSlot, h.calls.last().slot)
assertEquals(1L, h.engine.specMisses)
}
@Test
fun `miss with changed 0 posts the spec slot not stale screen content`() {
// The trap: spec rendered NEW content (never posted); a mismatched request
// then returns changed == 0 (identical to libass's LAST render = the spec
// content). Posting "last posted" content would be stale — the engine must
// post the slot libass last wrote.
val h = Harness()
val last = h.prime()
val specSlot = h.calls.last().slot
h.script.add(unchanged())
val outcome = h.engine.service(last + 200_000, pinned = true) // > ε, miss; estimator resets too
assertTrue(outcome is SpecRenderEngine.Outcome.Post)
outcome as SpecRenderEngine.Outcome.Post
assertFalse(outcome.specHit)
assertFalse(outcome.newContent)
assertEquals("changed==0 must repost libass's last-rendered slot", specSlot, outcome.slot)
}
@Test
fun `state generation change invalidates speculation`() {
val h = Harness()
val last = h.prime()
h.generation++ // margins/zoom/track switch between spec render and the request
h.script.add(changed())
val before = h.calls.size
val outcome = h.engine.service(last + DELTA, pinned = true)
assertTrue(outcome is SpecRenderEngine.Outcome.Post)
assertFalse((outcome as SpecRenderEngine.Outcome.Post).specHit)
assertEquals("stale spec must be re-rendered", before + 1, h.calls.size)
assertEquals(1L, h.engine.specMisses)
}
@Test
fun `unpinned requests never feed the estimator or speculate`() {
val h = Harness()
val last = h.prime()
// Paused invalidate-repaint at an arbitrary position.
h.script.add(changed())
h.engine.service(last, pinned = false)
val specBefore = h.calls.size
assertNull(h.engine.speculateAfter(last, pinned = false, hasPending = false))
assertEquals("unpinned must not speculate", specBefore, h.calls.size)
// Cadence survives the unpinned request: the next pinned pair still hits.
h.engine.service(last + DELTA, pinned = true)
h.engine.speculateAfter(last + DELTA, pinned = true, hasPending = false)
val outcome = h.engine.service(last + 2 * DELTA, pinned = true)
assertTrue((outcome as SpecRenderEngine.Outcome.Post).specHit)
}
@Test
fun `speculation skipped while a request is pending`() {
val h = Harness()
val last = h.prime()
val before = h.calls.size
assertNull(h.engine.speculateAfter(last, pinned = true, hasPending = true))
assertEquals(before, h.calls.size)
assertTrue(h.engine.specSkips > 0)
}
@Test
fun `no speculation until cadence is confident`() {
val h = Harness()
h.engine.service(0, pinned = true)
// Only one delta sample so far (needs 4).
h.engine.service(DELTA, pinned = true)
val before = h.calls.size
assertNull(h.engine.speculateAfter(DELTA, pinned = true, hasPending = false))
assertEquals(before, h.calls.size)
}
@Test
fun `render target avoids posted and gl taken slots`() {
val h = Harness()
h.script.add(changed())
val first = h.engine.service(0, pinned = true) as SpecRenderEngine.Outcome.Post
val posted = first.slot
h.glTaken = (posted + 1) % 3
h.script.add(changed())
val second = h.engine.service(1_000_000, pinned = true) as SpecRenderEngine.Outcome.Post
val expected = (0 until 3).first { it != posted && it != h.glTaken }
assertEquals(expected, second.slot)
}
@Test
fun `changed 0 before any content skips`() {
val h = Harness()
h.script.add(unchanged())
assertEquals(SpecRenderEngine.Outcome.Skip, h.engine.service(0, pinned = true))
}
@Test
fun `renderer gone skips`() {
val h = Harness()
h.script.add(null)
assertEquals(SpecRenderEngine.Outcome.Skip, h.engine.service(0, pinned = true))
}
@Test
fun `two slot mode alternates and never speculates`() {
val h = Harness(slotCount = 2, speculationEnabled = false)
h.script.add(changed())
val a = h.engine.service(0, pinned = true) as SpecRenderEngine.Outcome.Post
assertNull(h.engine.speculateAfter(0, pinned = true, hasPending = false))
h.script.add(changed())
val b = h.engine.service(DELTA, pinned = true) as SpecRenderEngine.Outcome.Post
assertTrue(a.slot != b.slot)
assertTrue(a.slot in 0..1 && b.slot in 0..1)
assertEquals(0, h.calls.count { it.slot >= 2 })
}
@Test
fun `static dialogue hit reposts the same slot without new content`() {
// Spec render returns changed == 0 (nothing moves): a hit must repost the
// last-rendered slot so GL skips the upload entirely.
val h = Harness()
h.script.add(changed())
val first = h.engine.service(0, pinned = true) as SpecRenderEngine.Outcome.Post
var pts = 0L
repeat(4) { // build cadence; renders return changed for simplicity
pts += DELTA
h.engine.service(pts, pinned = true)
}
h.script.add(unchanged()) // speculative render: nothing changes at pts+Δ
assertNull(h.engine.speculateAfter(pts, pinned = true, hasPending = false))
val before = h.calls.size
val outcome = h.engine.service(pts + DELTA, pinned = true) as SpecRenderEngine.Outcome.Post
assertTrue(outcome.specHit)
assertFalse(outcome.newContent)
assertEquals(before, h.calls.size)
assertNotNull(first) // first slot existed; hit reposts whichever slot was last rendered
}
@Test
fun `speculative write reports slot for seq bump`() {
val h = Harness()
val last = h.prime()
h.script.add(changed())
val write = h.engine.speculateAfter(last, pinned = true, hasPending = false)
assertNotNull(write)
assertEquals(h.calls.last().slot, write!!.slot)
}
@Test
fun `spec predicts pts plus median delta`() {
val h = Harness()
val last = h.prime()
h.script.add(changed())
h.engine.speculateAfter(last, pinned = true, hasPending = false)
assertEquals((last + DELTA) / 1000, h.calls.last().timeMs)
}
@Test
fun `prefetch renders the requested future pts and reports its slot`() {
val h = Harness()
h.script.add(changed())
h.engine.service(0, pinned = true) // some content on screen
h.script.add(changed())
val write = h.engine.prefetch(5_000_000)
assertNotNull(write)
assertEquals(5_000L, h.calls.last().timeMs)
assertEquals(h.calls.last().slot, write!!.slot)
assertEquals(1L, h.engine.prefetchCount)
}
@Test
fun `prefetch invalidates pending speculation`() {
// Prefetch rewrites libass's last-rendered content (and possibly the spec
// slot itself); a stale spec hit would post future-event content.
val h = Harness()
val last = h.prime() // leaves a valid spec for last+Δ
h.script.add(changed())
h.engine.prefetch(last + 5_000_000)
val before = h.calls.size
h.script.add(changed())
val outcome = h.engine.service(last + DELTA, pinned = true)
assertTrue(outcome is SpecRenderEngine.Outcome.Post)
assertFalse((outcome as SpecRenderEngine.Outcome.Post).specHit)
assertEquals("post-prefetch request must render on demand", before + 1, h.calls.size)
}
@Test
fun `render after prefetch with changed 0 posts the prefetched slot`() {
// changed==0 after a prefetch means "identical to libass's last render" =
// the prefetched content — correct exactly when playback reached the
// prefetched event. The engine must post that slot, not older content.
val h = Harness()
h.script.add(changed())
h.engine.service(0, pinned = true)
h.script.add(changed())
val write = h.engine.prefetch(5_000_000)!!
h.script.add(unchanged())
val outcome = h.engine.service(5_000_000, pinned = true) as SpecRenderEngine.Outcome.Post
assertEquals(write.slot, outcome.slot)
assertFalse(outcome.newContent)
}
@Test
fun `prefetch avoids posted and gl taken slots`() {
val h = Harness()
h.script.add(changed())
val posted = (h.engine.service(0, pinned = true) as SpecRenderEngine.Outcome.Post).slot
h.glTaken = (posted + 1) % 3
h.script.add(changed())
val write = h.engine.prefetch(5_000_000)
val expected = (0 until 3).first { it != posted && it != h.glTaken }
assertEquals(expected, write!!.slot)
}
}
+4
View File
@@ -19,7 +19,11 @@ pluginManagement {
plugins {
id("dev.flutter.flutter-plugin-loader") version "1.0.0"
id("com.android.application") version "8.9.1" apply false
id("com.android.library") version "8.9.1" apply false
id("org.jetbrains.kotlin.android") version "2.1.0" apply false
}
include(":app")
// libass subtitle module (Kotlin/JNI bindings + Media3 glue); the native libass
// core stays the upstream Maven artifact io.github.peerless2012:ass.
include(":libass")
+143
View File
@@ -0,0 +1,143 @@
#!/usr/bin/env bash
# Generates MKV test assets that prove (or disprove) frame-perfect ASS rendering:
# the video has a per-frame counter burned in (top) and the MKV carries an ASS
# track flipping the same counter (bottom) on exactly the same frame boundaries.
# Any captured frame showing video number N with subtitle number != N is a sync
# failure.
#
# Outputs (in scripts/framesync/):
# framesync-2397.mkv 23.976 fps, plain per-frame counter events
# framesync-60.mkv 60 fps variant
# framesync-stress.mkv 23.976 fps + heavy animated typesetting (blur/scale/
# rotation re-rendered every frame) to stress libass
#
# The burn-in uses, in order of preference: ffmpeg drawtext, ffmpeg subtitles
# filter, or mpv's encoding mode (mpv always bundles libass) — all three
# evaluate the counter at each frame's exact PTS, so the burned reference is
# frame-exact by construction.
#
# Verification procedure (Android, ExoPlayer path):
# 1. Play the file with the ASS track selected (SDR content and the
# ASS-tunneling block mean the layers are screen-recordable).
# 2. adb shell screenrecord /sdcard/sync.mp4 (record ~30 s, pull it)
# 3. Step through frames (e.g. ffmpeg -i sync.mp4 frames/%05d.png, or mpv with
# '.' frame-step): top (video) and bottom (player-rendered ASS) counters
# must match on EVERY captured frame. Desktop mpv is the reference player.
# 4. Cross-check at the compositor: while playing,
# adb shell dumpsys SurfaceFlinger --latency
# lists per-layer (desired, actual) present times for the video SurfaceView
# and the ASS overlay layer — matched content must share vsync timestamps.
# 5. Cheap regression: the in-app stats overlay should show subLateSwaps ≈ 0,
# subOverflows == 0, subMinLeadMs ≥ 0 and a >95% subSpecHits ratio.
set -euo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
mkdir -p framesync
cd framesync
DURATION_S=${DURATION_S:-60}
# drawtext resolves the family via fontconfig; override with FONT if needed.
FONT=${FONT:-Sans}
have_filter() { ffmpeg -hide_banner -filters 2>/dev/null | grep -q " $1 "; }
# Emits an ASS file with one Dialogue event per frame. Frame i starts at
# i*den/num seconds; timestamps are floored to ASS centisecond precision, which
# is safe because the floor error (<10 ms) is smaller than any frame interval
# generated here (16.7 / 41.7 ms) — an event can never leak onto the previous
# frame. burn=1 emits the top-aligned white reference style (for burn-in),
# burn=0 the bottom-aligned yellow player style; heavy=1 adds per-frame
# animated blur/scale/rotation events.
gen_ass() { # $1=fps_num $2=fps_den $3=frames $4=burn(0/1) $5=heavy(0/1) $6=outfile
awk -v num="$1" -v den="$2" -v frames="$3" -v burn="$4" -v heavy="$5" '
function ts(cs, h, m, s) {
h = int(cs / 360000); cs -= h * 360000
m = int(cs / 6000); cs -= m * 6000
s = int(cs / 100); cs -= s * 100
return sprintf("%d:%02d:%02d.%02d", h, m, s, cs)
}
BEGIN {
print "[Script Info]"
print "ScriptType: v4.00+"
print "PlayResX: 1280"
print "PlayResY: 720"
print ""
print "[V4+ Styles]"
print "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding"
if (burn)
print "Style: Counter,Arial,150,&H00FFFFFF,&H000000FF,&H00000000,&H00000000,-1,0,0,0,100,100,0,0,1,5,0,8,10,10,30,1"
else
print "Style: Counter,Arial,150,&H0000FFFF,&H000000FF,&H00000000,&H00000000,-1,0,0,0,100,100,0,0,1,5,0,2,10,10,30,1"
print "Style: Stress,Arial,80,&H40FF8800,&H000000FF,&H00000000,&H00000000,-1,0,0,0,100,100,0,0,1,2,0,5,10,10,10,1"
print ""
print "[Events]"
print "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text"
for (i = 0; i < frames; i++) {
start = int(i * den * 100 / num)
end = int((i + 1) * den * 100 / num)
if (end <= start) end = start + 1
printf "Dialogue: 0,%s,%s,Counter,,0,0,0,,%d\n", ts(start), ts(end), i
if (heavy) {
# Re-emitted every frame with a frame-dependent rotation/blur so libass
# reports changed content and re-rasterizes large blurred glyphs —
# approximates typesetting storms (the measured 100-400 ms frames).
printf "Dialogue: 1,%s,%s,Stress,,0,0,0,,{\\an5\\pos(640,260)\\blur12\\fscx320\\fscy320\\frz%d}#\n", ts(start), ts(end), (i * 7) % 360
printf "Dialogue: 1,%s,%s,Stress,,0,0,0,,{\\an5\\pos(640,500)\\blur18\\fscx500\\fscy220\\frz%d\\alpha&H80&}@\n", ts(start), ts(end), 359 - (i * 11) % 360
}
}
}
' /dev/null > "$6"
}
# Produces the reference video: gray frames with the frame counter burned in at
# the top. Tries drawtext, then the subtitles filter, then mpv encoding.
gen_video() { # $1=rate $2=burn_ass $3=outfile
if have_filter drawtext; then
ffmpeg -y -v error -f lavfi -i "color=c=0x202020:s=1280x720:r=$1" \
-vf "drawtext=font='${FONT}':text='%{n}':fontsize=150:fontcolor=white:borderw=5:bordercolor=black:x=(w-text_w)/2:y=30" \
-t "$DURATION_S" -c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p "$3"
elif have_filter subtitles; then
ffmpeg -y -v error -f lavfi -i "color=c=0x202020:s=1280x720:r=$1" \
-vf "subtitles=$2" \
-t "$DURATION_S" -c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p "$3"
elif command -v mpv > /dev/null; then
ffmpeg -y -v error -f lavfi -i "color=c=0x202020:s=1280x720:r=$1" \
-t "$DURATION_S" -c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p blank.mp4
mpv blank.mp4 --sub-files="$2" --vf=sub --no-audio \
--o="$3" --of=mp4 --ovc=libx264 --ovcopts=preset=veryfast,crf=18 \
--msg-level=all=error
rm -f blank.mp4
else
echo "error: need ffmpeg with drawtext or subtitles filter, or mpv (for the burn-in)" >&2
exit 1
fi
}
# Muxes the burned video + silent audio + the player-rendered ASS track.
mux() { # $1=video $2=ass $3=outfile
ffmpeg -y -v error -i "$1" -f lavfi -i "anullsrc=r=48000:cl=stereo" -i "$2" \
-map 0:v -map 1:a -map 2 -c:v copy -c:a aac -b:a 64k -c:s copy -shortest \
-metadata:s:s:0 language=eng -disposition:s:0 default "$3"
}
frames_2397=$(awk -v d="$DURATION_S" 'BEGIN { print int(d * 24000 / 1001) }')
frames_60=$(awk -v d="$DURATION_S" 'BEGIN { print d * 60 }')
echo "Generating ${DURATION_S}s assets..."
gen_ass 24000 1001 "$frames_2397" 1 0 burn-2397.ass
gen_ass 60 1 "$frames_60" 1 0 burn-60.ass
gen_ass 24000 1001 "$frames_2397" 0 0 counter-2397.ass
gen_ass 60 1 "$frames_60" 0 0 counter-60.ass
gen_ass 24000 1001 "$frames_2397" 0 1 counter-stress.ass
gen_video "24000/1001" burn-2397.ass video-2397.mp4
gen_video "60" burn-60.ass video-60.mp4
mux video-2397.mp4 counter-2397.ass framesync-2397.mkv
mux video-60.mp4 counter-60.ass framesync-60.mkv
mux video-2397.mp4 counter-stress.ass framesync-stress.mkv
rm -f video-2397.mp4 video-60.mp4 burn-2397.ass burn-60.ass
echo "Done:"
ls -la framesync-*.mkv
echo
echo "Reference check: open framesync-2397.mkv in mpv, frame-step with '.' —"
echo "top (video) and bottom (subtitle) counters must match on every frame."
echo "Then repeat in Plezy on-device and screenrecord (see header comments)."