feat(android): frame-accurate libass pipeline + optimized native core
This commit is contained in:
+1
-1
@@ -70,4 +70,4 @@ duplication-report/
|
||||
**/fastlane/report.xml
|
||||
maestro/**/*.png
|
||||
|
||||
server/deploy.sh
|
||||
server/deploy.shscripts/framesync/
|
||||
|
||||
@@ -23,24 +23,6 @@ val downloadLibmpv by tasks.registering {
|
||||
}
|
||||
}
|
||||
|
||||
val assVersion = "fp-3"
|
||||
val assDir = layout.buildDirectory.dir("libass").get().asFile
|
||||
val assAars = listOf("lib_ass-release.aar", "lib_ass_kt-release.aar", "lib_ass_media-release.aar")
|
||||
|
||||
val downloadLibass by tasks.registering {
|
||||
val stamp = File(assDir, ".version")
|
||||
outputs.upToDateWhen { stamp.exists() && stamp.readText().trim() == assVersion }
|
||||
doLast {
|
||||
assDir.mkdirs()
|
||||
val baseUrl = "https://github.com/edde746/libass-android/releases/download/$assVersion"
|
||||
assAars.forEach { name ->
|
||||
val dest = File(assDir, name)
|
||||
exec { commandLine("curl", "-sfL", "$baseUrl/$name", "-o", dest.absolutePath) }
|
||||
}
|
||||
stamp.writeText(assVersion)
|
||||
}
|
||||
}
|
||||
|
||||
val doviVersion = "2.3.1"
|
||||
val doviDir = layout.buildDirectory.dir("libdovi").get().asFile
|
||||
val doviAbis = mapOf(
|
||||
@@ -160,10 +142,9 @@ tasks.matching { it.name.contains("CMake") || it.name.contains("externalNative")
|
||||
dependsOn(downloadLibdovi)
|
||||
}
|
||||
|
||||
// Download libmpv and libass AARs before compilation
|
||||
// Download the libmpv AAR before compilation
|
||||
tasks.matching { it.name.startsWith("pre") && it.name.endsWith("Build") }.configureEach {
|
||||
dependsOn(downloadLibmpv)
|
||||
dependsOn(downloadLibass)
|
||||
}
|
||||
|
||||
dependencies {
|
||||
@@ -186,8 +167,13 @@ dependencies {
|
||||
// FFmpeg audio decoder for unsupported codecs (ALAC, DTS, TrueHD, etc.)
|
||||
implementation("org.jellyfin.media3:media3-ffmpeg-decoder:1.9.0+1")
|
||||
|
||||
// libass-android for ASS/SSA subtitle rendering
|
||||
assAars.forEach { implementation(files(File(assDir, it))) }
|
||||
// libass ASS/SSA subtitle rendering: optimized native core (libass.so +
|
||||
// prefab headers) from the edde746/libass-android fork's releases; Kotlin/JNI
|
||||
// bindings + Media3 glue live in the android/libass module. -PlocalAssCore
|
||||
// swaps in a mavenLocal()-published core (0.4.0-local) for native A/B tests.
|
||||
val assCoreVersion = if (project.hasProperty("localAssCore")) "0.4.0-local" else "0.4.1-plezy.1"
|
||||
implementation("io.github.peerless2012:ass:$assCoreVersion@aar")
|
||||
implementation(project(":libass"))
|
||||
|
||||
testImplementation("junit:junit:4.13.2")
|
||||
}
|
||||
|
||||
@@ -59,14 +59,13 @@ import androidx.media3.extractor.ts.TsExtractor
|
||||
import androidx.media3.ui.AspectRatioFrameLayout
|
||||
import androidx.media3.ui.CaptionStyleCompat
|
||||
import androidx.media3.ui.SubtitleView
|
||||
import com.edde746.plezy.libass.media.AssHandler
|
||||
import com.edde746.plezy.libass.media.parser.AssSubtitleParserFactory
|
||||
import com.edde746.plezy.libass.media.widget.AssSubtitleSurfaceView
|
||||
import com.edde746.plezy.shared.AudioFocusManager
|
||||
import com.edde746.plezy.shared.DeviceQuirks
|
||||
import com.edde746.plezy.shared.FlutterOverlayHelper
|
||||
import com.edde746.plezy.shared.FrameRateManager
|
||||
import io.github.peerless2012.ass.media.AssHandler
|
||||
import io.github.peerless2012.ass.media.parser.AssSubtitleParserFactory
|
||||
import io.github.peerless2012.ass.media.type.AssRenderType
|
||||
import io.github.peerless2012.ass.media.widget.AssSubtitleSurfaceView
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import org.chromium.net.CronetEngine
|
||||
@@ -97,6 +96,10 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
private const val DECODER_HANG_TIMEOUT_MS = 5000L
|
||||
private const val MAX_AUDIO_RECOVERY_ATTEMPTS = 2
|
||||
private const val FPS_SAMPLE_COUNT = 8
|
||||
|
||||
/** Per-frame "video is at X" logcat stream (tag AssFrameCb) for diagnosing
|
||||
* ASS subtitle lag against the libass pipeline's render/swap lines. */
|
||||
private const val ASS_FRAME_LOGS = false
|
||||
private const val TS_TIMESTAMP_SEARCH_PACKETS = 1800
|
||||
private val DV_CODEC_PROFILE_REGEX = Regex("""(?:^|,)\s*dvh[1e]\.(\d{2})""")
|
||||
|
||||
@@ -123,6 +126,9 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
private var subtitleView: SubtitleView? = null
|
||||
private var videoZoomScale: Float = 1.0f
|
||||
private var assHandler: AssHandler? = null
|
||||
private var assSubtitleView: AssSubtitleSurfaceView? = null
|
||||
private var assForceMargins = false
|
||||
private var lastAssMargins: IntArray? = null
|
||||
private var overlayLayoutListener: ViewTreeObserver.OnGlobalLayoutListener? = null
|
||||
private var lastVideoSize: VideoSize? = null
|
||||
private var exoPlayer: ExoPlayer? = null
|
||||
@@ -139,6 +145,11 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
private var tunnelingDisabledForVideoCodec: Boolean = false
|
||||
private var tunnelingDisabledForDecodedTrueHdPcm: Boolean = false
|
||||
private var tunnelingDisabledForAudioRecovery: Boolean = false
|
||||
|
||||
// Tunneled playback never fires the VideoFrameMetadataListener (media3 releases
|
||||
// frames inside the codec), which is the libass pipeline's only render trigger —
|
||||
// ASS subs would freeze. Correctness over tunneling while an ASS track is active.
|
||||
private var tunnelingDisabledForAssSubtitles: Boolean = false
|
||||
private val tunnelingDisabledForCodec: Boolean
|
||||
get() = tunnelingDisabledForAudioCodec || tunnelingDisabledForVideoCodec || tunnelingDisabledForDecodedTrueHdPcm || tunnelingDisabledForAudioRecovery
|
||||
private var currentTunneledPlayback: Boolean = false
|
||||
@@ -478,8 +489,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
.setTsExtractorTimestampSearchBytes(TS_TIMESTAMP_SEARCH_PACKETS * TsExtractor.TS_PACKET_SIZE)
|
||||
|
||||
// Inline buildWithAssSupport to retain AssHandler reference for font scale control.
|
||||
val renderType = AssRenderType.OVERLAY_OPEN_GL
|
||||
val handler = AssHandler(renderType)
|
||||
val handler = AssHandler()
|
||||
assHandler = handler
|
||||
|
||||
val assParserFactory = AssSubtitleParserFactory(handler)
|
||||
@@ -567,26 +577,28 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
.setRenderersFactory(wrappedRenderersFactory)
|
||||
.build()
|
||||
|
||||
// Add ASS overlay view inside videoAspectContainer, sibling of the video
|
||||
// SurfaceView, so it inherits the video's resize-mode measurement, zoom
|
||||
// transform, and crop: the libass frame (= surface size) maps 1:1 onto the
|
||||
// displayed video rect at any fit mode or zoom level. Parenting it outside
|
||||
// the container desyncs ASS positioning whenever the video rect exceeds the
|
||||
// screen (cover mode, zoom > 100%).
|
||||
// We use AssSubtitleSurfaceView directly (not AssSubtitleView) so we get a
|
||||
// SurfaceFlinger-layer-backed overlay that eglPresentationTimeANDROID can
|
||||
// vsync-pin. Z-order: video SurfaceView (-2) < this MediaOverlay-flagged
|
||||
// Add ASS overlay view to the full-screen surfaceContainer (NOT the zoom-scaled
|
||||
// videoAspectContainer): the libass frame = screen, and mpv-style ass_set_margins
|
||||
// describe where the video dst rect sits inside it (negative when zoomed past the
|
||||
// edges) — see updateAssMargins(). Non-positioned dialogue can then be forced
|
||||
// on-screen (sub-ass-force-margins) while positioned/typeset events stay glued to
|
||||
// the video rect, matching mpv. Also keeps the subtitle surface unscaled (crisp
|
||||
// text, no per-gesture geometry churn).
|
||||
// AssSubtitleSurfaceView gives us a SurfaceFlinger-layer-backed overlay that
|
||||
// eglPresentationTimeANDROID can vsync-pin to the video frame.
|
||||
// Z-order: video SurfaceView (-2) < this MediaOverlay-flagged
|
||||
// SurfaceView (-1) < parent canvas < Flutter SurfaceView (+1) in the window.
|
||||
// Both punches run while this subtree draws, BEFORE the later subtitleView
|
||||
// sibling renders non-ASS cues on the parent canvas.
|
||||
var assSubtitleSurfaceView: AssSubtitleSurfaceView? = null
|
||||
videoAspectContainer?.let { container ->
|
||||
// Inserted before subtitleView so both punches run before SRT/VTT cues draw
|
||||
// on the parent canvas.
|
||||
surfaceContainer?.let { container ->
|
||||
val assView = AssSubtitleSurfaceView(container.context, handler)
|
||||
assSubtitleSurfaceView = assView
|
||||
assSubtitleView = assView
|
||||
// Pre-36 sublayer is already set by the view's own setZOrderMediaOverlay(true).
|
||||
FlutterOverlayHelper.applyCompositionOrder(assView, -1)
|
||||
val subtitleIndex = container.indexOfChild(subtitleView)
|
||||
container.addView(
|
||||
assView,
|
||||
if (subtitleIndex >= 0) subtitleIndex else container.childCount,
|
||||
FrameLayout.LayoutParams(
|
||||
FrameLayout.LayoutParams.MATCH_PARENT,
|
||||
FrameLayout.LayoutParams.MATCH_PARENT
|
||||
@@ -623,7 +635,19 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
exoPlayer!!.addAnalyticsListener(decoderHangListener)
|
||||
exoPlayer!!.setVideoFrameMetadataListener { presentationTimeUs, releaseTimeNs, _, _ ->
|
||||
// ASS bypasses Media3's text renderer, so apply sub-delay before libass renders.
|
||||
assSubtitleSurfaceView?.requestRender(presentationTimeUs - subtitleDelayUs.get(), releaseTimeNs)
|
||||
assSubtitleView?.requestRender(presentationTimeUs - subtitleDelayUs.get(), releaseTimeNs)
|
||||
if (ASS_FRAME_LOGS) {
|
||||
// Reference stream for subtitle-lag diagnosis: the video frame ExoPlayer
|
||||
// is releasing right now and how far ahead of its vsync we are. Subtitle
|
||||
// "render pts=" lines lagging these pts values = pipeline behind;
|
||||
// budgetMs far from ~10-50 = release-time clock-domain trouble.
|
||||
val budgetMs = (releaseTimeNs - System.nanoTime()) / 1_000_000
|
||||
Log.d(
|
||||
"AssFrameCb",
|
||||
"video pts=${presentationTimeUs / 1000}ms budgetMs=$budgetMs" +
|
||||
(subtitleDelayUs.get().takeIf { it != 0L }?.let { " subDelayMs=${it / 1000}" } ?: "")
|
||||
)
|
||||
}
|
||||
val count = fpsTimestampCount
|
||||
if (count < FPS_SAMPLE_COUNT) {
|
||||
fpsTimestamps[count] = presentationTimeUs
|
||||
@@ -737,7 +761,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
// Player.Listener
|
||||
|
||||
override fun onCues(cueGroup: CueGroup) {
|
||||
// With OVERLAY_CANVAS mode, ASS subtitles are rendered directly by AssSubtitleView
|
||||
// ASS subtitles are rendered by the libass overlay surface.
|
||||
// This callback is for non-ASS subtitles (SRT, VTT, etc.)
|
||||
val incoming = cueGroup.cues
|
||||
lastSubtitleCues = incoming
|
||||
@@ -919,6 +943,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
|
||||
evaluateAudioCodecForTunneling()
|
||||
evaluateVideoCodecForTunneling()
|
||||
evaluateAssSubtitlesForTunneling(tracks)
|
||||
updateTunnelingState("tracks changed")
|
||||
emitTrackList()
|
||||
}
|
||||
@@ -1231,6 +1256,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
videoAspectContainer?.setAspectRatio(videoAspect)
|
||||
}
|
||||
updateSubtitleViewSize(videoWidth, videoHeight, pixelRatio)
|
||||
updateAssMargins()
|
||||
}
|
||||
|
||||
private fun updateSubtitleViewSize(videoWidth: Int, videoHeight: Int, pixelRatio: Float) {
|
||||
@@ -1243,8 +1269,8 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
val containerHeight = contentView.height
|
||||
if (containerWidth == 0 || containerHeight == 0) return
|
||||
|
||||
// Sizes the non-ASS SubtitleView only (the ASS overlay lives inside
|
||||
// videoAspectContainer and tracks the video rect automatically).
|
||||
// Sizes the non-ASS SubtitleView only (the ASS overlay is screen-sized and
|
||||
// tracks the video rect via libass margins — see updateAssMargins()).
|
||||
// In cover/stretch/zoomed-in modes text cues stay at container size so they
|
||||
// never get cropped. In letterbox mode they follow the visible video rect.
|
||||
val isLetterbox = videoAspectContainer?.resizeMode == AspectRatioFrameLayout.RESIZE_MODE_FIT
|
||||
@@ -1287,6 +1313,83 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
}
|
||||
}
|
||||
|
||||
// Pushes mpv-style libass margins: the offsets of the video dst rect within the
|
||||
// full-screen container (= the libass frame). Negative when the video extends past
|
||||
// the screen (cover mode, zoom > 1) — libass supports that explicitly. Pure math +
|
||||
// a native setter (no view mutation), so it is safe to run per layout pass and per
|
||||
// pinch-zoom tick (#1261 no-churn invariant).
|
||||
private fun updateAssMargins() {
|
||||
if (disposing) return
|
||||
val handler = assHandler ?: return
|
||||
val vs = lastVideoSize ?: return
|
||||
if (vs.width == 0 || vs.height == 0) return
|
||||
|
||||
activity.runOnUiThread {
|
||||
if (disposing) return@runOnUiThread
|
||||
val containerWidth = surfaceContainer?.width ?: 0
|
||||
val containerHeight = surfaceContainer?.height ?: 0
|
||||
if (containerWidth == 0 || containerHeight == 0) return@runOnUiThread
|
||||
|
||||
val videoAspect = (vs.width * vs.pixelWidthHeightRatio) / vs.height
|
||||
val containerAspect = containerWidth.toFloat() / containerHeight
|
||||
val resizeMode = videoAspectContainer?.resizeMode ?: AspectRatioFrameLayout.RESIZE_MODE_FIT
|
||||
|
||||
// Mirror AspectRatioFrameLayout.onMeasure: aspect mismatches <= 1% are
|
||||
// absorbed by stretching to the container instead of resizing.
|
||||
val (baseWidth, baseHeight) = if (kotlin.math.abs(videoAspect / containerAspect - 1f) <= 0.01f) {
|
||||
containerWidth.toFloat() to containerHeight.toFloat()
|
||||
} else {
|
||||
when (resizeMode) {
|
||||
// Cover: scale up to fill the container, cropping the overflow
|
||||
AspectRatioFrameLayout.RESIZE_MODE_ZOOM ->
|
||||
if (videoAspect > containerAspect) {
|
||||
containerHeight * videoAspect to containerHeight.toFloat()
|
||||
} else {
|
||||
containerWidth.toFloat() to containerWidth / videoAspect
|
||||
}
|
||||
// Stretch: video fills the container, aspect overridden
|
||||
AspectRatioFrameLayout.RESIZE_MODE_FILL ->
|
||||
containerWidth.toFloat() to containerHeight.toFloat()
|
||||
// Fit: letterbox within the container
|
||||
else ->
|
||||
if (videoAspect > containerAspect) {
|
||||
containerWidth.toFloat() to containerWidth / videoAspect
|
||||
} else {
|
||||
containerHeight * videoAspect to containerHeight.toFloat()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// videoAspectContainer is centered and zoom-scaled about its center.
|
||||
val videoWidth = Math.round(baseWidth * videoZoomScale)
|
||||
val videoHeight = Math.round(baseHeight * videoZoomScale)
|
||||
val left = (containerWidth - videoWidth) / 2
|
||||
val top = (containerHeight - videoHeight) / 2
|
||||
val right = containerWidth - videoWidth - left
|
||||
val bottom = containerHeight - videoHeight - top
|
||||
|
||||
val margins = intArrayOf(top, bottom, left, right)
|
||||
if (lastAssMargins?.contentEquals(margins) == true) return@runOnUiThread
|
||||
lastAssMargins = margins
|
||||
handler.setMargins(top, bottom, left, right)
|
||||
// Repaint at the current position so changes are visible while paused; during
|
||||
// playback the next video frame's render supersedes it (latest-wins).
|
||||
assSubtitleView?.invalidateSubtitles()
|
||||
}
|
||||
}
|
||||
|
||||
// mpv's sub-ass-force-margins, live-applied from the Dart-managed property: lay out
|
||||
// non-positioned ASS events against the visible screen instead of the video rect.
|
||||
fun setAssForceMargins(force: Boolean) {
|
||||
if (disposing) return
|
||||
activity.runOnUiThread {
|
||||
if (disposing || assForceMargins == force) return@runOnUiThread
|
||||
assForceMargins = force
|
||||
assHandler?.setUseMargins(force)
|
||||
assSubtitleView?.invalidateSubtitles()
|
||||
}
|
||||
}
|
||||
|
||||
private fun boxFitModeToResizeMode(mode: Int): Int = when (mode) {
|
||||
1 -> AspectRatioFrameLayout.RESIZE_MODE_ZOOM
|
||||
2 -> AspectRatioFrameLayout.RESIZE_MODE_FILL
|
||||
@@ -1305,6 +1408,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
updateSubtitleViewSize(vs.width, vs.height, vs.pixelWidthHeightRatio)
|
||||
}
|
||||
}
|
||||
updateAssMargins()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1321,6 +1425,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
updateSubtitleViewSize(vs.width, vs.height, vs.pixelWidthHeightRatio)
|
||||
}
|
||||
}
|
||||
updateAssMargins()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1974,6 +2079,31 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
applyTrackSelectorPolicy(reason = reason, forceSelector = forceSelector)
|
||||
}
|
||||
|
||||
/** True when [format] is an ASS/SSA subtitle track (rendered by libass). */
|
||||
private fun isAssSubtitleFormat(format: Format): Boolean =
|
||||
format.sampleMimeType == MimeTypes.TEXT_SSA || format.codecs == MimeTypes.TEXT_SSA
|
||||
|
||||
/** Sets the ASS-subtitles tunneling block; returns true when the flag changed. */
|
||||
private fun updateAssSubtitlesForTunneling(assActive: Boolean): Boolean {
|
||||
if (assActive == tunnelingDisabledForAssSubtitles) return false
|
||||
tunnelingDisabledForAssSubtitles = assActive
|
||||
emitLog(
|
||||
"info",
|
||||
"tunneling",
|
||||
if (assActive) "ASS subtitle track selected: tunneling DISABLED (frame metadata required for libass)"
|
||||
else "ASS subtitle track deselected: tunneling unblocked"
|
||||
)
|
||||
return true
|
||||
}
|
||||
|
||||
private fun evaluateAssSubtitlesForTunneling(tracks: Tracks) {
|
||||
val assSelected = tracks.groups.any { group ->
|
||||
group.type == C.TRACK_TYPE_TEXT && group.isSelected &&
|
||||
(0 until group.length).any { isAssSubtitleFormat(group.getTrackFormat(it)) }
|
||||
}
|
||||
updateAssSubtitlesForTunneling(assSelected)
|
||||
}
|
||||
|
||||
private fun applyTrackSelectorPolicy(
|
||||
reason: String,
|
||||
forceSelector: Boolean = false,
|
||||
@@ -2022,7 +2152,8 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
private fun calculateTunnelingEnabled(): Boolean? {
|
||||
val player = exoPlayer ?: return null
|
||||
val audioDelayActive = (renderersFactory?.audioDelayUs?.get() ?: 0L) != 0L
|
||||
return tunnelingUserEnabled && (player.playbackParameters.speed == 1f) && !tunnelingDisabledForCodec && !audioDelayActive
|
||||
return tunnelingUserEnabled && (player.playbackParameters.speed == 1f) && !tunnelingDisabledForCodec &&
|
||||
!tunnelingDisabledForAssSubtitles && !audioDelayActive
|
||||
}
|
||||
|
||||
private fun updateCurrentTunnelingState(reason: String, shouldTunnel: Boolean): Boolean {
|
||||
@@ -2030,7 +2161,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
currentTunneledPlayback = shouldTunnel
|
||||
val speed = exoPlayer?.playbackParameters?.speed ?: 1f
|
||||
val audioDelayActive = (renderersFactory?.audioDelayUs?.get() ?: 0L) != 0L
|
||||
emitLog("info", "tunneling", "Toggling tunneling=$shouldTunnel (reason=$reason, user=$tunnelingUserEnabled, speed=$speed, audioCodecDisabled=$tunnelingDisabledForAudioCodec, videoCodecDisabled=$tunnelingDisabledForVideoCodec, decodedTrueHdPcmDisabled=$tunnelingDisabledForDecodedTrueHdPcm, audioRecoveryDisabled=$tunnelingDisabledForAudioRecovery, audioDelay=$audioDelayActive)")
|
||||
emitLog("info", "tunneling", "Toggling tunneling=$shouldTunnel (reason=$reason, user=$tunnelingUserEnabled, speed=$speed, audioCodecDisabled=$tunnelingDisabledForAudioCodec, videoCodecDisabled=$tunnelingDisabledForVideoCodec, decodedTrueHdPcmDisabled=$tunnelingDisabledForDecodedTrueHdPcm, audioRecoveryDisabled=$tunnelingDisabledForAudioRecovery, assSubtitlesDisabled=$tunnelingDisabledForAssSubtitles, audioDelay=$audioDelayActive)")
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -2423,6 +2554,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
tunnelingDisabledForVideoCodec = false
|
||||
tunnelingDisabledForDecodedTrueHdPcm = false
|
||||
tunnelingDisabledForAudioRecovery = false
|
||||
tunnelingDisabledForAssSubtitles = false
|
||||
currentTunneledPlayback = false
|
||||
pendingStartPositionMs = startPositionMs
|
||||
pendingPlayWhenReady = autoPlay
|
||||
@@ -2605,6 +2737,9 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
fun selectSubtitleTrack(trackId: String?) {
|
||||
if (trackId == null || trackId == "no") {
|
||||
selectedSubtitleTrackId = "no"
|
||||
// Flip the tunneling block in the same parameters update as the text
|
||||
// disable so the renderer re-initializes once, not twice.
|
||||
updateAssSubtitlesForTunneling(false)
|
||||
applyTrackSelectorPolicy(
|
||||
reason = "subtitle disabled",
|
||||
textDisabled = true
|
||||
@@ -2615,6 +2750,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
|
||||
val trackGroup = subtitleTrackGroupMap[trackId] ?: return
|
||||
selectedSubtitleTrackId = trackId
|
||||
updateAssSubtitlesForTunneling(isAssSubtitleFormat(trackGroup.getFormat(0)))
|
||||
applyTrackSelectorPolicy(
|
||||
reason = "subtitle track selected",
|
||||
textOverride = TrackSelectionOverride(trackGroup, 0),
|
||||
@@ -2898,6 +3034,26 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
"videoDecoderName" to videoDecoderName,
|
||||
"videoDroppedFrames" to player.videoDecoderCounters?.droppedBufferCount,
|
||||
"videoRenderedFrames" to player.videoDecoderCounters?.renderedOutputBufferCount,
|
||||
// ASS overlay swap timing (vsync-pinned; late = past the swap-time budget)
|
||||
"subSwapCount" to assSubtitleView?.swapCount,
|
||||
"subLateSwaps" to assSubtitleView?.lateSwapCount,
|
||||
"subMaxLateMs" to assSubtitleView?.maxLateMs,
|
||||
// ASS libass render cost (changed renders rewrite the atlas; histogram
|
||||
// buckets: ≤10/≤25/≤42/≤84/>84 ms)
|
||||
"subRenderCount" to assSubtitleView?.renderCount,
|
||||
"subChangedRenders" to assSubtitleView?.changedRenderCount,
|
||||
"subOverflows" to assSubtitleView?.overflowCount,
|
||||
"subLibassLastMs" to assSubtitleView?.lastLibassMs,
|
||||
"subLibassMaxMs" to assSubtitleView?.maxLibassMs,
|
||||
"subLibassHist" to assSubtitleView?.libassMsHistogram,
|
||||
// ASS render-ahead: hits = served from a pre-rendered frame (GL-only path);
|
||||
// minLead ≥ 0 means changed content reached the queue before the video
|
||||
// frame's vsync — the frame-perfection signal.
|
||||
"subSpecHits" to assSubtitleView?.specHits,
|
||||
"subSpecMisses" to assSubtitleView?.specMisses,
|
||||
"subSpecSkips" to assSubtitleView?.specSkips,
|
||||
"subPrefetches" to assSubtitleView?.prefetchCount,
|
||||
"subMinLeadMs" to assSubtitleView?.minLeadChangedMs,
|
||||
// Color info
|
||||
"colorSpace" to videoFormat?.colorInfo?.colorSpace,
|
||||
"colorRange" to videoFormat?.colorInfo?.colorRange,
|
||||
@@ -2992,6 +3148,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
if (tunnelingDisabledForDecodedTrueHdPcm) return "Off (decoded TrueHD PCM)"
|
||||
if (tunnelingDisabledForVideoCodec) return "Off (video codec unsupported)"
|
||||
if (tunnelingDisabledForAudioCodec) return "Off (no HW audio decoder)"
|
||||
if (tunnelingDisabledForAssSubtitles) return "Off (ASS subtitles active)"
|
||||
return "Off"
|
||||
}
|
||||
|
||||
@@ -3036,6 +3193,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
tunnelingDisabledForVideoCodec = false
|
||||
tunnelingDisabledForDecodedTrueHdPcm = false
|
||||
tunnelingDisabledForAudioRecovery = false
|
||||
tunnelingDisabledForAssSubtitles = false
|
||||
currentTunneledPlayback = false
|
||||
pendingStartPositionMs = 0L
|
||||
pendingPlayWhenReady = null
|
||||
@@ -3070,6 +3228,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener {
|
||||
videoAspectContainer = null
|
||||
surfaceView = null
|
||||
subtitleView = null
|
||||
assSubtitleView = null
|
||||
|
||||
// Remove layout listener synchronously
|
||||
overlayLayoutListener?.let { listener ->
|
||||
|
||||
@@ -622,6 +622,9 @@ class ExoPlayerPlugin :
|
||||
when (name) {
|
||||
"audio-delay" -> playerCore?.setAudioDelay(value.toDoubleOrNull() ?: 0.0)
|
||||
"sub-delay" -> playerCore?.setSubtitleDelay(value.toDoubleOrNull() ?: 0.0)
|
||||
// mpv semantics mirrored on the libass overlay: anchor non-positioned ASS
|
||||
// events to the visible screen (Dart sets 'yes' for cover mode / zoom > 1)
|
||||
"sub-ass-force-margins" -> playerCore?.setAssForceMargins(value == "yes")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,8 +7,8 @@ import androidx.media3.extractor.SeekMap
|
||||
import androidx.media3.extractor.TrackOutput
|
||||
import androidx.media3.extractor.mkv.MatroskaExtractor
|
||||
import androidx.media3.extractor.text.SubtitleParser
|
||||
import io.github.peerless2012.ass.media.AssHandler
|
||||
import io.github.peerless2012.ass.media.extractor.AssMatroskaExtractor
|
||||
import com.edde746.plezy.libass.media.AssHandler
|
||||
import com.edde746.plezy.libass.media.extractor.AssMatroskaExtractor
|
||||
|
||||
/**
|
||||
* Extends AssMatroskaExtractor to add support for MKV ContentCompAlgo 0 (zlib).
|
||||
|
||||
@@ -2,6 +2,28 @@ allprojects {
|
||||
repositories {
|
||||
google()
|
||||
mavenCentral()
|
||||
if (providers.gradleProperty("localAssCore").isPresent) {
|
||||
// Opt-in A/B of a locally built libass native core: build it in the
|
||||
// libass-android fork with `./gradlew :lib_ass:publishToMavenLocal
|
||||
// -PVERSION_NAME=0.4.0-local`, then build this app with -PlocalAssCore.
|
||||
mavenLocal()
|
||||
} else {
|
||||
// Production libass native core: the -O3/NEON/asm AAR published on
|
||||
// the edde746/libass-android fork's releases (the upstream Maven
|
||||
// artifact io.github.peerless2012:ass ships un-optimized natives —
|
||||
// see the fork's pinned libass-cmake fix). Resolved as
|
||||
// <tag>/ass-<tag>.aar with no metadata probing.
|
||||
exclusiveContent {
|
||||
forRepository {
|
||||
ivy {
|
||||
url = uri("https://github.com/edde746/libass-android/releases/download")
|
||||
patternLayout { artifact("[revision]/[artifact]-[revision].[ext]") }
|
||||
metadataSources { artifact() }
|
||||
}
|
||||
}
|
||||
filter { includeModule("io.github.peerless2012", "ass") }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
// libass ASS subtitle rendering: Kotlin/JNI bindings + Media3 integration
|
||||
// (extractor, parsers, AssHandler, GL atlas overlay). The native libass core
|
||||
// (libass.so + prefab headers) comes from the Maven artifact
|
||||
// io.github.peerless2012:ass; this module compiles its JNI against it.
|
||||
plugins {
|
||||
id("com.android.library")
|
||||
id("org.jetbrains.kotlin.android")
|
||||
}
|
||||
|
||||
android {
|
||||
namespace = "com.edde746.plezy.libass"
|
||||
compileSdk = 36
|
||||
ndkVersion = "28.2.13676358" // matches flutter.ndkVersion so only one NDK is provisioned
|
||||
|
||||
defaultConfig {
|
||||
minSdk = 21
|
||||
consumerProguardFiles("consumer-rules.pro")
|
||||
if (project.hasProperty("localAssCore")) {
|
||||
// The locally published A/B core only ships device ABIs (x86 would need
|
||||
// nasm on the host); match it so prefab resolution doesn't fail.
|
||||
ndk {
|
||||
abiFilters += listOf("armeabi-v7a", "arm64-v8a")
|
||||
}
|
||||
}
|
||||
externalNativeBuild {
|
||||
cmake {
|
||||
// libass.so in the prefab AAR is built against c++_shared (abi.json: stl=c++_shared);
|
||||
// prefab validates consumer STL compatibility.
|
||||
arguments += listOf("-DANDROID_STL=c++_shared")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
buildFeatures {
|
||||
prefab = true
|
||||
}
|
||||
|
||||
packaging {
|
||||
jniLibs {
|
||||
// libass.so is a prefab IMPORTED target (linked, not owned) — the app packages
|
||||
// it from the io.github.peerless2012:ass AAR; don't duplicate it here.
|
||||
excludes.add("**/libass.so")
|
||||
}
|
||||
}
|
||||
|
||||
compileOptions {
|
||||
sourceCompatibility = JavaVersion.VERSION_11
|
||||
targetCompatibility = JavaVersion.VERSION_11
|
||||
}
|
||||
|
||||
kotlinOptions {
|
||||
jvmTarget = JavaVersion.VERSION_11.toString()
|
||||
}
|
||||
|
||||
externalNativeBuild {
|
||||
cmake {
|
||||
path = file("src/main/cpp/CMakeLists.txt")
|
||||
version = "3.22.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dependencies {
|
||||
// compileOnly: prefab headers + link-time libass.so come from the AAR; runtime
|
||||
// packaging of libass.so is the app's implementation dependency.
|
||||
// -PlocalAssCore swaps in a mavenLocal()-published core for A/B tests (must
|
||||
// match the app module's version so one libass.so is linked and packaged).
|
||||
val assCoreVersion = if (project.hasProperty("localAssCore")) "0.4.0-local" else "0.4.1-plezy.1"
|
||||
compileOnly("io.github.peerless2012:ass:$assCoreVersion@aar")
|
||||
|
||||
implementation("androidx.annotation:annotation:1.9.1")
|
||||
implementation("androidx.annotation:annotation-experimental:1.5.1")
|
||||
implementation("androidx.media3:media3-exoplayer:1.9.2")
|
||||
implementation("androidx.media3:media3-ui:1.9.2")
|
||||
|
||||
testImplementation("junit:junit:4.13.2")
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
# Constructed from JNI via FindClass/NewObject (AssKt.c).
|
||||
-keep class com.edde746.plezy.libass.AssAtlasFrame { *; }
|
||||
# JNI exports bind by name (Java_com_edde746_plezy_libass_*); keep the names stable.
|
||||
-keepclasseswithmembernames class com.edde746.plezy.libass.* {
|
||||
native <methods>;
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
|
||||
|
||||
</manifest>
|
||||
@@ -0,0 +1,436 @@
|
||||
// JNI bindings for libass. Exports use standard Java_<package>_<Class>_<method>
|
||||
// naming so no RegisterNatives/JNI_OnLoad registration is needed.
|
||||
#include <android/log.h>
|
||||
#include <jni.h>
|
||||
#include <stdarg.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
|
||||
static inline long long nowMs(void) {
|
||||
struct timespec ts;
|
||||
clock_gettime(CLOCK_MONOTONIC, &ts);
|
||||
return (long long)ts.tv_sec * 1000 + ts.tv_nsec / 1000000;
|
||||
}
|
||||
|
||||
#include "ass/ass.h"
|
||||
|
||||
#define LOG_TAG "SubtitleRenderer"
|
||||
|
||||
static void assMessageCallback(int level, const char* fmt, va_list args, void* data) {
|
||||
if (level > 4) return;
|
||||
|
||||
if (level >= 2) {
|
||||
__android_log_vprint(ANDROID_LOG_WARN, LOG_TAG, fmt, args);
|
||||
} else {
|
||||
__android_log_vprint(ANDROID_LOG_ERROR, LOG_TAG, fmt, args);
|
||||
}
|
||||
}
|
||||
|
||||
// --- Ass (library) ---
|
||||
|
||||
JNIEXPORT jlong JNICALL Java_com_edde746_plezy_libass_Ass_nativeAssInit(JNIEnv* env, jclass clazz) {
|
||||
ASS_Library* assLibrary = ass_library_init();
|
||||
ass_set_message_cb(assLibrary, assMessageCallback, NULL);
|
||||
ass_set_extract_fonts(assLibrary, 1);
|
||||
return (jlong)assLibrary;
|
||||
}
|
||||
|
||||
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_Ass_nativeAssAddFont(
|
||||
JNIEnv* env, jclass clazz, jlong ass, jstring name, jbyteArray byteArray) {
|
||||
jsize length = (*env)->GetArrayLength(env, byteArray);
|
||||
jbyte* bytePtr = (*env)->GetByteArrayElements(env, byteArray, NULL);
|
||||
if (bytePtr == NULL) {
|
||||
return;
|
||||
}
|
||||
const char* cName = (*env)->GetStringUTFChars(env, name, NULL);
|
||||
ass_add_font(((ASS_Library*)ass), cName, (char*)bytePtr, length);
|
||||
(*env)->ReleaseByteArrayElements(env, byteArray, bytePtr, 0);
|
||||
if (cName != NULL) {
|
||||
(*env)->ReleaseStringUTFChars(env, name, cName);
|
||||
}
|
||||
}
|
||||
|
||||
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_Ass_nativeAssDeinit(JNIEnv* env, jclass clazz, jlong ass) {
|
||||
if (ass) {
|
||||
ass_library_done((ASS_Library*)ass);
|
||||
}
|
||||
}
|
||||
|
||||
// --- AssTrack ---
|
||||
|
||||
JNIEXPORT jlong JNICALL
|
||||
Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackInit(JNIEnv* env, jclass clazz, jlong ass) {
|
||||
return (jlong)ass_new_track((ASS_Library*)ass);
|
||||
}
|
||||
|
||||
// Shared body of readBuffer/readChunk: pins the byte array and feeds libass.
|
||||
// chunked != 0 routes to ass_process_chunk (timed dialogue), else ass_process_data.
|
||||
static void processTrackBytes(
|
||||
JNIEnv* env, jlong track, jbyteArray buffer, jint offset, jint length, jlong start, jlong duration, int chunked) {
|
||||
if (!track) return;
|
||||
jbyte* elements = (*env)->GetByteArrayElements(env, buffer, NULL);
|
||||
if (elements == NULL) {
|
||||
return;
|
||||
}
|
||||
if (chunked) {
|
||||
ass_process_chunk((ASS_Track*)track, (char*)(elements + offset), length, start, duration);
|
||||
} else {
|
||||
ass_process_data((ASS_Track*)track, (char*)(elements + offset), length);
|
||||
}
|
||||
(*env)->ReleaseByteArrayElements(env, buffer, elements, 0);
|
||||
}
|
||||
|
||||
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackReadBuffer(
|
||||
JNIEnv* env, jclass clazz, jlong track, jbyteArray buffer, jint offset, jint length) {
|
||||
processTrackBytes(env, track, buffer, offset, length, 0, 0, 0);
|
||||
}
|
||||
|
||||
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackReadChunk(
|
||||
JNIEnv* env, jclass clazz, jlong track, jlong start, jlong duration, jbyteArray buffer, jint offset, jint length) {
|
||||
processTrackBytes(env, track, buffer, offset, length, start, duration, 1);
|
||||
}
|
||||
|
||||
JNIEXPORT void JNICALL
|
||||
Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackDeinit(JNIEnv* env, jclass clazz, jlong track) {
|
||||
if (!track) return;
|
||||
ass_free_track((ASS_Track*)track);
|
||||
}
|
||||
|
||||
// Earliest event Start strictly after afterMs, or -1. Lets the render pipeline
|
||||
// pre-render (cache-warm) the next upcoming event during idle stretches so
|
||||
// heavy typesetting doesn't pay its cache-cold rasterization at appearance.
|
||||
JNIEXPORT jlong JNICALL Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackNextEventStart(
|
||||
JNIEnv* env, jclass clazz, jlong track, jlong afterMs) {
|
||||
if (!track) return -1;
|
||||
ASS_Track* t = (ASS_Track*)track;
|
||||
long long best = -1;
|
||||
for (int i = 0; i < t->n_events; i++) {
|
||||
const long long start = t->events[i].Start;
|
||||
if (start > afterMs && (best < 0 || start < best)) best = start;
|
||||
}
|
||||
return (jlong)best;
|
||||
}
|
||||
|
||||
// Earliest visible-content boundary (event Start OR End) strictly after afterMs,
|
||||
// or -1. A cache-warming prefetch is only invisible while no boundary passes:
|
||||
// the render pipeline uses this to ensure nothing on screen is due to change
|
||||
// before the event it is about to warm.
|
||||
JNIEXPORT jlong JNICALL Java_com_edde746_plezy_libass_AssTrack_nativeAssTrackNextEventChange(
|
||||
JNIEnv* env, jclass clazz, jlong track, jlong afterMs) {
|
||||
if (!track) return -1;
|
||||
ASS_Track* t = (ASS_Track*)track;
|
||||
long long best = -1;
|
||||
for (int i = 0; i < t->n_events; i++) {
|
||||
const long long start = t->events[i].Start;
|
||||
const long long end = start + t->events[i].Duration;
|
||||
if (start > afterMs && (best < 0 || start < best)) best = start;
|
||||
if (end > afterMs && (best < 0 || end < best)) best = end;
|
||||
}
|
||||
return (jlong)best;
|
||||
}
|
||||
|
||||
// --- AssRender ---
|
||||
|
||||
JNIEXPORT jlong JNICALL
|
||||
Java_com_edde746_plezy_libass_AssRender_nativeAssRenderInit(JNIEnv* env, jclass clazz, jlong ass) {
|
||||
ASS_Renderer* assRenderer = ass_renderer_init((ASS_Library*)ass);
|
||||
ass_set_fonts(assRenderer, NULL, "sans-serif", ASS_FONTPROVIDER_FONTCONFIG, NULL, 1);
|
||||
return (jlong)assRenderer;
|
||||
}
|
||||
|
||||
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderSetFontScale(
|
||||
JNIEnv* env, jclass clazz, jlong render, jfloat scale) {
|
||||
if (!render) return;
|
||||
ass_set_font_scale((ASS_Renderer*)render, scale);
|
||||
}
|
||||
|
||||
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderSetCacheLimit(
|
||||
JNIEnv* env, jclass clazz, jlong render, jint glyphMax, jint bitmapMaxSize) {
|
||||
if (!render) return;
|
||||
ass_set_cache_limits((ASS_Renderer*)render, glyphMax, bitmapMaxSize);
|
||||
}
|
||||
|
||||
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderSetFrameSize(
|
||||
JNIEnv* env, jclass clazz, jlong render, jint width, jint height) {
|
||||
if (!render) return;
|
||||
ass_set_frame_size((ASS_Renderer*)render, width, height);
|
||||
}
|
||||
|
||||
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderSetStorageSize(
|
||||
JNIEnv* env, jclass clazz, jlong render, jint width, jint height) {
|
||||
if (!render) return;
|
||||
ass_set_storage_size((ASS_Renderer*)render, width, height);
|
||||
}
|
||||
|
||||
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderSetMargins(
|
||||
JNIEnv* env, jclass clazz, jlong render, jint top, jint bottom, jint left, jint right) {
|
||||
if (!render) return;
|
||||
ass_set_margins((ASS_Renderer*)render, top, bottom, left, right);
|
||||
}
|
||||
|
||||
JNIEXPORT void JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderSetUseMargins(
|
||||
JNIEnv* env, jclass clazz, jlong render, jboolean use) {
|
||||
if (!render) return;
|
||||
ass_set_use_margins((ASS_Renderer*)render, use ? 1 : 0);
|
||||
}
|
||||
|
||||
JNIEXPORT void JNICALL
|
||||
Java_com_edde746_plezy_libass_AssRender_nativeAssRenderDeinit(JNIEnv* env, jclass clazz, jlong render) {
|
||||
if (render) {
|
||||
ass_renderer_done((ASS_Renderer*)render);
|
||||
}
|
||||
}
|
||||
|
||||
// (image, original-list index) pair so packing can run in height-sorted order
|
||||
// while slots stay keyed by list position (= blend order) for pass 2.
|
||||
typedef struct {
|
||||
ASS_Image* img;
|
||||
int idx;
|
||||
} PackItem;
|
||||
|
||||
static int comparePackItemsByHeightDesc(const void* a, const void* b) {
|
||||
const PackItem* ia = (const PackItem*)a;
|
||||
const PackItem* ib = (const PackItem*)b;
|
||||
return ib->img->h - ia->img->h;
|
||||
}
|
||||
|
||||
// Throttle for truncation warnings (shared across renderers; logging only).
|
||||
static int truncationLogCounter = 0;
|
||||
|
||||
// Renders a frame into the provided atlas + vertex direct ByteBuffers.
|
||||
//
|
||||
// - atlasBuf holds packed ALPHA_8 pixels with row stride atlasMaxW. Only the first
|
||||
// atlasHeight rows are written; the caller uploads that region to a texture
|
||||
// allocated once at atlasMaxW × atlasMaxH.
|
||||
// - vertexBuf holds a per-quad vertex stream (6 vertices × (2 pos + 2 uv + 4 color)
|
||||
// floats = 48 floats = 192 bytes per quad). Must match BYTES_PER_QUAD/VERTEX in
|
||||
// AssSubtitleAtlasPipeline.kt. Ready for a single glDrawArrays(GL_TRIANGLES, 0, N * 6).
|
||||
// - UVs are normalized against atlasMaxW × atlasMaxH (the allocated texture dims),
|
||||
// not the packed region, so the texture never needs reallocation.
|
||||
// - Images are packed in height-sorted rows (minimizes packed height) but vertices
|
||||
// are emitted in original list order — the list order is libass's painter order.
|
||||
//
|
||||
// Never fails on content size: images that don't fit the remaining atlas/vertex
|
||||
// capacity are dropped and counted in AssAtlasFrame.truncated, so a heavy frame
|
||||
// degrades instead of going stale. Returns NULL only for missing buffers/handles.
|
||||
// On changed == 0, returns (0, 0, 0, changed, 0) without touching the buffers —
|
||||
// caller reuses the atlas texture already on the GPU.
|
||||
JNIEXPORT jobject JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderFrameAtlas(
|
||||
JNIEnv* env, jclass clazz, jlong render, jlong track, jlong time, jobject atlasBuf, jint atlasMaxW,
|
||||
jint atlasMaxH, jobject vertexBuf) {
|
||||
if (!render || !track || !atlasBuf || !vertexBuf || atlasMaxW <= 0 || atlasMaxH <= 0) return NULL;
|
||||
|
||||
jclass atlasFrameClass = (*env)->FindClass(env, "com/edde746/plezy/libass/AssAtlasFrame");
|
||||
if (!atlasFrameClass) return NULL;
|
||||
jmethodID ctor = (*env)->GetMethodID(env, atlasFrameClass, "<init>", "(IIIII)V");
|
||||
if (!ctor) return NULL;
|
||||
|
||||
const long long t0 = nowMs();
|
||||
int changed;
|
||||
ASS_Image* image = ass_render_frame((ASS_Renderer*)render, (ASS_Track*)track, time, &changed);
|
||||
const long long tAss = nowMs();
|
||||
|
||||
if (changed == 0 || image == NULL) {
|
||||
if (tAss - t0 > 40) {
|
||||
__android_log_print(
|
||||
ANDROID_LOG_WARN, LOG_TAG, "slow render t=%lldms: ass=%lldms (changed=%d, no output)", (long long)time,
|
||||
tAss - t0, changed);
|
||||
}
|
||||
return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, 0);
|
||||
}
|
||||
|
||||
uint8_t* atlasPixels = (uint8_t*)(*env)->GetDirectBufferAddress(env, atlasBuf);
|
||||
jlong atlasCap = (*env)->GetDirectBufferCapacity(env, atlasBuf);
|
||||
float* vertices = (float*)(*env)->GetDirectBufferAddress(env, vertexBuf);
|
||||
jlong vertexCap = (*env)->GetDirectBufferCapacity(env, vertexBuf);
|
||||
if (!atlasPixels || !vertices) return NULL;
|
||||
if ((jlong)atlasMaxW * atlasMaxH > atlasCap) {
|
||||
__android_log_print(
|
||||
ANDROID_LOG_ERROR, LOG_TAG, "atlas buffer smaller than %dx%d (capacity %lld bytes)", atlasMaxW, atlasMaxH,
|
||||
(long long)atlasCap);
|
||||
return NULL;
|
||||
}
|
||||
// 48 floats per quad × 4 bytes = 192 bytes/quad
|
||||
const int maxQuads = (int)(vertexCap / 192);
|
||||
|
||||
int total = 0;
|
||||
for (ASS_Image* img = image; img != NULL; img = img->next) {
|
||||
if (img->w > 0 && img->h > 0) total++;
|
||||
}
|
||||
if (total == 0) {
|
||||
return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, 0);
|
||||
}
|
||||
|
||||
// Pass 1: assign packing slots in height-sorted order so mixed-size frames pack
|
||||
// tight rows. slotX/slotY are keyed by the image's position in the original
|
||||
// list; -1 marks images dropped for capacity.
|
||||
PackItem* items = (PackItem*)malloc(sizeof(PackItem) * (size_t)total);
|
||||
int* slotX = (int*)malloc(sizeof(int) * (size_t)total);
|
||||
int* slotY = (int*)malloc(sizeof(int) * (size_t)total);
|
||||
if (!items || !slotX || !slotY) {
|
||||
free(items);
|
||||
free(slotX);
|
||||
free(slotY);
|
||||
return NULL;
|
||||
}
|
||||
int n = 0;
|
||||
for (ASS_Image* img = image; img != NULL; img = img->next) {
|
||||
if (img->w > 0 && img->h > 0) {
|
||||
items[n].img = img;
|
||||
items[n].idx = n;
|
||||
n++;
|
||||
}
|
||||
}
|
||||
qsort(items, (size_t)n, sizeof(PackItem), comparePackItemsByHeightDesc);
|
||||
|
||||
int cursorX = 0, cursorY = 0, rowH = 0;
|
||||
int truncated = 0;
|
||||
int packedH = 0;
|
||||
int accepted = 0;
|
||||
long long srcPixels = 0;
|
||||
for (int i = 0; i < n; i++) {
|
||||
ASS_Image* img = items[i].img;
|
||||
srcPixels += (long long)img->w * img->h;
|
||||
int sx = -1, sy = -1;
|
||||
if (img->w <= atlasMaxW && accepted < maxQuads) {
|
||||
int cx = cursorX, cy = cursorY, rh = rowH;
|
||||
if (cx + img->w > atlasMaxW) {
|
||||
cy += rh;
|
||||
cx = 0;
|
||||
rh = 0;
|
||||
}
|
||||
if (cy + img->h <= atlasMaxH) {
|
||||
sx = cx;
|
||||
sy = cy;
|
||||
cursorX = cx + img->w;
|
||||
cursorY = cy;
|
||||
rowH = (img->h > rh) ? img->h : rh;
|
||||
if (cy + img->h > packedH) packedH = cy + img->h;
|
||||
accepted++;
|
||||
}
|
||||
}
|
||||
if (sx < 0) truncated++;
|
||||
slotX[items[i].idx] = sx;
|
||||
slotY[items[i].idx] = sy;
|
||||
}
|
||||
|
||||
if (truncated > 0 && (truncationLogCounter++ & 63) == 0) {
|
||||
__android_log_print(
|
||||
ANDROID_LOG_WARN, LOG_TAG, "atlas truncation: %d of %d images dropped (atlas %dx%d, %d quads max)",
|
||||
truncated, n, atlasMaxW, atlasMaxH, maxQuads);
|
||||
}
|
||||
if (accepted == 0) {
|
||||
free(items);
|
||||
free(slotX);
|
||||
free(slotY);
|
||||
return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, truncated);
|
||||
}
|
||||
|
||||
memset(atlasPixels, 0, (size_t)atlasMaxW * packedH);
|
||||
|
||||
// Pass 2: walk the original list (= libass's painter/blend order), copying each
|
||||
// accepted image into its assigned slot and emitting its quad.
|
||||
int qi = 0;
|
||||
int k = 0;
|
||||
for (ASS_Image* img = image; img != NULL; img = img->next) {
|
||||
if (img->w <= 0 || img->h <= 0) continue;
|
||||
const int px = slotX[k];
|
||||
const int py = slotY[k];
|
||||
k++;
|
||||
if (px < 0) continue;
|
||||
|
||||
for (int y = 0; y < img->h; y++) {
|
||||
uint8_t* dst = atlasPixels + (size_t)(py + y) * atlasMaxW + px;
|
||||
const uint8_t* src = img->bitmap + (size_t)y * img->stride;
|
||||
memcpy(dst, src, (size_t)img->w);
|
||||
}
|
||||
|
||||
const float x0 = (float)img->dst_x;
|
||||
const float y0 = (float)img->dst_y;
|
||||
const float x1 = x0 + (float)img->w;
|
||||
const float y1 = y0 + (float)img->h;
|
||||
const float u0 = (float)px / (float)atlasMaxW;
|
||||
const float v0 = (float)py / (float)atlasMaxH;
|
||||
const float u1 = (float)(px + img->w) / (float)atlasMaxW;
|
||||
const float v1 = (float)(py + img->h) / (float)atlasMaxH;
|
||||
const unsigned int c = img->color;
|
||||
const float r = (float)((c >> 24) & 0xFFu) / 255.0f;
|
||||
const float g = (float)((c >> 16) & 0xFFu) / 255.0f;
|
||||
const float b = (float)((c >> 8) & 0xFFu) / 255.0f;
|
||||
const float a = (float)(0xFFu - (c & 0xFFu)) / 255.0f;
|
||||
|
||||
float* vx = vertices + (size_t)qi * 48;
|
||||
// 8 floats per vertex: x, y, u, v, r, g, b, a.
|
||||
// Triangle 1: (x0,y0) (x1,y0) (x0,y1)
|
||||
vx[0] = x0;
|
||||
vx[1] = y0;
|
||||
vx[2] = u0;
|
||||
vx[3] = v0;
|
||||
vx[4] = r;
|
||||
vx[5] = g;
|
||||
vx[6] = b;
|
||||
vx[7] = a;
|
||||
vx[8] = x1;
|
||||
vx[9] = y0;
|
||||
vx[10] = u1;
|
||||
vx[11] = v0;
|
||||
vx[12] = r;
|
||||
vx[13] = g;
|
||||
vx[14] = b;
|
||||
vx[15] = a;
|
||||
vx[16] = x0;
|
||||
vx[17] = y1;
|
||||
vx[18] = u0;
|
||||
vx[19] = v1;
|
||||
vx[20] = r;
|
||||
vx[21] = g;
|
||||
vx[22] = b;
|
||||
vx[23] = a;
|
||||
// Triangle 2: (x1,y0) (x1,y1) (x0,y1)
|
||||
vx[24] = x1;
|
||||
vx[25] = y0;
|
||||
vx[26] = u1;
|
||||
vx[27] = v0;
|
||||
vx[28] = r;
|
||||
vx[29] = g;
|
||||
vx[30] = b;
|
||||
vx[31] = a;
|
||||
vx[32] = x1;
|
||||
vx[33] = y1;
|
||||
vx[34] = u1;
|
||||
vx[35] = v1;
|
||||
vx[36] = r;
|
||||
vx[37] = g;
|
||||
vx[38] = b;
|
||||
vx[39] = a;
|
||||
vx[40] = x0;
|
||||
vx[41] = y1;
|
||||
vx[42] = u0;
|
||||
vx[43] = v1;
|
||||
vx[44] = r;
|
||||
vx[45] = g;
|
||||
vx[46] = b;
|
||||
vx[47] = a;
|
||||
|
||||
qi++;
|
||||
}
|
||||
|
||||
free(items);
|
||||
free(slotX);
|
||||
free(slotY);
|
||||
|
||||
// Slow-render breakdown: separates libass's own cost (rasterize/blur/shape)
|
||||
// from this function's packing + memcpy, so device logs attribute the time.
|
||||
const long long tEnd = nowMs();
|
||||
if (tEnd - t0 > 40) {
|
||||
__android_log_print(
|
||||
ANDROID_LOG_WARN, LOG_TAG,
|
||||
"slow render t=%lldms: total=%lldms ass=%lldms pack+copy=%lldms images=%d srcPx=%lldk atlas=%dx%d quads=%d",
|
||||
(long long)time, tEnd - t0, tAss - t0, tEnd - tAss, n, srcPixels / 1000, atlasMaxW, packedH, qi);
|
||||
}
|
||||
|
||||
// atlasWidth is the full row stride (GLES2 can't upload with stride ≠ width);
|
||||
// atlasHeight is the packed height — the only rows worth uploading.
|
||||
return (*env)->NewObject(env, atlasFrameClass, ctor, atlasMaxW, packedH, qi, changed, truncated);
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
cmake_minimum_required(VERSION 3.22.1)
|
||||
|
||||
project("asskt")
|
||||
|
||||
# libass headers + libass.so come from the io.github.peerless2012:ass AAR via prefab.
|
||||
add_library(${CMAKE_PROJECT_NAME} SHARED AssKt.c)
|
||||
find_package(lib_ass REQUIRED CONFIG)
|
||||
target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE
|
||||
lib_ass::ass
|
||||
log)
|
||||
@@ -0,0 +1,62 @@
|
||||
package com.edde746.plezy.libass
|
||||
|
||||
import java.util.concurrent.locks.ReentrantLock
|
||||
import kotlin.concurrent.withLock
|
||||
|
||||
class Ass {
|
||||
|
||||
companion object {
|
||||
|
||||
init {
|
||||
System.loadLibrary("asskt")
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
external fun nativeAssInit(): Long
|
||||
|
||||
@JvmStatic
|
||||
external fun nativeAssAddFont(ptr: Long, name: String, buffer: ByteArray)
|
||||
|
||||
@JvmStatic
|
||||
external fun nativeAssDeinit(ptr: Long)
|
||||
}
|
||||
|
||||
/** Single lock for all libass calls on this library instance. */
|
||||
val lock = ReentrantLock()
|
||||
|
||||
private var nativeAss: Long = nativeAssInit()
|
||||
|
||||
@Volatile
|
||||
var released = false
|
||||
private set
|
||||
|
||||
private fun <T> create(block: (Long) -> T): T = lock.withLock {
|
||||
check(!released && nativeAss != 0L) { "Ass already released" }
|
||||
block(nativeAss)
|
||||
}
|
||||
|
||||
fun createTrack(): AssTrack = create { AssTrack(it, lock) }
|
||||
|
||||
fun createRender(): AssRender = create { AssRender(it, lock) }
|
||||
|
||||
fun addFont(name: String, buffer: ByteArray) {
|
||||
lock.withLock {
|
||||
if (!released && nativeAss != 0L) nativeAssAddFont(nativeAss, name, buffer)
|
||||
}
|
||||
}
|
||||
|
||||
fun release() {
|
||||
lock.withLock {
|
||||
if (released) return
|
||||
released = true
|
||||
if (nativeAss != 0L) {
|
||||
nativeAssDeinit(nativeAss)
|
||||
nativeAss = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected fun finalize() {
|
||||
release()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.edde746.plezy.libass
|
||||
|
||||
/**
|
||||
* Result of a packed-atlas render. The atlas pixel data is stored in the direct ByteBuffer
|
||||
* that was passed into [AssRender.renderFrameAtlas]; the vertex stream is in the other.
|
||||
*
|
||||
* @param atlasWidth atlas row stride in pixels (= the allocated width; 0 when [changed] == 0)
|
||||
* @param atlasHeight packed atlas height in pixels — the rows worth uploading
|
||||
* @param quadCount number of quads; the vertex buffer holds [quadCount] * 6 vertices
|
||||
* @param changed libass change flag (0 = no change, 1 = positions, 2 = content)
|
||||
* @param truncated images dropped because they exceeded the atlas/vertex capacity;
|
||||
* the frame is incomplete but never stale (> 0 should be rare —
|
||||
* it means even the GL-max-sized atlas couldn't fit the frame)
|
||||
*/
|
||||
class AssAtlasFrame(
|
||||
val atlasWidth: Int,
|
||||
val atlasHeight: Int,
|
||||
val quadCount: Int,
|
||||
val changed: Int,
|
||||
val truncated: Int
|
||||
)
|
||||
@@ -0,0 +1,164 @@
|
||||
package com.edde746.plezy.libass
|
||||
|
||||
import java.nio.ByteBuffer
|
||||
import java.util.concurrent.locks.ReentrantLock
|
||||
import kotlin.concurrent.withLock
|
||||
|
||||
class AssRender(nativeAss: Long, private val lock: ReentrantLock) {
|
||||
|
||||
companion object {
|
||||
|
||||
@JvmStatic
|
||||
external fun nativeAssRenderInit(ass: Long): Long
|
||||
|
||||
@JvmStatic
|
||||
external fun nativeAssRenderSetFontScale(render: Long, scale: Float)
|
||||
|
||||
@JvmStatic
|
||||
external fun nativeAssRenderSetCacheLimit(render: Long, glyphMax: Int, bitmapMaxSize: Int)
|
||||
|
||||
@JvmStatic
|
||||
external fun nativeAssRenderSetStorageSize(render: Long, width: Int, height: Int)
|
||||
|
||||
@JvmStatic
|
||||
external fun nativeAssRenderSetFrameSize(render: Long, width: Int, height: Int)
|
||||
|
||||
@JvmStatic
|
||||
external fun nativeAssRenderSetMargins(render: Long, top: Int, bottom: Int, left: Int, right: Int)
|
||||
|
||||
@JvmStatic
|
||||
external fun nativeAssRenderSetUseMargins(render: Long, use: Boolean)
|
||||
|
||||
@JvmStatic
|
||||
external fun nativeAssRenderFrameAtlas(
|
||||
render: Long,
|
||||
track: Long,
|
||||
time: Long,
|
||||
atlasBuf: ByteBuffer,
|
||||
atlasMaxWidth: Int,
|
||||
atlasMaxHeight: Int,
|
||||
vertexBuf: ByteBuffer
|
||||
): AssAtlasFrame?
|
||||
|
||||
@JvmStatic
|
||||
external fun nativeAssRenderDeinit(render: Long)
|
||||
}
|
||||
|
||||
private var nativeRender: Long = nativeAssRenderInit(nativeAss)
|
||||
|
||||
@Volatile
|
||||
var released = false
|
||||
private set
|
||||
|
||||
private var track: AssTrack? = null
|
||||
|
||||
/**
|
||||
* Bumped on every renderer-state mutation (track, sizes, margins, font scale).
|
||||
* Lets the render-ahead pipeline detect that a speculatively rendered frame was
|
||||
* produced against stale state and must not be presented.
|
||||
*/
|
||||
private val generation = java.util.concurrent.atomic.AtomicInteger(0)
|
||||
|
||||
/** Current renderer-state generation; see [generation]. */
|
||||
val stateGeneration: Int get() = generation.get()
|
||||
|
||||
/** Runs [block] with the native handle under the shared libass lock; no-op once released. */
|
||||
private inline fun withNative(block: (Long) -> Unit) {
|
||||
lock.withLock {
|
||||
if (!released && nativeRender != 0L) block(nativeRender)
|
||||
}
|
||||
}
|
||||
|
||||
fun setTrack(track: AssTrack?) {
|
||||
generation.incrementAndGet()
|
||||
lock.withLock { this.track = track }
|
||||
}
|
||||
|
||||
fun setFontScale(scale: Float) {
|
||||
generation.incrementAndGet()
|
||||
withNative { nativeAssRenderSetFontScale(it, scale) }
|
||||
}
|
||||
|
||||
fun setCacheLimit(glyphMax: Int, bitmapMaxSize: Int) = withNative { nativeAssRenderSetCacheLimit(it, glyphMax, bitmapMaxSize) }
|
||||
|
||||
fun setStorageSize(width: Int, height: Int) {
|
||||
generation.incrementAndGet()
|
||||
withNative { nativeAssRenderSetStorageSize(it, width, height) }
|
||||
}
|
||||
|
||||
fun setFrameSize(width: Int, height: Int) {
|
||||
generation.incrementAndGet()
|
||||
withNative { nativeAssRenderSetFrameSize(it, width, height) }
|
||||
}
|
||||
|
||||
/**
|
||||
* mpv-style frame margins: offsets of the video dst rect within the frame set by
|
||||
* [setFrameSize]. Negative when the video extends beyond the frame (zoomed in / cover).
|
||||
*/
|
||||
fun setMargins(top: Int, bottom: Int, left: Int, right: Int) {
|
||||
generation.incrementAndGet()
|
||||
withNative { nativeAssRenderSetMargins(it, top, bottom, left, right) }
|
||||
}
|
||||
|
||||
/**
|
||||
* mpv's sub-ass-force-margins: lay out non-positioned events against the full frame
|
||||
* (kept on the visible screen) instead of the video rect between the margins.
|
||||
*/
|
||||
fun setUseMargins(use: Boolean) {
|
||||
generation.incrementAndGet()
|
||||
withNative { nativeAssRenderSetUseMargins(it, use) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders a frame into a packed ALPHA_8 texture atlas plus a single vertex stream
|
||||
* ready for `glDrawArrays(GL_TRIANGLES, 0, quadCount * 6)`.
|
||||
*
|
||||
* UVs are normalized against ([atlasMaxW], [atlasMaxH]) — the allocated texture
|
||||
* dims — so the caller can allocate the texture once and `glTexSubImage2D` only
|
||||
* the packed rows. Images that exceed the capacity are dropped and counted in
|
||||
* [AssAtlasFrame.truncated]; the render never fails on content size.
|
||||
*
|
||||
* @param atlasBuf direct ByteBuffer receiving the packed pixels (≥ atlasMaxW × atlasMaxH)
|
||||
* @param atlasMaxW atlas row stride in pixels (bound by `GL_MAX_TEXTURE_SIZE`)
|
||||
* @param atlasMaxH atlas height in pixels (bound by `GL_MAX_TEXTURE_SIZE`)
|
||||
* @param vertexBuf direct ByteBuffer receiving the vertex stream (192 bytes per quad)
|
||||
*/
|
||||
/** How long the most recent [renderFrameAtlas] waited to acquire the shared
|
||||
* libass lock (contended by track dialogue/font feeding), in milliseconds. */
|
||||
@Volatile
|
||||
var lastLockWaitMs: Long = 0
|
||||
private set
|
||||
|
||||
fun renderFrameAtlas(
|
||||
time: Long,
|
||||
atlasBuf: ByteBuffer,
|
||||
atlasMaxW: Int,
|
||||
atlasMaxH: Int,
|
||||
vertexBuf: ByteBuffer
|
||||
): AssAtlasFrame? {
|
||||
val tQueue = System.nanoTime()
|
||||
lock.withLock {
|
||||
lastLockWaitMs = (System.nanoTime() - tQueue) / 1_000_000
|
||||
if (released || nativeRender == 0L) return null
|
||||
val t = track ?: return null
|
||||
if (t.released || t.nativeAssTrack == 0L) return null
|
||||
return nativeAssRenderFrameAtlas(nativeRender, t.nativeAssTrack, time, atlasBuf, atlasMaxW, atlasMaxH, vertexBuf)
|
||||
}
|
||||
}
|
||||
|
||||
fun release() {
|
||||
lock.withLock {
|
||||
if (released) return
|
||||
released = true
|
||||
track = null
|
||||
if (nativeRender != 0L) {
|
||||
nativeAssRenderDeinit(nativeRender)
|
||||
nativeRender = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected fun finalize() {
|
||||
release()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package com.edde746.plezy.libass
|
||||
|
||||
import java.util.concurrent.locks.ReentrantLock
|
||||
import kotlin.concurrent.withLock
|
||||
|
||||
class AssTrack(private val ass: Long, private val lock: ReentrantLock) {
|
||||
|
||||
companion object {
|
||||
|
||||
@JvmStatic
|
||||
external fun nativeAssTrackInit(track: Long): Long
|
||||
|
||||
@JvmStatic
|
||||
external fun nativeAssTrackReadBuffer(track: Long, byteArray: ByteArray, offset: Int, length: Int)
|
||||
|
||||
@JvmStatic
|
||||
external fun nativeAssTrackReadChunk(track: Long, start: Long, duration: Long, byteArray: ByteArray, offset: Int, length: Int)
|
||||
|
||||
@JvmStatic
|
||||
external fun nativeAssTrackDeinit(track: Long)
|
||||
|
||||
@JvmStatic
|
||||
external fun nativeAssTrackNextEventStart(track: Long, afterMs: Long): Long
|
||||
|
||||
@JvmStatic
|
||||
external fun nativeAssTrackNextEventChange(track: Long, afterMs: Long): Long
|
||||
}
|
||||
|
||||
var nativeAssTrack = nativeAssTrackInit(ass)
|
||||
private set
|
||||
|
||||
@Volatile
|
||||
var released = false
|
||||
private set
|
||||
|
||||
/** Runs [block] with the native handle under the shared libass lock; no-op once released. */
|
||||
private inline fun withNative(block: (Long) -> Unit) {
|
||||
lock.withLock {
|
||||
if (!released && nativeAssTrack != 0L) block(nativeAssTrack)
|
||||
}
|
||||
}
|
||||
|
||||
fun readBuffer(array: ByteArray, offset: Int = 0, length: Int = array.size) = withNative { nativeAssTrackReadBuffer(it, array, offset, length) }
|
||||
|
||||
fun readChunk(start: Long, duration: Long, array: ByteArray, offset: Int = 0, length: Int = array.size) = withNative { nativeAssTrackReadChunk(it, start, duration, array, offset, length) }
|
||||
|
||||
/** Earliest event start strictly after [afterMs], or -1 if none (yet). */
|
||||
fun nextEventStartMs(afterMs: Long): Long {
|
||||
lock.withLock {
|
||||
if (released || nativeAssTrack == 0L) return -1
|
||||
return nativeAssTrackNextEventStart(nativeAssTrack, afterMs)
|
||||
}
|
||||
}
|
||||
|
||||
/** Earliest event start OR end strictly after [afterMs], or -1 if none (yet). */
|
||||
fun nextEventChangeMs(afterMs: Long): Long {
|
||||
lock.withLock {
|
||||
if (released || nativeAssTrack == 0L) return -1
|
||||
return nativeAssTrackNextEventChange(nativeAssTrack, afterMs)
|
||||
}
|
||||
}
|
||||
|
||||
fun release() {
|
||||
lock.withLock {
|
||||
if (released) return
|
||||
released = true
|
||||
if (nativeAssTrack != 0L) {
|
||||
nativeAssTrackDeinit(nativeAssTrack)
|
||||
nativeAssTrack = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected fun finalize() {
|
||||
release()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,378 @@
|
||||
package com.edde746.plezy.libass.media
|
||||
|
||||
import android.util.Log
|
||||
import androidx.annotation.OptIn
|
||||
import androidx.media3.common.Format
|
||||
import androidx.media3.common.MediaItem
|
||||
import androidx.media3.common.MimeTypes
|
||||
import androidx.media3.common.MimeTypes.TEXT_SSA
|
||||
import androidx.media3.common.Player.Listener
|
||||
import androidx.media3.common.Tracks
|
||||
import androidx.media3.common.VideoSize
|
||||
import androidx.media3.common.util.Size
|
||||
import androidx.media3.common.util.UnstableApi
|
||||
import androidx.media3.exoplayer.ExoPlayer
|
||||
import androidx.media3.exoplayer.video.VideoFrameMetadataListener
|
||||
import com.edde746.plezy.libass.Ass
|
||||
import com.edde746.plezy.libass.AssRender
|
||||
import com.edde746.plezy.libass.AssTrack
|
||||
import com.edde746.plezy.libass.media.parser.AssHeaderParser
|
||||
|
||||
/**
|
||||
* Handles ASS subtitle rendering and integration with ExoPlayer.
|
||||
*
|
||||
* This class listens to ExoPlayer events and manages the creation, selection, and rendering of ASS
|
||||
* subtitle tracks. Rendering always happens on a GL overlay surface (atlas pipeline) — the
|
||||
* libass frame is the overlay surface, not the video.
|
||||
*/
|
||||
@OptIn(UnstableApi::class)
|
||||
class AssHandler(
|
||||
val config: AssHandlerConfig = AssHandlerConfig()
|
||||
) : Listener {
|
||||
|
||||
/** The ASS instance used for creating tracks and renderers. This is lazy to avoid loading
|
||||
* libass if the played media does not have ASS tracks. */
|
||||
private val assDelegate = lazy { Ass() }
|
||||
val ass by assDelegate
|
||||
|
||||
/** The current ASS renderer. It's created as soon as a ASS track is detected. */
|
||||
var render: AssRender? = null
|
||||
private set
|
||||
|
||||
/**
|
||||
* AssRender changed callback
|
||||
*/
|
||||
var renderCallback: ((AssRender?) -> Unit)? = null
|
||||
|
||||
/** The currently selected ASS track. */
|
||||
var track: AssTrack? = null
|
||||
private set
|
||||
|
||||
/** The available ASS tracks in the current media. */
|
||||
private val availableTracks = mutableMapOf<String, AssTrack>()
|
||||
|
||||
/** Fonts encountered before any ASS track was created. Flushed in [createTrack]. */
|
||||
private val pendingFonts = mutableListOf<Pair<String, ByteArray>>()
|
||||
|
||||
/** The size of the video track. */
|
||||
var videoSize = Size.ZERO
|
||||
private set
|
||||
|
||||
/** The size of the surface on which subtitles are rendered. */
|
||||
var surfaceSize = Size.ZERO
|
||||
private set
|
||||
|
||||
/**
|
||||
* True once an overlay widget reported its surface size via [setOverlaySurfaceSize].
|
||||
* From then on the Player.Listener surface size (= the video output surface, which
|
||||
* may differ from the subtitle overlay once the overlay is parented elsewhere) is
|
||||
* ignored as a frame-size source.
|
||||
*/
|
||||
private var overlaySizeFromWidget = false
|
||||
|
||||
/** mpv-style margins of the video rect within the frame: top, bottom, left, right. */
|
||||
private var margins: IntArray? = null
|
||||
|
||||
/** mpv's sub-ass-force-margins: anchor non-positioned events to the visible frame. */
|
||||
private var useMargins = false
|
||||
|
||||
/**
|
||||
* Per-video-frame callback. Fired by ExoPlayer just before MediaCodec releases the frame
|
||||
* to the output surface. Carries the exact PTS of the frame and the System.nanoTime()
|
||||
* domain target release time, so subtitle renderers can align composition to the same
|
||||
* display vsync as the video.
|
||||
*
|
||||
* - [presentationTimeUs] is track-relative microseconds matching the subtitle track's PTS.
|
||||
* - [releaseTimeNs] may be [androidx.media3.common.C.TIME_UNSET] in rare paths.
|
||||
*
|
||||
* Invoked on the playback thread.
|
||||
*/
|
||||
var videoFrameCallback: ((presentationTimeUs: Long, releaseTimeNs: Long) -> Unit)? = null
|
||||
|
||||
private val videoFrameMetadataListener = VideoFrameMetadataListener { pts, releaseNs, _, _ ->
|
||||
videoFrameCallback?.invoke(pts, releaseNs)
|
||||
}
|
||||
|
||||
private var player: ExoPlayer? = null
|
||||
|
||||
/** The current selected ass format. */
|
||||
private var format: Format? = null
|
||||
|
||||
/**
|
||||
* Initializes the handler with the provided ExoPlayer instance.
|
||||
* @param player The ExoPlayer instance to attach to.
|
||||
*/
|
||||
fun init(player: ExoPlayer) {
|
||||
this.player = player
|
||||
player.addListener(this)
|
||||
player.setVideoFrameMetadataListener(videoFrameMetadataListener)
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles transitions between media items in the player and resets everything to the initial
|
||||
* state.
|
||||
*/
|
||||
override fun onMediaItemTransition(mediaItem: MediaItem?, reason: Int) {
|
||||
super.onMediaItemTransition(mediaItem, reason)
|
||||
Log.i("AssHandler", "onMediaItemTransition: item = $mediaItem, reason = $reason")
|
||||
render = null
|
||||
track = null
|
||||
availableTracks.clear()
|
||||
pendingFonts.clear()
|
||||
videoSize = Size.ZERO
|
||||
renderCallback?.invoke(null)
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles changes to the tracks available in the current media.
|
||||
* Configures the selected ASS track if available.
|
||||
* @param tracks The selected tracks.
|
||||
*/
|
||||
override fun onTracksChanged(tracks: Tracks) {
|
||||
Log.i("AssHandler", "onTracksChanged $tracks")
|
||||
|
||||
val selectedVideoTrack = getSelectedVideoTrack(tracks)
|
||||
if (selectedVideoTrack != null) {
|
||||
setVideoSize(selectedVideoTrack.width, selectedVideoTrack.height)
|
||||
}
|
||||
|
||||
format = getSelectedAssTrack(tracks)
|
||||
if (format == null) {
|
||||
Log.i("AssHandler", "subtitle track disabled")
|
||||
track = null
|
||||
render?.setTrack(null)
|
||||
return
|
||||
}
|
||||
|
||||
updateTrack()
|
||||
}
|
||||
|
||||
private fun updateTrack() {
|
||||
val track = availableTracks.firstNotNullOfOrNull {
|
||||
// When media without external subtitles, format id will not change.
|
||||
// When media with external subtitles, format will become like 1:1 .
|
||||
// So to compat both situation, we extract the actual id after the colon.
|
||||
if (format?.id?.substringAfter(":") == it.key) {
|
||||
it.value
|
||||
} else {
|
||||
null
|
||||
}
|
||||
}
|
||||
if (track == null || this.track == track) return
|
||||
|
||||
Log.i("AssHandler", "subtitle track changed to $format")
|
||||
this.track = track
|
||||
val render = requireNotNull(render)
|
||||
applyRenderState(render)
|
||||
render.setTrack(track)
|
||||
}
|
||||
|
||||
/**
|
||||
* (Re)applies sizing and margin state to [render]. Called whenever a render is
|
||||
* (re)created or the selected track changes, so renderer state survives both.
|
||||
*/
|
||||
private fun applyRenderState(render: AssRender) {
|
||||
if (videoSize.isValid) render.setStorageSize(videoSize.width, videoSize.height)
|
||||
when {
|
||||
surfaceSize.isValid -> render.setFrameSize(surfaceSize.width, surfaceSize.height)
|
||||
// Fallback frame until the overlay surface reports its size.
|
||||
videoSize.isValid -> render.setFrameSize(videoSize.width, videoSize.height)
|
||||
}
|
||||
margins?.let { m -> render.setMargins(m[0], m[1], m[2], m[3]) }
|
||||
render.setUseMargins(useMargins)
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles changes to the surface size for video playback.
|
||||
* Notifies the callback if the size has changed.
|
||||
* @param width The new width of the surface.
|
||||
* @param height The new height of the surface.
|
||||
*/
|
||||
override fun onSurfaceSizeChanged(width: Int, height: Int) {
|
||||
super.onSurfaceSizeChanged(width, height)
|
||||
Log.i("AssHandler", "onSurfaceSizeChanged: width = $width, height = $height")
|
||||
// The video output surface is only a frame-size proxy until an overlay widget
|
||||
// reports its own size — they diverge when the overlay isn't video-rect-sized.
|
||||
if (overlaySizeFromWidget) return
|
||||
if (surfaceSize.width == width && surfaceSize.height == height) return
|
||||
surfaceSize = Size(width, height)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports the subtitle overlay widget's actual surface size — the authoritative
|
||||
* libass frame size for OVERLAY render types.
|
||||
*/
|
||||
fun setOverlaySurfaceSize(width: Int, height: Int) {
|
||||
overlaySizeFromWidget = true
|
||||
if (surfaceSize.width == width && surfaceSize.height == height) return
|
||||
Log.i("AssHandler", "setOverlaySurfaceSize: width = $width, height = $height")
|
||||
surfaceSize = Size(width, height)
|
||||
render?.setFrameSize(width, height)
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets mpv-style frame margins: the offsets of the video dst rect within the libass
|
||||
* frame ([setOverlaySurfaceSize]); negative when the video extends beyond the frame.
|
||||
* Applied to the current render and re-applied whenever a render is (re)created.
|
||||
*/
|
||||
fun setMargins(top: Int, bottom: Int, left: Int, right: Int) {
|
||||
margins = intArrayOf(top, bottom, left, right)
|
||||
render?.setMargins(top, bottom, left, right)
|
||||
}
|
||||
|
||||
/** mpv's sub-ass-force-margins: anchor non-positioned events to the visible frame. */
|
||||
fun setUseMargins(use: Boolean) {
|
||||
useMargins = use
|
||||
render?.setUseMargins(use)
|
||||
}
|
||||
|
||||
override fun onVideoSizeChanged(videoSize: VideoSize) {
|
||||
super.onVideoSizeChanged(videoSize)
|
||||
this.videoSize = Size(videoSize.width, videoSize.height)
|
||||
Log.i("AssHandler", "onVideoSizeChanged: width = ${videoSize.width}, height = ${videoSize.height}")
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates the video size for the ASS renderer. Called as soon as the video size is known in
|
||||
* order to properly render subtitles.
|
||||
* @param width The width of the video.
|
||||
* @param height The height of the video.
|
||||
*/
|
||||
fun setVideoSize(width: Int, height: Int) {
|
||||
Log.i("AssHandler", "setVideoSize: width = $width, height = $height")
|
||||
videoSize = Size(width, height)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the current media has ASS tracks, false otherwise.
|
||||
*/
|
||||
fun hasTracks(): Boolean = availableTracks.isNotEmpty()
|
||||
|
||||
/**
|
||||
* Adds a font to the ASS library. If no tracks have been created yet, the font is buffered
|
||||
* and will be added when the first track is created via [createTrack].
|
||||
*/
|
||||
@Synchronized
|
||||
fun addFont(name: String, data: ByteArray) {
|
||||
if (hasTracks()) {
|
||||
ass.addFont(name, data)
|
||||
} else {
|
||||
pendingFonts.add(name to data)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new ASS track from the given format and saves it in the [availableTracks].
|
||||
* The renderer and libass are also created if needed.
|
||||
* @param format The format of the ASS track.
|
||||
* @return The created ASS track.
|
||||
*/
|
||||
@Synchronized
|
||||
fun createTrack(format: Format): AssTrack {
|
||||
Log.i("AssHandler", "createTrack: format = $format")
|
||||
// Ensure the renderer is created before creating tracks.
|
||||
createRenderIfNeeded()
|
||||
|
||||
// Flush any fonts that were buffered before the first track was created.
|
||||
if (pendingFonts.isNotEmpty()) {
|
||||
for ((name, data) in pendingFonts) {
|
||||
ass.addFont(name, data)
|
||||
}
|
||||
pendingFonts.clear()
|
||||
}
|
||||
|
||||
val track = ass.createTrack()
|
||||
if (format.initializationData.size > 0) {
|
||||
val header = AssHeaderParser.parse(format)
|
||||
track.readBuffer(header)
|
||||
}
|
||||
availableTracks[format.id!!] = track
|
||||
|
||||
updateTrack()
|
||||
|
||||
return track
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures the ASS renderer is created if it does not already exist.
|
||||
*/
|
||||
private fun createRenderIfNeeded() {
|
||||
if (render != null) return
|
||||
Log.i("AssHandler", "createRender (cacheSize: ${config.cacheSize}MB, glyphSize: ${config.glyphSize})")
|
||||
render = ass.createRender().also { render ->
|
||||
render.setCacheLimit(config.glyphSize, config.cacheSize)
|
||||
applyRenderState(render)
|
||||
}
|
||||
renderCallback?.invoke(render)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a dialogue into the track of the given [trackId].
|
||||
* Thread-safe: AssTrack.readChunk internally acquires the shared libass lock.
|
||||
*/
|
||||
fun readTrackDialogue(
|
||||
trackId: String?,
|
||||
start: Long,
|
||||
duration: Long,
|
||||
data: ByteArray,
|
||||
offset: Int = 0,
|
||||
length: Int = data.size
|
||||
) {
|
||||
val t = availableTracks[trackId] ?: return
|
||||
t.readChunk(start, duration, data, offset, length)
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves the selected video track, if any.
|
||||
*/
|
||||
private fun getSelectedVideoTrack(tracks: Tracks): Format? = tracks.groups.find { group ->
|
||||
if (group.isSelected) {
|
||||
(0 until group.length).any { index ->
|
||||
val track = group.getTrackFormat(index)
|
||||
MimeTypes.isVideo(track.sampleMimeType)
|
||||
}
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}?.getTrackFormat(0)
|
||||
|
||||
/**
|
||||
* Retrieves the ID of the selected ASS track, if any.
|
||||
* @param tracks The selected tracks.
|
||||
* @return The ID of the selected ASS track, or null if none.
|
||||
*/
|
||||
private fun getSelectedAssTrack(tracks: Tracks): Format? = tracks.groups.find { group ->
|
||||
if (group.isSelected) {
|
||||
(0 until group.length).any { index ->
|
||||
val track = group.getTrackFormat(index)
|
||||
track.sampleMimeType == TEXT_SSA || track.codecs == TEXT_SSA
|
||||
}
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}?.getTrackFormat(0)
|
||||
|
||||
/**
|
||||
* Releases all native resources held by this handler.
|
||||
*/
|
||||
fun release() {
|
||||
videoFrameCallback = null
|
||||
player?.clearVideoFrameMetadataListener(videoFrameMetadataListener)
|
||||
player = null
|
||||
render?.release()
|
||||
render = null
|
||||
availableTracks.values.forEach { it.release() }
|
||||
availableTracks.clear()
|
||||
track = null
|
||||
pendingFonts.clear()
|
||||
if (assDelegate.isInitialized()) {
|
||||
ass.release()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the size is valid (both width and height are greater than 0).
|
||||
*/
|
||||
private val Size.isValid
|
||||
get() = width > 0 && height > 0
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package com.edde746.plezy.libass.media
|
||||
|
||||
data class AssHandlerConfig(
|
||||
val glyphSize: Int = 10000,
|
||||
val cacheSize: Int = 128
|
||||
)
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
package com.edde746.plezy.libass.media.extractor
|
||||
|
||||
import androidx.annotation.OptIn
|
||||
import androidx.media3.common.util.ParsableByteArray
|
||||
import androidx.media3.common.util.UnstableApi
|
||||
import androidx.media3.extractor.ExtractorInput
|
||||
import androidx.media3.extractor.ExtractorOutput
|
||||
import androidx.media3.extractor.mkv.EbmlProcessor
|
||||
import androidx.media3.extractor.mkv.MatroskaExtractor
|
||||
import androidx.media3.extractor.text.SubtitleParser
|
||||
import com.edde746.plezy.libass.media.AssHandler
|
||||
import com.edde746.plezy.libass.media.text.AssSubtitleExtractorOutput
|
||||
|
||||
@OptIn(UnstableApi::class)
|
||||
open class AssMatroskaExtractor(
|
||||
subtitleParserFactory: SubtitleParser.Factory,
|
||||
private val assHandler: AssHandler,
|
||||
flags: Int = 0
|
||||
) : MatroskaExtractor(subtitleParserFactory, flags) {
|
||||
|
||||
private var currentAttachmentName: String? = null
|
||||
private var currentAttachmentMime: String? = null
|
||||
|
||||
internal val subtitleSample = subtitleSampleField.get(this) as ParsableByteArray
|
||||
|
||||
override fun getElementType(id: Int): Int = when (id) {
|
||||
ID_ATTACHMENTS -> EbmlProcessor.ELEMENT_TYPE_MASTER
|
||||
ID_ATTACHED_FILE -> EbmlProcessor.ELEMENT_TYPE_MASTER
|
||||
ID_FILE_NAME -> EbmlProcessor.ELEMENT_TYPE_STRING
|
||||
ID_FILE_MIME_TYPE -> EbmlProcessor.ELEMENT_TYPE_STRING
|
||||
ID_FILE_DATA -> EbmlProcessor.ELEMENT_TYPE_BINARY
|
||||
else -> super.getElementType(id)
|
||||
}
|
||||
|
||||
override fun isLevel1Element(id: Int): Boolean = super.isLevel1Element(id) || id == ID_ATTACHMENTS
|
||||
|
||||
override fun startMasterElement(id: Int, contentPosition: Long, contentSize: Long) {
|
||||
when (id) {
|
||||
ID_EBML -> {
|
||||
val currentExtractor = extractorOutput.get(this) as ExtractorOutput
|
||||
if (currentExtractor !is AssSubtitleExtractorOutput) {
|
||||
extractorOutput.set(
|
||||
this,
|
||||
AssSubtitleExtractorOutput(currentExtractor, assHandler, this)
|
||||
)
|
||||
}
|
||||
super.startMasterElement(id, contentPosition, contentSize)
|
||||
}
|
||||
ID_ATTACHED_FILE -> clearAttachment()
|
||||
else -> super.startMasterElement(id, contentPosition, contentSize)
|
||||
}
|
||||
}
|
||||
|
||||
override fun endMasterElement(id: Int) {
|
||||
when (id) {
|
||||
ID_VIDEO -> {
|
||||
// We need to get the video dimensions very early
|
||||
val track = getCurrentTrack(id)
|
||||
assHandler.setVideoSize(track.width, track.height)
|
||||
super.endMasterElement(id)
|
||||
}
|
||||
ID_ATTACHED_FILE -> clearAttachment()
|
||||
else -> super.endMasterElement(id)
|
||||
}
|
||||
}
|
||||
|
||||
override fun stringElement(id: Int, value: String) {
|
||||
when (id) {
|
||||
ID_FILE_NAME -> currentAttachmentName = value
|
||||
ID_FILE_MIME_TYPE -> currentAttachmentMime = value
|
||||
else -> super.stringElement(id, value)
|
||||
}
|
||||
}
|
||||
|
||||
override fun binaryElement(id: Int, contentSize: Int, input: ExtractorInput) {
|
||||
when (id) {
|
||||
ID_FILE_DATA -> {
|
||||
val attachmentName = requireNotNull(currentAttachmentName)
|
||||
val attachmentMime = requireNotNull(currentAttachmentMime)
|
||||
|
||||
if (attachmentMime in fontMimeTypes) {
|
||||
val data = ByteArray(contentSize)
|
||||
input.readFully(data, 0, contentSize)
|
||||
assHandler.addFont(attachmentName, data)
|
||||
} else {
|
||||
input.skipFully(contentSize)
|
||||
}
|
||||
}
|
||||
else -> super.binaryElement(id, contentSize, input)
|
||||
}
|
||||
}
|
||||
|
||||
private fun clearAttachment() {
|
||||
currentAttachmentName = null
|
||||
currentAttachmentMime = null
|
||||
}
|
||||
|
||||
companion object {
|
||||
const val ID_EBML = 0x1A45DFA3
|
||||
const val ID_VIDEO = 0xE0
|
||||
const val ID_ATTACHMENTS = 0x1941A469
|
||||
const val ID_ATTACHED_FILE = 0x61A7
|
||||
const val ID_FILE_NAME = 0x466E
|
||||
const val ID_FILE_MIME_TYPE = 0x4660
|
||||
const val ID_FILE_DATA = 0x465C
|
||||
|
||||
val fontMimeTypes = listOf(
|
||||
"font/ttf",
|
||||
"font/otf",
|
||||
"font/sfnt",
|
||||
"font/woff",
|
||||
"font/woff2",
|
||||
"application/font-sfnt",
|
||||
"application/font-woff",
|
||||
"application/x-truetype-font",
|
||||
"application/vnd.ms-opentype",
|
||||
"application/x-font-ttf"
|
||||
)
|
||||
|
||||
val extractorOutput = MatroskaExtractor::class.java.getDeclaredField("extractorOutput").apply {
|
||||
isAccessible = true
|
||||
}
|
||||
val subtitleSampleField = MatroskaExtractor::class.java.getDeclaredField("subtitleSample").apply {
|
||||
isAccessible = true
|
||||
}
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package com.edde746.plezy.libass.media.parser
|
||||
|
||||
import androidx.media3.common.Format
|
||||
import androidx.media3.common.util.Consumer
|
||||
import androidx.media3.common.util.UnstableApi
|
||||
import androidx.media3.extractor.text.CuesWithTiming
|
||||
import androidx.media3.extractor.text.SubtitleParser
|
||||
import com.edde746.plezy.libass.AssTrack
|
||||
|
||||
/**
|
||||
* Parser for full (non-embedded) ASS documents, e.g. external sidecar files.
|
||||
* Feeds the whole document into the libass track; rendering happens on the
|
||||
* overlay surface, so no Media3 cues are emitted.
|
||||
*/
|
||||
@UnstableApi
|
||||
class AssFullSubtitleParser(private val track: AssTrack) : SubtitleParser {
|
||||
|
||||
override fun parse(
|
||||
data: ByteArray,
|
||||
offset: Int,
|
||||
length: Int,
|
||||
outputOptions: SubtitleParser.OutputOptions,
|
||||
output: Consumer<CuesWithTiming>
|
||||
) {
|
||||
track.readBuffer(data, offset, length)
|
||||
}
|
||||
|
||||
override fun getCueReplacementBehavior(): Int = Format.CUE_REPLACEMENT_BEHAVIOR_REPLACE
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package com.edde746.plezy.libass.media.parser
|
||||
|
||||
import androidx.annotation.OptIn
|
||||
import androidx.media3.common.Format
|
||||
import androidx.media3.common.util.UnstableApi
|
||||
|
||||
@OptIn(UnstableApi::class)
|
||||
object AssHeaderParser {
|
||||
|
||||
private const val ASS_EVENTS = "[Events]\n" +
|
||||
"Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text"
|
||||
|
||||
/**
|
||||
* Fix some ass header with error end.
|
||||
* https://github.com/jellyfin/jellyfin-ffmpeg/issues/506
|
||||
*/
|
||||
private fun fixAssHeaderIfNeed(buffer: ByteArray): ByteArray = if (buffer[buffer.size - 1] != 0.toByte()) {
|
||||
// validate ass header
|
||||
buffer
|
||||
} else {
|
||||
// remote the last null character and append the events tag
|
||||
(String(buffer, 0, buffer.size - 1) + "\n" + ASS_EVENTS).toByteArray()
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the headers from the initialization data of the given [format]. The original
|
||||
* headers are preserved (duplication checks are handled by libass).
|
||||
*/
|
||||
fun parse(format: Format): ByteArray = fixAssHeaderIfNeed(format.initializationData[1])
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
package com.edde746.plezy.libass.media.parser
|
||||
|
||||
import androidx.media3.common.Format
|
||||
import androidx.media3.common.util.Consumer
|
||||
import androidx.media3.common.util.UnstableApi
|
||||
import androidx.media3.extractor.text.CuesWithTiming
|
||||
import androidx.media3.extractor.text.SubtitleParser
|
||||
|
||||
/**
|
||||
* No operation subtitle parser.
|
||||
*/
|
||||
@UnstableApi
|
||||
class AssNoOpSubtitleParser : SubtitleParser {
|
||||
override fun parse(
|
||||
p0: ByteArray,
|
||||
p1: Int,
|
||||
p2: Int,
|
||||
p3: SubtitleParser.OutputOptions,
|
||||
p4: Consumer<CuesWithTiming>
|
||||
) {
|
||||
}
|
||||
|
||||
override fun getCueReplacementBehavior(): Int = Format.CUE_REPLACEMENT_BEHAVIOR_REPLACE
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
package com.edde746.plezy.libass.media.parser
|
||||
|
||||
import androidx.media3.common.Format
|
||||
import androidx.media3.common.MimeTypes
|
||||
import androidx.media3.common.util.UnstableApi
|
||||
import androidx.media3.extractor.text.DefaultSubtitleParserFactory
|
||||
import androidx.media3.extractor.text.SubtitleParser
|
||||
import com.edde746.plezy.libass.media.AssHandler
|
||||
|
||||
@UnstableApi
|
||||
class AssSubtitleParserFactory(private val assHandler: AssHandler) : SubtitleParser.Factory {
|
||||
|
||||
private val defaultSubtitleParserFactory = DefaultSubtitleParserFactory()
|
||||
|
||||
override fun supportsFormat(format: Format): Boolean = defaultSubtitleParserFactory.supportsFormat(format)
|
||||
|
||||
override fun getCueReplacementBehavior(format: Format): Int = defaultSubtitleParserFactory.getCueReplacementBehavior(format)
|
||||
|
||||
override fun create(format: Format): SubtitleParser = if (format.sampleMimeType == MimeTypes.TEXT_SSA) {
|
||||
val embeddedSubtitles = MimeTypes.VIDEO_MATROSKA
|
||||
.contentEquals(format.containerMimeType)
|
||||
val track = assHandler.createTrack(format)
|
||||
if (embeddedSubtitles) {
|
||||
// Embedded dialogue lines reach libass via AssTrackOutput; nothing to parse here.
|
||||
AssNoOpSubtitleParser()
|
||||
} else {
|
||||
AssFullSubtitleParser(track)
|
||||
}
|
||||
} else {
|
||||
defaultSubtitleParserFactory.create(format)
|
||||
}
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.edde746.plezy.libass.media.text
|
||||
|
||||
import androidx.media3.common.C
|
||||
import androidx.media3.common.util.UnstableApi
|
||||
import androidx.media3.extractor.ExtractorOutput
|
||||
import androidx.media3.extractor.TrackOutput
|
||||
import com.edde746.plezy.libass.media.AssHandler
|
||||
import com.edde746.plezy.libass.media.extractor.AssMatroskaExtractor
|
||||
|
||||
/**
|
||||
* This class is only used by the overlay renderer. It's needed to get the start time of the subtitles.
|
||||
*/
|
||||
@UnstableApi
|
||||
class AssSubtitleExtractorOutput(
|
||||
private val delegate: ExtractorOutput,
|
||||
private val assHandler: AssHandler,
|
||||
private val extractor: AssMatroskaExtractor
|
||||
) : ExtractorOutput by delegate {
|
||||
override fun track(id: Int, type: Int): TrackOutput = if (type == C.TRACK_TYPE_TEXT) {
|
||||
// We can't know at this time if the subtitle track is ASS or other format, so we wrap
|
||||
// every subtitle track
|
||||
AssTrackOutput(delegate.track(id, type), assHandler, extractor)
|
||||
} else {
|
||||
delegate.track(id, type)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package com.edde746.plezy.libass.media.text
|
||||
|
||||
import androidx.media3.common.C
|
||||
import androidx.media3.common.Format
|
||||
import androidx.media3.common.MimeTypes
|
||||
import androidx.media3.common.util.UnstableApi
|
||||
import androidx.media3.common.util.Util
|
||||
import androidx.media3.extractor.TrackOutput
|
||||
import com.edde746.plezy.libass.media.AssHandler
|
||||
import com.edde746.plezy.libass.media.extractor.AssMatroskaExtractor
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* This class is only used by the overlay renderer. It's needed to get the start time of the subtitles.
|
||||
*/
|
||||
@UnstableApi
|
||||
class AssTrackOutput(
|
||||
private val delegate: TrackOutput,
|
||||
private val assHandler: AssHandler,
|
||||
private val extractor: AssMatroskaExtractor
|
||||
) : TrackOutput by delegate {
|
||||
|
||||
private var isAss = false
|
||||
|
||||
private var trackId: String? = null
|
||||
|
||||
override fun format(format: Format) {
|
||||
if (format.sampleMimeType == MimeTypes.TEXT_SSA || format.codecs == MimeTypes.TEXT_SSA) {
|
||||
isAss = true
|
||||
trackId = format.id
|
||||
}
|
||||
delegate.format(format)
|
||||
}
|
||||
|
||||
override fun sampleMetadata(
|
||||
timeUs: Long,
|
||||
flags: Int,
|
||||
size: Int,
|
||||
offset: Int,
|
||||
cryptoData: TrackOutput.CryptoData?
|
||||
) {
|
||||
if (isAss && timeUs.isValidTs) {
|
||||
val sample = extractor.subtitleSample
|
||||
val endIndex = findTokenIndex(sample.data, 1)
|
||||
val lineIndex = findTokenIndex(sample.data, 2)
|
||||
|
||||
val rawDuration = sample.data.decodeToString(endIndex, lineIndex - 1)
|
||||
val durationUs = parseTimecodeUs(rawDuration)
|
||||
|
||||
assHandler.readTrackDialogue(
|
||||
trackId = trackId,
|
||||
start = timeUs / 1000,
|
||||
duration = durationUs / 1000,
|
||||
data = sample.data,
|
||||
offset = lineIndex,
|
||||
length = sample.limit() - lineIndex
|
||||
)
|
||||
}
|
||||
delegate.sampleMetadata(timeUs, flags, size, offset, cryptoData)
|
||||
}
|
||||
|
||||
private fun parseTimecodeUs(timeString: String): Long {
|
||||
val matcher = SSA_TIMECODE_PATTERN.matcher(timeString.trim { it <= ' ' })
|
||||
if (!matcher.matches()) {
|
||||
return C.TIME_UNSET
|
||||
}
|
||||
var timestampUs =
|
||||
Util.castNonNull(matcher.group(1)).toLong() * 60 * 60 * C.MICROS_PER_SECOND
|
||||
timestampUs += Util.castNonNull(matcher.group(2)).toLong() * 60 * C.MICROS_PER_SECOND
|
||||
timestampUs += Util.castNonNull(matcher.group(3)).toLong() * C.MICROS_PER_SECOND
|
||||
timestampUs += Util.castNonNull(matcher.group(4)).toLong() * 10000
|
||||
return timestampUs
|
||||
}
|
||||
|
||||
private fun findTokenIndex(array: ByteArray, tokenNumber: Int): Int {
|
||||
if (tokenNumber == 0) return 0
|
||||
var tokensFound = 0
|
||||
array.forEachIndexed { index, byte ->
|
||||
if (byte == COMMA && ++tokensFound == tokenNumber) {
|
||||
return index + 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
private val Long.isValidTs
|
||||
get() = this != C.TIME_UNSET
|
||||
|
||||
private companion object {
|
||||
val SSA_TIMECODE_PATTERN: Pattern =
|
||||
Pattern.compile("""(?:(\d+):)?(\d+):(\d+)[:.](\d+)""")
|
||||
|
||||
const val COMMA = ','.code.toByte()
|
||||
}
|
||||
}
|
||||
+991
@@ -0,0 +1,991 @@
|
||||
package com.edde746.plezy.libass.media.widget
|
||||
|
||||
import android.opengl.EGL14
|
||||
import android.opengl.EGLContext
|
||||
import android.opengl.EGLDisplay
|
||||
import android.opengl.EGLExt
|
||||
import android.opengl.EGLSurface
|
||||
import android.opengl.GLES20
|
||||
import android.os.Handler
|
||||
import android.os.HandlerThread
|
||||
import android.os.Process
|
||||
import android.util.Log
|
||||
import android.view.Surface
|
||||
import androidx.media3.common.C
|
||||
import androidx.media3.common.util.GlProgram
|
||||
import androidx.media3.common.util.GlUtil
|
||||
import androidx.media3.common.util.Size
|
||||
import androidx.media3.common.util.UnstableApi
|
||||
import com.edde746.plezy.libass.AssAtlasFrame
|
||||
import com.edde746.plezy.libass.media.AssHandler
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.ByteOrder
|
||||
import java.util.concurrent.atomic.AtomicReference
|
||||
|
||||
/**
|
||||
* Atlas-rendering pipeline behind [AssSubtitleSurfaceView].
|
||||
* Runs libass on its own [HandlerThread] into a packed
|
||||
* ALPHA_8 texture atlas plus a single vertex stream, and a GL thread that uploads
|
||||
* both and issues one `glDrawArrays` per frame. Each swap is pinned to the video's
|
||||
* target release time via [EGLExt.eglPresentationTimeANDROID]: SurfaceFlinger holds
|
||||
* the buffer and composes it on the same vsync as the corresponding video frame.
|
||||
*/
|
||||
@UnstableApi
|
||||
internal object AssAtlasPipelineConfig {
|
||||
/** Flip to `true` for `adb logcat -s AssSurfaceGlThread:D AssLibassThread:D` traces. */
|
||||
internal const val TIMING_LOGS = false
|
||||
|
||||
/** Fallback atlas row width when GL caps are unknown (EGL init failed/slow). */
|
||||
internal const val FALLBACK_ATLAS_W = 2048
|
||||
|
||||
/** Fallback atlas height; 2048 × 4096 = 8 MB matches the long-proven default. */
|
||||
internal const val FALLBACK_ATLAS_H = 4096
|
||||
|
||||
/**
|
||||
* Pixel budget per atlas slot once GL caps are known: 4096×4096 or 2048×8192
|
||||
* (16 MB ALPHA_8). Sized so heavy typesetting (full-screen gradients/blurs)
|
||||
* fits where the old fixed 2048×4096 overflowed and dropped frames.
|
||||
*/
|
||||
internal const val ATLAS_PIXEL_BUDGET = 16 * 1024 * 1024
|
||||
|
||||
/** Preallocated vertex-stream capacity (192 bytes × 16384 = 3 MB per buffer). */
|
||||
internal const val MAX_QUADS = 16384
|
||||
|
||||
/** Must match the byte layout produced by `nativeAssRenderFrameAtlas` in AssKt.c. */
|
||||
internal const val BYTES_PER_VERTEX = 32
|
||||
internal const val BYTES_PER_QUAD = BYTES_PER_VERTEX * 6
|
||||
|
||||
/**
|
||||
* Kill switch for speculative render-ahead (see [SpecRenderEngine]) and the
|
||||
* event prefetch. With it off (or on low-RAM devices, which stay at 2 slots)
|
||||
* every request renders on-demand inside the video frame's release deadline —
|
||||
* the pre-speculation behavior.
|
||||
*
|
||||
* Only pays off when typical changed renders fit the speculation coverage
|
||||
* window (~frame interval + release budget): with the un-optimized (-O0)
|
||||
* native core's 90ms+ renders it could only add overhead; with the -O3/NEON
|
||||
* core's ~35-50ms renders it converts near-misses into on-time latches.
|
||||
*/
|
||||
internal const val SPECULATION_ENABLED = true
|
||||
}
|
||||
|
||||
/** Payload handed from the libass worker to the GL thread. */
|
||||
internal class AtlasPayload(
|
||||
val slotIndex: Int,
|
||||
val atlasBuf: ByteBuffer,
|
||||
val vertexBuf: ByteBuffer,
|
||||
var frame: AssAtlasFrame,
|
||||
var presentationTimeUs: Long,
|
||||
var releaseTimeNs: Long,
|
||||
/** Bumped on every content-changing render; lets the GL thread tell "same slot,
|
||||
* new content" apart from "same slot, same content" when deciding to re-upload. */
|
||||
var contentSeq: Long = 0L
|
||||
)
|
||||
|
||||
/** Payload slots plus the atlas dims their buffers were sized for. */
|
||||
internal class AtlasSlots(
|
||||
val payloads: Array<AtlasPayload>,
|
||||
val atlasW: Int,
|
||||
val atlasH: Int
|
||||
)
|
||||
|
||||
/**
|
||||
* Owns both the libass worker and the GL thread. The two talk via a single-slot
|
||||
* atomic — a newer payload always replaces a pending one so the GL thread never
|
||||
* falls behind.
|
||||
*/
|
||||
@UnstableApi
|
||||
internal class AssAtlasPipeline(
|
||||
surface: Surface,
|
||||
width: Int,
|
||||
height: Int,
|
||||
assHandler: AssHandler,
|
||||
lowRamDevice: Boolean = false
|
||||
) {
|
||||
private val surfaceWidth = width
|
||||
private val surfaceHeight = height
|
||||
|
||||
// 3 slots give the render-ahead engine a writable target while one slot is
|
||||
// posted and another is in GL's hands; low-RAM devices stay at 2 slots with
|
||||
// speculation off (the legacy on-demand behavior, ~19 MB less in buffers).
|
||||
private val slotCount = if (lowRamDevice) 2 else 3
|
||||
private val speculationEnabled = AssAtlasPipelineConfig.SPECULATION_ENABLED && slotCount >= 3
|
||||
|
||||
// Atlas dims, resolved exactly once (first-wins): normally by the GL thread from
|
||||
// GL_MAX_TEXTURE_SIZE right after EGL init; by the libass thread's 1 s fallback
|
||||
// if GL never comes up. Both threads then agree on the dims, which matters
|
||||
// because the C side bakes UV denominators = these dims into the vertex stream
|
||||
// and the GL side allocates the texture once at these dims.
|
||||
private val dimsResolved = java.util.concurrent.atomic.AtomicBoolean(false)
|
||||
private val dimsLatch = java.util.concurrent.CountDownLatch(1)
|
||||
|
||||
@Volatile private var atlasW = 0
|
||||
|
||||
@Volatile private var atlasH = 0
|
||||
|
||||
private fun resolveAtlasDims(maxTextureSize: Int): Pair<Int, Int> {
|
||||
if (dimsResolved.compareAndSet(false, true)) {
|
||||
if (maxTextureSize < 4096) {
|
||||
// Query failed (or an ancient GPU): keep the long-proven fixed size.
|
||||
atlasW = AssAtlasPipelineConfig.FALLBACK_ATLAS_W
|
||||
atlasH = AssAtlasPipelineConfig.FALLBACK_ATLAS_H
|
||||
} else {
|
||||
val w = if (surfaceWidth > 2048) 4096 else 2048
|
||||
atlasW = w
|
||||
atlasH = minOf(maxTextureSize, AssAtlasPipelineConfig.ATLAS_PIXEL_BUDGET / w)
|
||||
}
|
||||
if (AssAtlasPipelineConfig.TIMING_LOGS) {
|
||||
Log.d("AssAtlasPipeline", "atlas dims ${atlasW}x$atlasH (glMaxTexture=$maxTextureSize)")
|
||||
}
|
||||
dimsLatch.countDown()
|
||||
}
|
||||
return atlasW to atlasH
|
||||
}
|
||||
|
||||
// Lazily allocated on the libass thread once atlas dims are known; ~19 MB of
|
||||
// direct buffers per slot at the full budget, so don't pay it before the
|
||||
// first actual render. Confined to the libass thread after creation.
|
||||
private var slots: AtlasSlots? = null
|
||||
|
||||
private fun acquireSlots(): AtlasSlots {
|
||||
slots?.let { return it }
|
||||
if (!dimsLatch.await(1, java.util.concurrent.TimeUnit.SECONDS)) {
|
||||
resolveAtlasDims(0) // first-wins: no-op if the GL thread resolved meanwhile
|
||||
}
|
||||
val w = atlasW
|
||||
val h = atlasH
|
||||
val payloads = Array(slotCount) { index ->
|
||||
AtlasPayload(
|
||||
slotIndex = index,
|
||||
atlasBuf = ByteBuffer.allocateDirect(w * h).order(ByteOrder.nativeOrder()),
|
||||
vertexBuf = ByteBuffer.allocateDirect(
|
||||
AssAtlasPipelineConfig.MAX_QUADS * AssAtlasPipelineConfig.BYTES_PER_QUAD
|
||||
).order(ByteOrder.nativeOrder()),
|
||||
frame = AssAtlasFrame(0, 0, 0, 0, 0),
|
||||
presentationTimeUs = 0L,
|
||||
releaseTimeNs = C.TIME_UNSET
|
||||
)
|
||||
}
|
||||
return AtlasSlots(payloads, w, h).also { slots = it }
|
||||
}
|
||||
|
||||
/** Slot index the GL thread most recently took for drawing; the libass side
|
||||
* never writes it. Written only inside [takePending] on the GL thread. */
|
||||
@Volatile private var glLastTakenSlot = -1
|
||||
|
||||
private val pendingPayload = AtomicReference<AtlasPayload?>(null)
|
||||
private val glThread = AtlasGlThread(
|
||||
surface,
|
||||
width,
|
||||
height,
|
||||
assHandler,
|
||||
takePending = {
|
||||
pendingPayload.getAndSet(null)?.also { glLastTakenSlot = it.slotIndex }
|
||||
},
|
||||
resolveAtlasDims = ::resolveAtlasDims
|
||||
)
|
||||
private val libassThread = AtlasLibassThread(
|
||||
assHandler,
|
||||
acquireSlots = ::acquireSlots,
|
||||
speculationEnabled = speculationEnabled,
|
||||
glTakenSlot = { glLastTakenSlot },
|
||||
onFrameReady = { payload ->
|
||||
pendingPayload.set(payload)
|
||||
glThread.triggerDraw()
|
||||
}
|
||||
)
|
||||
|
||||
fun start() {
|
||||
if (AssAtlasPipelineConfig.TIMING_LOGS) {
|
||||
Log.d(
|
||||
"AssAtlasPipeline",
|
||||
"start surface=${surfaceWidth}x$surfaceHeight slots=$slotCount speculation=$speculationEnabled"
|
||||
)
|
||||
}
|
||||
glThread.start()
|
||||
libassThread.start()
|
||||
}
|
||||
|
||||
fun requestRender(presentationTimeUs: Long, releaseTimeNs: Long) {
|
||||
libassThread.enqueue(presentationTimeUs, releaseTimeNs)
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-renders the last requested position — for renderer state changes (margins,
|
||||
* use-margins) that must become visible while playback is paused. Safe during
|
||||
* playback: the next video frame's [requestRender] supersedes it (latest-wins).
|
||||
*/
|
||||
fun invalidate() {
|
||||
libassThread.invalidate()
|
||||
}
|
||||
|
||||
fun onSurfaceSizeChanged(width: Int, height: Int) {
|
||||
glThread.onSurfaceSizeChanged(width, height)
|
||||
}
|
||||
|
||||
/** Vsync-pinned swaps performed (excludes untimed invalidate repaints). */
|
||||
val swapCount: Long get() = glThread.swapCount
|
||||
|
||||
/** Pinned swaps that finished past the swap-time budget (possible missed vsync). */
|
||||
val lateSwapCount: Long get() = glThread.lateSwapCount
|
||||
|
||||
/** Worst observed swap lateness past the target release time, in milliseconds. */
|
||||
val maxLateMs: Long get() = glThread.maxLateMs
|
||||
|
||||
/** Total libass renders performed (one per serviced request). */
|
||||
val renderCount: Long get() = libassThread.renderCount
|
||||
|
||||
/** Renders where libass reported changed content (atlas/vertex rewritten). */
|
||||
val changedRenderCount: Long get() = libassThread.changedRenderCount
|
||||
|
||||
/** Renders that overflowed the atlas/vertex capacity (frame content incomplete). */
|
||||
val overflowCount: Long get() = libassThread.overflowCount
|
||||
|
||||
/** Duration of the most recent libass render, in milliseconds. */
|
||||
val lastLibassMs: Long get() = libassThread.lastLibassMs
|
||||
|
||||
/** Worst observed libass render duration, in milliseconds. */
|
||||
val maxLibassMs: Long get() = libassThread.maxLibassMs
|
||||
|
||||
/** Changed-render duration histogram: [≤10ms, ≤25ms, ≤42ms, ≤84ms, >84ms]. */
|
||||
val libassMsHistogram: List<Long> get() = libassThread.histogramSnapshot()
|
||||
|
||||
/** Requests served from a pre-rendered (speculative) frame — GL-only hot path. */
|
||||
val specHits: Long get() = libassThread.specHits
|
||||
|
||||
/** Requests where a speculative frame existed but didn't match (seek, state change). */
|
||||
val specMisses: Long get() = libassThread.specMisses
|
||||
|
||||
/** Speculation rounds skipped (paused, pending request, no confident cadence). */
|
||||
val specSkips: Long get() = libassThread.specSkips
|
||||
|
||||
/** Cache-warming prefetch renders of upcoming events. */
|
||||
val prefetchCount: Long get() = libassThread.prefetchCount
|
||||
|
||||
/** Worst (minimum) lead of a changed-content pinned swap vs its target release
|
||||
* time, in ms; negative = the new content was queued after the video frame's
|
||||
* vsync. Long.MAX_VALUE until a changed pinned swap happened. */
|
||||
val minLeadChangedMs: Long get() = glThread.minLeadChangedMs
|
||||
|
||||
fun releaseAndWait() {
|
||||
libassThread.releaseAndWait()
|
||||
glThread.releaseAndWait()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops a [Handler] synchronously: posts [releaseWhat] with an [Ack], waits up to 1 s
|
||||
* for the handler to invoke [onReleased] (on its own looper) and signal the latch.
|
||||
*/
|
||||
private fun postShutdownAndWait(
|
||||
handler: Handler,
|
||||
releaseWhat: Int,
|
||||
onReleased: () -> Unit
|
||||
) {
|
||||
val latch = Object()
|
||||
synchronized(latch) {
|
||||
handler.obtainMessage(releaseWhat, Ack(latch, onReleased)).sendToTarget()
|
||||
try {
|
||||
latch.wait(1_000)
|
||||
} catch (_: InterruptedException) {
|
||||
Thread.currentThread().interrupt()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Transport for [postShutdownAndWait] — the handler callback calls [release] then notifies. */
|
||||
private class Ack(val latch: Any, val release: () -> Unit)
|
||||
|
||||
/**
|
||||
* Runs libass off the GL thread into a packed atlas + vertex stream. Latest-wins:
|
||||
* older pending renders are dropped when a newer one arrives. Slot choice, the
|
||||
* changed-flag bookkeeping and speculative render-ahead live in [SpecRenderEngine];
|
||||
* this thread owns the buffers, the timing/stat accounting and the GL handoff.
|
||||
*/
|
||||
@UnstableApi
|
||||
private class AtlasLibassThread(
|
||||
private val assHandler: AssHandler,
|
||||
private val acquireSlots: () -> AtlasSlots,
|
||||
private val speculationEnabled: Boolean,
|
||||
private val glTakenSlot: () -> Int,
|
||||
private val onFrameReady: (AtlasPayload) -> Unit
|
||||
) : HandlerThread(TAG, Process.THREAD_PRIORITY_DISPLAY) {
|
||||
|
||||
/** Immutable (pts, release) request — handed off through a single atomic so a
|
||||
* concurrent enqueue can neither be lost by drain's consume nor torn in half.
|
||||
* [enqueueNs] timestamps the handoff so drain can report how long the request
|
||||
* sat behind an in-flight render (the queue-wait component of subtitle lag). */
|
||||
private class PendingFrame(val ptsUs: Long, val releaseNs: Long, val enqueueNs: Long = System.nanoTime())
|
||||
|
||||
private lateinit var handler: Handler
|
||||
|
||||
private val pending = AtomicReference<PendingFrame?>(null)
|
||||
|
||||
@Volatile private var lastRequestedPtsUs = UNSET
|
||||
private var contentSeqCounter = 0L
|
||||
|
||||
// Thread-confined; created on first render so non-ASS playback never allocates.
|
||||
private var engine: SpecRenderEngine? = null
|
||||
|
||||
val specHits: Long get() = engine?.specHits ?: 0L
|
||||
val specMisses: Long get() = engine?.specMisses ?: 0L
|
||||
val specSkips: Long get() = engine?.specSkips ?: 0L
|
||||
val prefetchCount: Long get() = engine?.prefetchCount ?: 0L
|
||||
|
||||
// Telemetry; single-writer (this thread), read from the stats path.
|
||||
@Volatile var renderCount = 0L
|
||||
private set
|
||||
|
||||
@Volatile var changedRenderCount = 0L
|
||||
private set
|
||||
|
||||
@Volatile var overflowCount = 0L
|
||||
private set
|
||||
|
||||
@Volatile var lastLibassMs = 0L
|
||||
private set
|
||||
|
||||
@Volatile var maxLibassMs = 0L
|
||||
private set
|
||||
|
||||
/** Changed-render durations bucketed at ≤10 / ≤25 / ≤42 / ≤84 / >84 ms. */
|
||||
private val histogram = java.util.concurrent.atomic.AtomicLongArray(5)
|
||||
|
||||
fun histogramSnapshot(): List<Long> = List(histogram.length()) { histogram.get(it) }
|
||||
|
||||
private fun recordChangedRenderMs(ms: Long) {
|
||||
val bucket = when {
|
||||
ms <= 10 -> 0
|
||||
ms <= 25 -> 1
|
||||
ms <= 42 -> 2
|
||||
ms <= 84 -> 3
|
||||
else -> 4
|
||||
}
|
||||
histogram.incrementAndGet(bucket)
|
||||
}
|
||||
|
||||
override fun start() {
|
||||
super.start()
|
||||
handler = Handler(looper) { msg ->
|
||||
when (msg.what) {
|
||||
MSG_RENDER -> drainAndRender()
|
||||
MSG_RELEASE -> {
|
||||
val ack = msg.obj as Ack
|
||||
ack.release()
|
||||
quit()
|
||||
synchronized(ack.latch) { (ack.latch as Object).notifyAll() }
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
fun enqueue(presentationTimeUs: Long, releaseTimeNs: Long) {
|
||||
if (!::handler.isInitialized) return
|
||||
val dropped = pending.getAndSet(PendingFrame(presentationTimeUs, releaseTimeNs))
|
||||
if (dropped != null && AssAtlasPipelineConfig.TIMING_LOGS) {
|
||||
// A request was coalesced away — the renderer is behind by at least one
|
||||
// frame. agedMs = how long the dropped request had been waiting.
|
||||
Log.d(
|
||||
TAG,
|
||||
"drop pts=${dropped.ptsUs / 1000}ms agedMs=${(System.nanoTime() - dropped.enqueueNs) / 1_000_000} " +
|
||||
"replacedBy=${presentationTimeUs / 1000}ms"
|
||||
)
|
||||
}
|
||||
handler.removeMessages(MSG_RENDER)
|
||||
handler.sendEmptyMessage(MSG_RENDER)
|
||||
}
|
||||
|
||||
/** Re-enqueues the last requested PTS (renderer state changed, possibly while paused). */
|
||||
fun invalidate() {
|
||||
val pts = lastRequestedPtsUs
|
||||
if (pts == UNSET) return
|
||||
// TIME_UNSET release time => the GL thread swaps immediately instead of
|
||||
// vsync-pinning to a video frame that may never come while paused.
|
||||
enqueue(pts, C.TIME_UNSET)
|
||||
}
|
||||
|
||||
private fun ensureEngine(slots: AtlasSlots): SpecRenderEngine {
|
||||
engine?.let { return it }
|
||||
return SpecRenderEngine(
|
||||
slotCount = slots.payloads.size,
|
||||
speculationEnabled = speculationEnabled,
|
||||
renderAt = { timeMs, slot -> timedRender(timeMs, slots, slot) },
|
||||
// Renderer identity in the high bits + its state generation in the low bits:
|
||||
// a recreated renderer (media item transition) can never alias a stale
|
||||
// speculation, even if the new generation counter happens to match.
|
||||
stateGeneration = {
|
||||
assHandler.render?.let {
|
||||
(System.identityHashCode(it).toLong() shl 32) or (it.stateGeneration.toLong() and 0xffffffffL)
|
||||
} ?: -1L
|
||||
},
|
||||
glTakenSlot = glTakenSlot,
|
||||
debugLog = if (AssAtlasPipelineConfig.TIMING_LOGS) ({ msg -> Log.d(TAG, msg) }) else null
|
||||
).also { engine = it }
|
||||
}
|
||||
|
||||
/** Consecutive renders that reported no content change — a static screen.
|
||||
* Reset by any changed render (dialogue flips, karaoke, animated signs). */
|
||||
private var unchangedStreak = 0
|
||||
|
||||
/** Renders into [slot]'s buffers, owning the per-render timing/stat accounting
|
||||
* for both on-demand and speculative renders. */
|
||||
private fun timedRender(timeMs: Long, slots: AtlasSlots, slot: Int): AssAtlasFrame? {
|
||||
val render = assHandler.render ?: return null
|
||||
val payload = slots.payloads[slot]
|
||||
val t0 = System.nanoTime()
|
||||
val frame = render.renderFrameAtlas(timeMs, payload.atlasBuf, slots.atlasW, slots.atlasH, payload.vertexBuf)
|
||||
?: return null
|
||||
val libassMs = (System.nanoTime() - t0) / 1_000_000
|
||||
renderCount++
|
||||
lastLibassMs = libassMs
|
||||
if (libassMs > maxLibassMs) maxLibassMs = libassMs
|
||||
if (frame.truncated > 0) overflowCount++
|
||||
if (frame.changed != 0) {
|
||||
changedRenderCount++
|
||||
recordChangedRenderMs(libassMs)
|
||||
unchangedStreak = 0
|
||||
} else {
|
||||
unchangedStreak++
|
||||
}
|
||||
return frame
|
||||
}
|
||||
|
||||
private fun drainAndRender() {
|
||||
val request = pending.getAndSet(null) ?: return
|
||||
val pts = request.ptsUs
|
||||
val releaseNs = request.releaseNs
|
||||
lastRequestedPtsUs = pts
|
||||
val tDrain = System.nanoTime()
|
||||
// How long the request sat in the handoff (behind an in-flight on-demand or
|
||||
// speculative render) — the queue-wait component of any subtitle lag.
|
||||
val waitMs = (tDrain - request.enqueueNs) / 1_000_000
|
||||
// Before any ASS render exists (SRT/VTT or no subs) do nothing — this also
|
||||
// keeps the slot buffers unallocated for non-ASS playback.
|
||||
if (assHandler.render == null) return
|
||||
val slots = acquireSlots()
|
||||
val engine = ensureEngine(slots)
|
||||
val pinned = releaseNs != C.TIME_UNSET
|
||||
// Budget left until the video frame's vsync when we START servicing.
|
||||
val budgetMs = if (pinned) (releaseNs - tDrain) / 1_000_000 else -1L
|
||||
|
||||
when (val outcome = engine.service(pts, pinned)) {
|
||||
is SpecRenderEngine.Outcome.Post -> {
|
||||
val payload = slots.payloads[outcome.slot]
|
||||
if (outcome.newContent) {
|
||||
payload.frame = outcome.frame
|
||||
payload.contentSeq = ++contentSeqCounter
|
||||
}
|
||||
payload.presentationTimeUs = pts
|
||||
payload.releaseTimeNs = releaseNs
|
||||
onFrameReady(payload)
|
||||
if (AssAtlasPipelineConfig.TIMING_LOGS) {
|
||||
Log.d(
|
||||
TAG,
|
||||
"render pts=${pts / 1000}ms seq=${payload.contentSeq} waitMs=$waitMs budgetMs=$budgetMs " +
|
||||
"libassMs=$lastLibassMs lockWaitMs=${assHandler.render?.lastLockWaitMs} " +
|
||||
"specHit=${outcome.specHit} changed=${payload.frame.changed} quads=${payload.frame.quadCount} " +
|
||||
"atlas=${payload.frame.atlasWidth}x${payload.frame.atlasHeight} truncated=${payload.frame.truncated}"
|
||||
)
|
||||
}
|
||||
}
|
||||
SpecRenderEngine.Outcome.Skip -> {
|
||||
if (AssAtlasPipelineConfig.TIMING_LOGS) {
|
||||
Log.d(TAG, "skip pts=${pts / 1000}ms waitMs=$waitMs budgetMs=$budgetMs (no content)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pre-render the predicted next frame in the dead time between requests so the
|
||||
// next service is (usually) a GL-only hit. Never delays a waiting request.
|
||||
engine.speculateAfter(pts, pinned, hasPending = pending.get() != null)?.let { write ->
|
||||
val payload = slots.payloads[write.slot]
|
||||
payload.frame = write.frame
|
||||
payload.contentSeq = ++contentSeqCounter
|
||||
if (AssAtlasPipelineConfig.TIMING_LOGS) {
|
||||
Log.d(
|
||||
TAG,
|
||||
"spec after=${pts / 1000}ms seq=${payload.contentSeq} libassMs=$lastLibassMs " +
|
||||
"lockWaitMs=${assHandler.render?.lastLockWaitMs} slot=${write.slot} quads=${write.frame.quadCount}"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
maybePrefetch(engine, slots, pts, pinned)
|
||||
}
|
||||
|
||||
/** Start time of the last event boundary we cache-warmed (libass/track ms). */
|
||||
private var lastPrefetchedStartMs = Long.MIN_VALUE
|
||||
|
||||
/** Wall time of the last prefetch render, for the cooldown. */
|
||||
private var lastPrefetchNs = Long.MIN_VALUE / 2
|
||||
|
||||
/**
|
||||
* Cache-warms the next upcoming subtitle event so heavy typesetting pays its
|
||||
* cache-cold rasterization (seconds on weak devices) before the sign appears
|
||||
* instead of at appearance.
|
||||
*
|
||||
* The render blocks this thread, and during playback a new request is never
|
||||
* more than one frame interval away — so a prefetch is only allowed when its
|
||||
* delay cannot be SEEN, not merely when the queue is momentarily empty
|
||||
* (the v1 mistake, which thrashed on densely-authored per-frame events and
|
||||
* stalled visible dialogue):
|
||||
* - the screen must be static ([unchangedStreak]): requests delayed behind
|
||||
* the prefetch re-render identical content, so their lateness is invisible;
|
||||
* - the warmed event must be the NEXT on-screen change (no other event start
|
||||
* or end before it) — this also kills dense per-frame event sections,
|
||||
* where the next change is always ≤ one frame away;
|
||||
* - a cooldown bounds the worst-case overhead to one render per window.
|
||||
*/
|
||||
private fun maybePrefetch(engine: SpecRenderEngine, slots: AtlasSlots, ptsUs: Long, pinned: Boolean) {
|
||||
if (!speculationEnabled) return
|
||||
if (!pinned) return
|
||||
if (pending.get() != null) return
|
||||
if (unchangedStreak < PREFETCH_STATIC_STREAK) return
|
||||
val now = System.nanoTime()
|
||||
if (now - lastPrefetchNs < PREFETCH_COOLDOWN_NS) return
|
||||
val track = assHandler.track ?: return
|
||||
val nowMs = ptsUs / 1000
|
||||
// Events closer than MIN_AHEAD are the regular per-frame path's business;
|
||||
// beyond HORIZON the warmed bitmaps may be evicted before they're needed.
|
||||
val targetMs = track.nextEventStartMs(nowMs + PREFETCH_MIN_AHEAD_MS)
|
||||
if (targetMs < 0 || targetMs == lastPrefetchedStartMs) return
|
||||
if (targetMs > nowMs + PREFETCH_HORIZON_MS) return
|
||||
// Invisibility gate, time-budgeted: a prefetch is invisible as long as it
|
||||
// finishes before anything on screen is due to change (the screen is static
|
||||
// per the streak gate, so requests delayed behind it re-render identical
|
||||
// content). Estimate the cost from the worst render seen this session —
|
||||
// an overestimate only skips warming; an underestimate delays one boundary
|
||||
// by the shortfall. nextEventChangeMs also sees ends, so a dialogue line
|
||||
// due to disappear inside the budget skips the prefetch.
|
||||
val nextChangeMs = track.nextEventChangeMs(nowMs)
|
||||
if (nextChangeMs in 0 until targetMs) {
|
||||
val runwayMs = nextChangeMs - nowMs
|
||||
val costEstimateMs = (maxLibassMs * 5 / 4).coerceIn(PREFETCH_COST_FLOOR_MS, PREFETCH_COST_CEIL_MS)
|
||||
if (runwayMs < costEstimateMs + PREFETCH_SAFETY_MS) return
|
||||
}
|
||||
lastPrefetchedStartMs = targetMs
|
||||
lastPrefetchNs = now
|
||||
engine.prefetch(targetMs * 1000)?.let { write ->
|
||||
val payload = slots.payloads[write.slot]
|
||||
payload.frame = write.frame
|
||||
payload.contentSeq = ++contentSeqCounter
|
||||
}
|
||||
if (AssAtlasPipelineConfig.TIMING_LOGS) {
|
||||
Log.d(
|
||||
TAG,
|
||||
"prefetch evt=${targetMs}ms aheadMs=${targetMs - nowMs} " +
|
||||
"tookMs=${(System.nanoTime() - now) / 1_000_000} libassMs=$lastLibassMs"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fun releaseAndWait() {
|
||||
if (!::handler.isInitialized) {
|
||||
quit()
|
||||
return
|
||||
}
|
||||
postShutdownAndWait(handler, MSG_RELEASE) { /* nothing thread-local to tear down */ }
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val TAG = "AssLibassThread"
|
||||
private const val MSG_RENDER = 1
|
||||
private const val MSG_RELEASE = 2
|
||||
private const val UNSET = Long.MIN_VALUE
|
||||
|
||||
/** Don't prefetch events the per-frame speculation will reach imminently. */
|
||||
private const val PREFETCH_MIN_AHEAD_MS = 1_000L
|
||||
|
||||
/** Don't warm caches so early that the bitmaps could be evicted again. */
|
||||
private const val PREFETCH_HORIZON_MS = 15_000L
|
||||
|
||||
/** Static-screen requirement before a prefetch may block this thread. */
|
||||
private const val PREFETCH_STATIC_STREAK = 3
|
||||
|
||||
/** Minimum spacing between prefetch renders. */
|
||||
private const val PREFETCH_COOLDOWN_NS = 2_000_000_000L
|
||||
|
||||
/** Cost-estimate clamp for the time-budgeted invisibility gate: never
|
||||
* assume a prefetch cheaper than the floor (estimator may not have seen a
|
||||
* heavy frame yet) nor pointlessly demand more runway than the ceiling. */
|
||||
private const val PREFETCH_COST_FLOOR_MS = 250L
|
||||
private const val PREFETCH_COST_CEIL_MS = 1_500L
|
||||
|
||||
/** Slack added to the cost estimate when checking the static runway. */
|
||||
private const val PREFETCH_SAFETY_MS = 150L
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns the EGL surface, uploads the atlas + vertex stream and issues a single
|
||||
* `glDrawArrays` per frame. Swaps immediately with the swap pinned to the video's
|
||||
* target release time via [EGLExt.eglPresentationTimeANDROID]; SurfaceFlinger
|
||||
* holds the buffer until then, so the thread is never blocked waiting for a vsync.
|
||||
*/
|
||||
@UnstableApi
|
||||
private class AtlasGlThread(
|
||||
private val surface: Surface,
|
||||
@Volatile private var width: Int,
|
||||
@Volatile private var height: Int,
|
||||
private val assHandler: AssHandler,
|
||||
private val takePending: () -> AtlasPayload?,
|
||||
private val resolveAtlasDims: (maxTextureSize: Int) -> Pair<Int, Int>
|
||||
) : HandlerThread(TAG, Process.THREAD_PRIORITY_DISPLAY) {
|
||||
|
||||
private lateinit var handler: Handler
|
||||
private var eglDisplay: EGLDisplay = EGL14.EGL_NO_DISPLAY
|
||||
private var eglContext: EGLContext = EGL14.EGL_NO_CONTEXT
|
||||
private var eglSurface: EGLSurface = EGL14.EGL_NO_SURFACE
|
||||
|
||||
private val renderer = AtlasRenderer(assHandler)
|
||||
private var lastUploadedPayload: AtlasPayload? = null
|
||||
private var lastUploadedSeq = -1L
|
||||
private var lastSwappedSeq = -1L
|
||||
|
||||
// Lateness telemetry; single-writer (this thread), read from the stats path.
|
||||
@Volatile var swapCount = 0L
|
||||
private set
|
||||
|
||||
@Volatile var lateSwapCount = 0L
|
||||
private set
|
||||
|
||||
@Volatile var maxLateMs = 0L
|
||||
private set
|
||||
|
||||
/** Minimum lead (release target − swap completion) over changed-content pinned
|
||||
* swaps; negative = content queued after the video frame's vsync. */
|
||||
@Volatile var minLeadChangedMs = Long.MAX_VALUE
|
||||
private set
|
||||
|
||||
override fun start() {
|
||||
super.start()
|
||||
handler = Handler(looper) { msg ->
|
||||
try {
|
||||
when (msg.what) {
|
||||
MSG_INIT -> initEgl()
|
||||
MSG_DRAW -> drawAndSwap()
|
||||
MSG_SIZE_CHANGED -> sizeChanged(width, height)
|
||||
MSG_RELEASE -> {
|
||||
val ack = msg.obj as Ack
|
||||
ack.release()
|
||||
quit()
|
||||
synchronized(ack.latch) { (ack.latch as Object).notifyAll() }
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "GL thread error", e)
|
||||
releaseEgl()
|
||||
}
|
||||
true
|
||||
}
|
||||
handler.sendEmptyMessage(MSG_INIT)
|
||||
}
|
||||
|
||||
fun onSurfaceSizeChanged(width: Int, height: Int) {
|
||||
this.width = width
|
||||
this.height = height
|
||||
handler.sendEmptyMessage(MSG_SIZE_CHANGED)
|
||||
}
|
||||
|
||||
fun triggerDraw() {
|
||||
if (!::handler.isInitialized) return
|
||||
handler.removeMessages(MSG_DRAW)
|
||||
handler.sendEmptyMessage(MSG_DRAW)
|
||||
}
|
||||
|
||||
fun releaseAndWait() {
|
||||
if (!::handler.isInitialized) {
|
||||
quit()
|
||||
return
|
||||
}
|
||||
postShutdownAndWait(handler, MSG_RELEASE) { releaseEgl() }
|
||||
}
|
||||
|
||||
private fun initEgl() {
|
||||
try {
|
||||
eglDisplay = GlUtil.getDefaultEglDisplay()
|
||||
eglContext = GlUtil.createEglContext(eglDisplay)
|
||||
eglSurface = GlUtil.createEglSurface(eglDisplay, surface, C.COLOR_TRANSFER_SDR, false)
|
||||
EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext)
|
||||
renderer.onSurfaceCreated()
|
||||
// Resolve atlas dims from real GL caps (first-wins against the libass
|
||||
// thread's fallback) and allocate the texture once at those dims — uploads
|
||||
// are glTexSubImage2D of the packed rows from then on.
|
||||
val maxTexture = IntArray(1)
|
||||
GLES20.glGetIntegerv(GLES20.GL_MAX_TEXTURE_SIZE, maxTexture, 0)
|
||||
val (atlasW, atlasH) = resolveAtlasDims(maxTexture[0])
|
||||
renderer.allocateAtlasTexture(atlasW, atlasH)
|
||||
sizeChanged(width, height)
|
||||
} catch (e: GlUtil.GlException) {
|
||||
Log.e(TAG, "Failed to initialize EGL", e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun sizeChanged(width: Int, height: Int) {
|
||||
renderer.onSurfaceChanged(width, height)
|
||||
if (eglDisplay != EGL14.EGL_NO_DISPLAY) {
|
||||
GlUtil.clearFocusedBuffers()
|
||||
EGL14.eglSwapBuffers(eglDisplay, eglSurface)
|
||||
}
|
||||
}
|
||||
|
||||
private fun drawAndSwap() {
|
||||
if (eglDisplay == EGL14.EGL_NO_DISPLAY) return
|
||||
val payload = takePending() ?: return
|
||||
|
||||
// Render immediately (GL commands queue on the GPU). Re-upload only when the
|
||||
// slot's content actually changed — identity alone is not enough because the
|
||||
// libass side rewrites slot buffers in place (contentSeq tracks the rewrites).
|
||||
val t0 = System.nanoTime()
|
||||
val reuse = payload === lastUploadedPayload && payload.contentSeq == lastUploadedSeq
|
||||
renderer.onDrawFrame(payload, reuseUploads = reuse)
|
||||
lastUploadedPayload = payload
|
||||
lastUploadedSeq = payload.contentSeq
|
||||
val t1 = System.nanoTime()
|
||||
|
||||
// Swap immediately with the presentation time set: SurfaceFlinger holds the
|
||||
// queued buffer until the video frame's target release time, so the subtitle
|
||||
// can never appear early, and the GL thread is free again within a couple of
|
||||
// milliseconds. Sleeping here until near the target vsync (as a removed
|
||||
// TextureView path once required) made this thread blind for almost a whole
|
||||
// frame interval — the single-slot latest-wins handoff would then drop an
|
||||
// intermediate subtitle state (e.g. the transition to blank), showing the
|
||||
// previous state one frame too long.
|
||||
if (payload.releaseTimeNs != C.TIME_UNSET) {
|
||||
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, payload.releaseTimeNs)
|
||||
}
|
||||
EGL14.eglSwapBuffers(eglDisplay, eglSurface)
|
||||
val t2 = System.nanoTime()
|
||||
if (payload.releaseTimeNs != C.TIME_UNSET) {
|
||||
swapCount++
|
||||
val lateNs = t2 - payload.releaseTimeNs
|
||||
if (lateNs > LATE_THRESHOLD_NS) {
|
||||
lateSwapCount++
|
||||
val lateMs = lateNs / 1_000_000
|
||||
if (lateMs > maxLateMs) maxLateMs = lateMs
|
||||
}
|
||||
// Lead of changed-content swaps is the frame-perfection signal: ≥ 0 means
|
||||
// the new subtitle content reached the queue before the video frame's vsync.
|
||||
if (payload.contentSeq != lastSwappedSeq) {
|
||||
val leadMs = -lateNs / 1_000_000
|
||||
if (leadMs < minLeadChangedMs) minLeadChangedMs = leadMs
|
||||
}
|
||||
}
|
||||
lastSwappedSeq = payload.contentSeq
|
||||
if (AssAtlasPipelineConfig.TIMING_LOGS) {
|
||||
val pinned = payload.releaseTimeNs != C.TIME_UNSET
|
||||
// headroomMs: slack before the target vsync when GL STARTED; leadMs: slack
|
||||
// when the buffer was actually queued (negative = queued after the vsync).
|
||||
val headroomMs = if (pinned) (payload.releaseTimeNs - t0) / 1_000_000 else -1L
|
||||
val leadMs = if (pinned) (payload.releaseTimeNs - t2) / 1_000_000 else -1L
|
||||
Log.d(
|
||||
TAG,
|
||||
"swap pts=${payload.presentationTimeUs / 1000}ms seq=${payload.contentSeq} " +
|
||||
"quads=${payload.frame.quadCount} reused=$reuse drawMs=${(t1 - t0) / 1_000_000} " +
|
||||
"swapMs=${(t2 - t1) / 1_000_000} headroomMs=$headroomMs leadMs=$leadMs pinned=$pinned"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun releaseEgl() {
|
||||
if (eglDisplay != EGL14.EGL_NO_DISPLAY) {
|
||||
try {
|
||||
renderer.onSurfaceDestroyed()
|
||||
GlUtil.destroyEglSurface(eglDisplay, eglSurface)
|
||||
GlUtil.destroyEglContext(eglDisplay, eglContext)
|
||||
} catch (e: GlUtil.GlException) {
|
||||
Log.e(TAG, "Failed to release EGL", e)
|
||||
} finally {
|
||||
eglDisplay = EGL14.EGL_NO_DISPLAY
|
||||
eglContext = EGL14.EGL_NO_CONTEXT
|
||||
eglSurface = EGL14.EGL_NO_SURFACE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val TAG = "AssSurfaceGlThread"
|
||||
private const val MSG_INIT = 1
|
||||
private const val MSG_DRAW = 2
|
||||
private const val MSG_SIZE_CHANGED = 3
|
||||
private const val MSG_RELEASE = 4
|
||||
|
||||
/**
|
||||
* Swaps finishing this far past the target release time arrived after the
|
||||
* buffer should already have been queued and may miss the frame's vsync —
|
||||
* actual slack depends on the display's vsync offset from the release time.
|
||||
*/
|
||||
private const val LATE_THRESHOLD_NS = 4_000_000L
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* GL-side work for the atlas-based path. Maintains a single atlas texture and a
|
||||
* single vertex buffer; uploads them per frame (unless the payload identity
|
||||
* matches the last upload) and issues one `glDrawArrays` for the whole frame.
|
||||
*/
|
||||
@UnstableApi
|
||||
private class AtlasRenderer(private val assHandler: AssHandler) {
|
||||
|
||||
private val vertexShaderCode = """
|
||||
attribute vec2 a_Position;
|
||||
attribute vec2 a_TexCoord;
|
||||
attribute vec4 a_Color;
|
||||
uniform vec2 u_SurfaceSize;
|
||||
varying vec2 v_TexCoord;
|
||||
varying vec4 v_Color;
|
||||
void main() {
|
||||
vec2 clip = (a_Position / u_SurfaceSize) * 2.0 - 1.0;
|
||||
clip.y = -clip.y;
|
||||
gl_Position = vec4(clip, 0.0, 1.0);
|
||||
v_TexCoord = a_TexCoord;
|
||||
v_Color = a_Color;
|
||||
}
|
||||
""".trimIndent()
|
||||
|
||||
private val fragmentShaderCode = """
|
||||
precision mediump float;
|
||||
varying vec2 v_TexCoord;
|
||||
varying vec4 v_Color;
|
||||
uniform sampler2D u_Texture;
|
||||
void main() {
|
||||
float alpha = texture2D(u_Texture, v_TexCoord).a;
|
||||
gl_FragColor = v_Color * alpha;
|
||||
}
|
||||
""".trimIndent()
|
||||
|
||||
private var surfaceSize = Size.ZERO
|
||||
private lateinit var glProgram: GlProgram
|
||||
|
||||
private var atlasTexId = 0
|
||||
private var vertexBufferId = 0
|
||||
|
||||
private var aPosition = 0
|
||||
private var aTexCoord = 0
|
||||
private var aColor = 0
|
||||
private var uTexture = 0
|
||||
private var uSurfaceSize = 0
|
||||
|
||||
private var atlasAllocatedW = 0
|
||||
private var atlasAllocatedH = 0
|
||||
|
||||
/**
|
||||
* Allocates the atlas texture once at the resolved dims. The C side bakes UV
|
||||
* denominators = these dims into the vertex stream, so per-frame uploads can be
|
||||
* partial ([uploadAtlas]) without ever reallocating — drivers keep one stable
|
||||
* texture allocation instead of churning on packed-height changes.
|
||||
*/
|
||||
fun allocateAtlasTexture(width: Int, height: Int) {
|
||||
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, atlasTexId)
|
||||
GLES20.glTexImage2D(
|
||||
GLES20.GL_TEXTURE_2D, 0, GLES20.GL_ALPHA,
|
||||
width, height, 0,
|
||||
GLES20.GL_ALPHA, GLES20.GL_UNSIGNED_BYTE, null
|
||||
)
|
||||
atlasAllocatedW = width
|
||||
atlasAllocatedH = height
|
||||
}
|
||||
|
||||
fun onSurfaceCreated() {
|
||||
glProgram = GlProgram(vertexShaderCode, fragmentShaderCode)
|
||||
GlUtil.checkGlError()
|
||||
glProgram.use()
|
||||
|
||||
aPosition = glProgram.getAttributeArrayLocationAndEnable("a_Position")
|
||||
aTexCoord = glProgram.getAttributeArrayLocationAndEnable("a_TexCoord")
|
||||
aColor = glProgram.getAttributeArrayLocationAndEnable("a_Color")
|
||||
uTexture = glProgram.getUniformLocation("u_Texture")
|
||||
uSurfaceSize = glProgram.getUniformLocation("u_SurfaceSize")
|
||||
|
||||
val tex = IntArray(1)
|
||||
GLES20.glGenTextures(1, tex, 0)
|
||||
atlasTexId = tex[0]
|
||||
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
|
||||
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, atlasTexId)
|
||||
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE)
|
||||
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE)
|
||||
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR)
|
||||
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR)
|
||||
GLES20.glUniform1i(uTexture, 0)
|
||||
|
||||
val buf = IntArray(1)
|
||||
GLES20.glGenBuffers(1, buf, 0)
|
||||
vertexBufferId = buf[0]
|
||||
|
||||
GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1)
|
||||
GLES20.glEnable(GLES20.GL_BLEND)
|
||||
GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA)
|
||||
}
|
||||
|
||||
fun onSurfaceChanged(width: Int, height: Int) {
|
||||
surfaceSize = Size(width, height)
|
||||
assHandler.render?.setFrameSize(width, height)
|
||||
GLES20.glViewport(0, 0, width, height)
|
||||
GLES20.glUniform2f(uSurfaceSize, width.toFloat(), height.toFloat())
|
||||
}
|
||||
|
||||
fun onDrawFrame(payload: AtlasPayload, reuseUploads: Boolean) {
|
||||
GlUtil.clearFocusedBuffers()
|
||||
|
||||
val frame = payload.frame
|
||||
val quadCount = frame.quadCount
|
||||
if (quadCount == 0) return
|
||||
|
||||
if (!reuseUploads) {
|
||||
uploadAtlas(payload.atlasBuf, frame.atlasWidth, frame.atlasHeight)
|
||||
uploadVertices(payload.vertexBuf, quadCount)
|
||||
}
|
||||
|
||||
GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vertexBufferId)
|
||||
val stride = AssAtlasPipelineConfig.BYTES_PER_VERTEX
|
||||
GLES20.glVertexAttribPointer(aPosition, 2, GLES20.GL_FLOAT, false, stride, 0)
|
||||
GLES20.glVertexAttribPointer(aTexCoord, 2, GLES20.GL_FLOAT, false, stride, 8)
|
||||
GLES20.glVertexAttribPointer(aColor, 4, GLES20.GL_FLOAT, false, stride, 16)
|
||||
GLES20.glDrawArrays(GLES20.GL_TRIANGLES, 0, quadCount * 6)
|
||||
}
|
||||
|
||||
private fun uploadAtlas(atlasBuf: ByteBuffer, atlasW: Int, atlasH: Int) {
|
||||
atlasBuf.position(0).limit(atlasW * atlasH)
|
||||
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, atlasTexId)
|
||||
if (atlasW == atlasAllocatedW && atlasH <= atlasAllocatedH) {
|
||||
// Steady state: packed rows into the once-allocated texture.
|
||||
GLES20.glTexSubImage2D(
|
||||
GLES20.GL_TEXTURE_2D, 0, 0, 0, atlasW, atlasH,
|
||||
GLES20.GL_ALPHA, GLES20.GL_UNSIGNED_BYTE, atlasBuf
|
||||
)
|
||||
} else {
|
||||
// Defensive: dims disagree with the allocation (shouldn't happen — both
|
||||
// sides resolve dims through the same first-wins gate).
|
||||
Log.w("AssAtlasRenderer", "atlas upload ${atlasW}x$atlasH outside allocation ${atlasAllocatedW}x$atlasAllocatedH")
|
||||
GLES20.glTexImage2D(
|
||||
GLES20.GL_TEXTURE_2D, 0, GLES20.GL_ALPHA,
|
||||
atlasW, atlasH, 0,
|
||||
GLES20.GL_ALPHA, GLES20.GL_UNSIGNED_BYTE, atlasBuf
|
||||
)
|
||||
atlasAllocatedW = atlasW
|
||||
atlasAllocatedH = atlasH
|
||||
}
|
||||
}
|
||||
|
||||
private fun uploadVertices(vertexBuf: ByteBuffer, quadCount: Int) {
|
||||
val size = quadCount * AssAtlasPipelineConfig.BYTES_PER_QUAD
|
||||
vertexBuf.position(0).limit(size)
|
||||
GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vertexBufferId)
|
||||
GLES20.glBufferData(GLES20.GL_ARRAY_BUFFER, size, vertexBuf, GLES20.GL_STREAM_DRAW)
|
||||
}
|
||||
|
||||
fun onSurfaceDestroyed() {
|
||||
if (atlasTexId != 0) {
|
||||
val tex = intArrayOf(atlasTexId)
|
||||
GLES20.glDeleteTextures(1, tex, 0)
|
||||
atlasTexId = 0
|
||||
}
|
||||
if (vertexBufferId != 0) {
|
||||
val buf = intArrayOf(vertexBufferId)
|
||||
GLES20.glDeleteBuffers(1, buf, 0)
|
||||
vertexBufferId = 0
|
||||
}
|
||||
if (::glProgram.isInitialized) glProgram.delete()
|
||||
}
|
||||
}
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
package com.edde746.plezy.libass.media.widget
|
||||
|
||||
import android.app.ActivityManager
|
||||
import android.content.Context
|
||||
import android.graphics.PixelFormat
|
||||
import android.view.SurfaceHolder
|
||||
import android.view.SurfaceView
|
||||
import androidx.media3.common.util.UnstableApi
|
||||
import com.edde746.plezy.libass.media.AssHandler
|
||||
|
||||
/**
|
||||
* Subtitle overlay rendered through a dedicated [SurfaceView] layer.
|
||||
*
|
||||
* Uses a SurfaceFlinger layer directly, which lets the atlas-based pipeline
|
||||
* vsync-align its swap with the corresponding video frame via
|
||||
* `eglPresentationTimeANDROID`.
|
||||
*/
|
||||
@UnstableApi
|
||||
class AssSubtitleSurfaceView(
|
||||
context: Context,
|
||||
private val assHandler: AssHandler
|
||||
) : SurfaceView(context),
|
||||
SurfaceHolder.Callback {
|
||||
|
||||
private var pipeline: AssAtlasPipeline? = null
|
||||
|
||||
init {
|
||||
setZOrderMediaOverlay(true)
|
||||
holder.setFormat(PixelFormat.TRANSLUCENT)
|
||||
holder.addCallback(this)
|
||||
}
|
||||
|
||||
fun requestRender(presentationTimeUs: Long, releaseTimeNs: Long) {
|
||||
pipeline?.requestRender(presentationTimeUs, releaseTimeNs)
|
||||
}
|
||||
|
||||
/** Re-renders the last position, e.g. after margin changes while paused. */
|
||||
fun invalidateSubtitles() {
|
||||
pipeline?.invalidate()
|
||||
}
|
||||
|
||||
/** Vsync-pinned swaps performed by the current pipeline. */
|
||||
val swapCount: Long get() = pipeline?.swapCount ?: 0L
|
||||
|
||||
/** Pinned swaps that finished past the swap-time budget (possible missed vsync). */
|
||||
val lateSwapCount: Long get() = pipeline?.lateSwapCount ?: 0L
|
||||
|
||||
/** Worst observed swap lateness past the target release time, in milliseconds. */
|
||||
val maxLateMs: Long get() = pipeline?.maxLateMs ?: 0L
|
||||
|
||||
/** Total libass renders performed by the current pipeline. */
|
||||
val renderCount: Long get() = pipeline?.renderCount ?: 0L
|
||||
|
||||
/** Renders where libass reported changed content. */
|
||||
val changedRenderCount: Long get() = pipeline?.changedRenderCount ?: 0L
|
||||
|
||||
/** Renders that overflowed the atlas/vertex capacity. */
|
||||
val overflowCount: Long get() = pipeline?.overflowCount ?: 0L
|
||||
|
||||
/** Duration of the most recent libass render, in milliseconds. */
|
||||
val lastLibassMs: Long get() = pipeline?.lastLibassMs ?: 0L
|
||||
|
||||
/** Worst observed libass render duration, in milliseconds. */
|
||||
val maxLibassMs: Long get() = pipeline?.maxLibassMs ?: 0L
|
||||
|
||||
/** Changed-render duration histogram: [≤10ms, ≤25ms, ≤42ms, ≤84ms, >84ms]. */
|
||||
val libassMsHistogram: List<Long> get() = pipeline?.libassMsHistogram ?: emptyList()
|
||||
|
||||
/** Requests served from a pre-rendered (speculative) frame. */
|
||||
val specHits: Long get() = pipeline?.specHits ?: 0L
|
||||
|
||||
/** Requests where speculation existed but didn't match (seek, state change). */
|
||||
val specMisses: Long get() = pipeline?.specMisses ?: 0L
|
||||
|
||||
/** Speculation rounds skipped (paused, pending request, no confident cadence). */
|
||||
val specSkips: Long get() = pipeline?.specSkips ?: 0L
|
||||
|
||||
/** Cache-warming prefetch renders of upcoming events. */
|
||||
val prefetchCount: Long get() = pipeline?.prefetchCount ?: 0L
|
||||
|
||||
/** Minimum lead of changed-content pinned swaps vs the video frame's release
|
||||
* time, in ms (negative = late); null until one happened. */
|
||||
val minLeadChangedMs: Long? get() = pipeline?.minLeadChangedMs?.takeIf { it != Long.MAX_VALUE }
|
||||
|
||||
override fun surfaceCreated(holder: SurfaceHolder) {
|
||||
val rect = holder.surfaceFrame
|
||||
assHandler.setOverlaySurfaceSize(rect.width(), rect.height())
|
||||
val lowRam = (context.getSystemService(Context.ACTIVITY_SERVICE) as? ActivityManager)
|
||||
?.isLowRamDevice ?: false
|
||||
pipeline = AssAtlasPipeline(holder.surface, rect.width(), rect.height(), assHandler, lowRam)
|
||||
.also { it.start() }
|
||||
}
|
||||
|
||||
override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) {
|
||||
assHandler.setOverlaySurfaceSize(width, height)
|
||||
pipeline?.onSurfaceSizeChanged(width, height)
|
||||
}
|
||||
|
||||
override fun surfaceDestroyed(holder: SurfaceHolder) {
|
||||
pipeline?.releaseAndWait()
|
||||
pipeline = null
|
||||
}
|
||||
}
|
||||
+265
@@ -0,0 +1,265 @@
|
||||
package com.edde746.plezy.libass.media.widget
|
||||
|
||||
import com.edde746.plezy.libass.AssAtlasFrame
|
||||
import kotlin.math.abs
|
||||
|
||||
/**
|
||||
* Decision core of the speculative render-ahead pipeline.
|
||||
*
|
||||
* The problem it solves: ExoPlayer's frame-metadata callback gives only ~40 ms
|
||||
* between "video frame scheduled" and that frame's vsync, while a changed-content
|
||||
* libass render can take longer — so a render started at the callback lands a
|
||||
* frame (or more) late. The engine renders the *predicted next* PTS right after
|
||||
* servicing the current one, in the dead time between frames; when the next
|
||||
* request matches the prediction, only the GL upload+swap (~6–11 ms) sits inside
|
||||
* the deadline.
|
||||
*
|
||||
* Pure logic, single-threaded (the libass worker) — rendering, buffers and GL
|
||||
* are behind injected closures, so the state machine is unit-testable. Only the
|
||||
* stat fields are read from other threads.
|
||||
*
|
||||
* Slot ownership invariant: a slot handed to GL ([lastPostedSlot]) or currently
|
||||
* read by GL ([glTakenSlot]) is never chosen as a render target. With 3 slots and
|
||||
* at most 2 exclusions a target always exists; with 2 slots (low-RAM devices)
|
||||
* speculation must be disabled and the legacy alternation behavior falls out.
|
||||
*
|
||||
* Key subtlety inherited from libass: `ass_render_frame`'s `changed` flag compares
|
||||
* against libass's *previous render* — which, with speculation, may be content
|
||||
* that never reached the screen. The engine therefore tracks [libassLastSlot]
|
||||
* (the slot holding libass's most recent render output) rather than the last
|
||||
* *posted* slot, and prefers it as the render target: on `changed == 0` the
|
||||
* buffers were untouched and that slot already holds exactly the right content.
|
||||
*/
|
||||
internal class SpecRenderEngine(
|
||||
private val slotCount: Int,
|
||||
private val speculationEnabled: Boolean,
|
||||
/**
|
||||
* Renders the current track at [timeMs] into the slot's buffers. Returns null
|
||||
* when the renderer/track is gone. The closure owns timing/stat accounting.
|
||||
*/
|
||||
private val renderAt: (timeMs: Long, slot: Int) -> AssAtlasFrame?,
|
||||
/**
|
||||
* Renderer identity + state generation as one comparable value. Must change
|
||||
* whenever a speculatively rendered frame could be stale: margins, sizes,
|
||||
* track switches, renderer recreation.
|
||||
*/
|
||||
private val stateGeneration: () -> Long,
|
||||
/** Slot the GL thread most recently took for drawing, or -1. Never write it. */
|
||||
private val glTakenSlot: () -> Int,
|
||||
/** Diagnostic sink for hit/miss/skip decisions; null disables (no string cost). */
|
||||
private val debugLog: ((String) -> Unit)? = null
|
||||
) {
|
||||
|
||||
/** What the caller should do after [service]. */
|
||||
sealed class Outcome {
|
||||
/** Nothing to post (no renderer, or no rendered content exists yet). */
|
||||
object Skip : Outcome()
|
||||
|
||||
/**
|
||||
* Post [slot] to GL. [newContent] means this call wrote fresh content into
|
||||
* the slot (the caller bumps the payload's content seq). [specHit] means the
|
||||
* content was pre-rendered — no libass render ran inside the deadline.
|
||||
*/
|
||||
class Post(val slot: Int, val frame: AssAtlasFrame, val newContent: Boolean, val specHit: Boolean) : Outcome()
|
||||
}
|
||||
|
||||
/** A speculative render that wrote new content into [slot] (bump its seq). */
|
||||
class SpecWrite(val slot: Int, val frame: AssAtlasFrame)
|
||||
|
||||
// Speculation state: content for specPtsUs is pre-rendered and lives either in
|
||||
// specSlot, or — when the spec render returned changed == 0 — is identical to
|
||||
// libassLastSlot's content (specIsLibassLast).
|
||||
private var specPtsUs = UNSET
|
||||
private var specSlot = -1
|
||||
private var specIsLibassLast = false
|
||||
private var specGen = 0L
|
||||
|
||||
// The slot holding libass's most recent render output and its frame.
|
||||
private var libassLastSlot = -1
|
||||
private var libassLastFrame: AssAtlasFrame? = null
|
||||
private var lastPostedSlot = -1
|
||||
|
||||
// Request-cadence estimator over pinned (playing) requests: median of the last
|
||||
// 8 PTS deltas, valid after 4, reset on any non-monotonic or > 250 ms jump.
|
||||
private val deltas = LongArray(DELTA_SAMPLES)
|
||||
private var deltaCount = 0
|
||||
private var deltaIndex = 0
|
||||
private var lastPinnedPtsUs = UNSET
|
||||
|
||||
// Stats; single-writer (the libass thread), read from the stats path.
|
||||
@Volatile
|
||||
var specHits = 0L
|
||||
private set
|
||||
|
||||
@Volatile
|
||||
var specMisses = 0L
|
||||
private set
|
||||
|
||||
@Volatile
|
||||
var specSkips = 0L
|
||||
private set
|
||||
|
||||
/**
|
||||
* Services a render request for [ptsUs]. [pinned] is false for invalidate
|
||||
* repaints (paused margin changes etc.), which never feed the cadence
|
||||
* estimator and never count as speculation misses against playback.
|
||||
*/
|
||||
fun service(ptsUs: Long, pinned: Boolean): Outcome {
|
||||
if (pinned) updateDeltaEstimator(ptsUs)
|
||||
|
||||
if (specPtsUs != UNSET) {
|
||||
val eps = epsilonUs()
|
||||
val genNow = stateGeneration()
|
||||
val hit = genNow == specGen && eps > 0 && abs(ptsUs - specPtsUs) <= eps
|
||||
val slot = if (specIsLibassLast) libassLastSlot else specSlot
|
||||
val frame = libassLastFrame
|
||||
val specPts = specPtsUs
|
||||
specPtsUs = UNSET
|
||||
if (hit && slot >= 0 && frame != null) {
|
||||
specHits++
|
||||
lastPostedSlot = slot
|
||||
debugLog?.invoke("hit pts=${ptsUs / 1000}ms spec=${specPts / 1000}ms d=${(ptsUs - specPts) / 1000}ms slot=$slot")
|
||||
return Outcome.Post(slot, frame, newContent = false, specHit = true)
|
||||
}
|
||||
specMisses++
|
||||
debugLog?.invoke(
|
||||
"miss pts=${ptsUs / 1000}ms spec=${specPts / 1000}ms d=${(ptsUs - specPts) / 1000}ms eps=${eps / 1000}ms " +
|
||||
"gen=${if (genNow == specGen) "ok" else "CHANGED"} slot=$slot frame=${frame != null}"
|
||||
)
|
||||
} else {
|
||||
debugLog?.invoke("no-spec pts=${ptsUs / 1000}ms")
|
||||
}
|
||||
|
||||
// On-demand render. Preferring libassLastSlot as the target makes changed == 0
|
||||
// unambiguous: the buffers were untouched and already hold the right content.
|
||||
val target = renderTargetSlot() ?: return Outcome.Skip
|
||||
val frame = renderAt(ptsUs / 1000, target) ?: return Outcome.Skip
|
||||
if (frame.changed == 0) {
|
||||
val lastSlot = libassLastSlot
|
||||
val lastFrame = libassLastFrame ?: return Outcome.Skip
|
||||
if (lastSlot < 0) return Outcome.Skip
|
||||
lastPostedSlot = lastSlot
|
||||
return Outcome.Post(lastSlot, lastFrame, newContent = false, specHit = false)
|
||||
}
|
||||
libassLastSlot = target
|
||||
libassLastFrame = frame
|
||||
lastPostedSlot = target
|
||||
return Outcome.Post(target, frame, newContent = true, specHit = false)
|
||||
}
|
||||
|
||||
/**
|
||||
* Speculatively renders the predicted next request ([servicedPtsUs] + median
|
||||
* delta) into a free slot. Call after posting the current frame; skipped while
|
||||
* paused ([pinned] false), when a newer request is already waiting, or while
|
||||
* the cadence estimator has no confident delta.
|
||||
*/
|
||||
fun speculateAfter(servicedPtsUs: Long, pinned: Boolean, hasPending: Boolean): SpecWrite? {
|
||||
if (!speculationEnabled) return null
|
||||
if (!pinned || hasPending || !deltaValid()) {
|
||||
specSkips++
|
||||
debugLog?.invoke(
|
||||
"spec-skip after=${servicedPtsUs / 1000}ms pinned=$pinned pending=$hasPending cadence=${deltaValid()}"
|
||||
)
|
||||
return null
|
||||
}
|
||||
val target = renderTargetSlot() ?: run {
|
||||
specSkips++
|
||||
debugLog?.invoke("spec-skip after=${servicedPtsUs / 1000}ms no-free-slot")
|
||||
return null
|
||||
}
|
||||
val gen = stateGeneration()
|
||||
val specPts = servicedPtsUs + medianDeltaUs()
|
||||
val frame = renderAt(specPts / 1000, target) ?: run {
|
||||
specSkips++
|
||||
return null
|
||||
}
|
||||
specGen = gen
|
||||
specPtsUs = specPts
|
||||
if (frame.changed == 0) {
|
||||
// Content at specPts is identical to libass's last render — nothing was
|
||||
// written; a hit will repost libassLastSlot (and GL will skip the upload).
|
||||
specIsLibassLast = true
|
||||
specSlot = -1
|
||||
return null
|
||||
}
|
||||
libassLastSlot = target
|
||||
libassLastFrame = frame
|
||||
specIsLibassLast = false
|
||||
specSlot = target
|
||||
return SpecWrite(target, frame)
|
||||
}
|
||||
|
||||
/**
|
||||
* Pre-renders [ptsUs] (an upcoming event's start) purely to warm the
|
||||
* renderer's glyph/bitmap caches before that content is actually needed —
|
||||
* heavy typesetting otherwise pays its cache-cold rasterization (measured
|
||||
* 0.8–2.6 s on weak devices) exactly when the sign appears. The result is
|
||||
* never posted; like any render it becomes libass's last-rendered content,
|
||||
* so any pending speculation is invalidated first (its slot/content
|
||||
* references would go stale).
|
||||
*
|
||||
* Returns the slot written (caller bumps its content seq) or null when
|
||||
* nothing was rendered.
|
||||
*/
|
||||
fun prefetch(ptsUs: Long): SpecWrite? {
|
||||
specPtsUs = UNSET
|
||||
val target = renderTargetSlot() ?: return null
|
||||
val frame = renderAt(ptsUs / 1000, target) ?: return null
|
||||
prefetchCount++
|
||||
if (frame.changed == 0) return null
|
||||
libassLastSlot = target
|
||||
libassLastFrame = frame
|
||||
return SpecWrite(target, frame)
|
||||
}
|
||||
|
||||
/** Cache-warming prefetch renders performed; single-writer, read cross-thread. */
|
||||
@Volatile
|
||||
var prefetchCount = 0L
|
||||
private set
|
||||
|
||||
private fun renderTargetSlot(): Int? {
|
||||
// The GL exclusion only exists in ≥3-slot mode; with 2 slots this reduces to
|
||||
// the legacy "don't write the posted slot" alternation.
|
||||
val taken = if (slotCount > 2) glTakenSlot() else -1
|
||||
val last = libassLastSlot
|
||||
if (last >= 0 && last != lastPostedSlot && last != taken) return last
|
||||
for (s in 0 until slotCount) {
|
||||
if (s != lastPostedSlot && s != taken) return s
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private fun updateDeltaEstimator(ptsUs: Long) {
|
||||
val prev = lastPinnedPtsUs
|
||||
lastPinnedPtsUs = ptsUs
|
||||
if (prev == UNSET) return
|
||||
val d = ptsUs - prev
|
||||
if (d <= 0 || d > MAX_DELTA_US) {
|
||||
// Seek/discontinuity (or duplicate PTS): forget the cadence.
|
||||
deltaCount = 0
|
||||
deltaIndex = 0
|
||||
return
|
||||
}
|
||||
deltas[deltaIndex] = d
|
||||
deltaIndex = (deltaIndex + 1) % DELTA_SAMPLES
|
||||
if (deltaCount < DELTA_SAMPLES) deltaCount++
|
||||
}
|
||||
|
||||
private fun deltaValid() = deltaCount >= MIN_DELTA_SAMPLES
|
||||
|
||||
private fun medianDeltaUs(): Long {
|
||||
val copy = deltas.copyOfRange(0, deltaCount)
|
||||
copy.sort()
|
||||
return copy[deltaCount / 2]
|
||||
}
|
||||
|
||||
private fun epsilonUs(): Long = if (deltaValid()) minOf(medianDeltaUs() / 2, EPSILON_CAP_US) else 0L
|
||||
|
||||
private companion object {
|
||||
const val UNSET = Long.MIN_VALUE
|
||||
const val DELTA_SAMPLES = 8
|
||||
const val MIN_DELTA_SAMPLES = 4
|
||||
const val MAX_DELTA_US = 250_000L
|
||||
const val EPSILON_CAP_US = 8_000L
|
||||
}
|
||||
}
|
||||
+332
@@ -0,0 +1,332 @@
|
||||
package com.edde746.plezy.libass.media.widget
|
||||
|
||||
import com.edde746.plezy.libass.AssAtlasFrame
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertFalse
|
||||
import org.junit.Assert.assertNotNull
|
||||
import org.junit.Assert.assertNull
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
|
||||
/**
|
||||
* Tests for the speculative render-ahead state machine. The render function is
|
||||
* scripted: each call records (timeMs, slot) and returns the next queued frame.
|
||||
*/
|
||||
class SpecRenderEngineTest {
|
||||
|
||||
private class RenderCall(val timeMs: Long, val slot: Int)
|
||||
|
||||
private class Harness(
|
||||
slotCount: Int = 3,
|
||||
speculationEnabled: Boolean = true
|
||||
) {
|
||||
val calls = mutableListOf<RenderCall>()
|
||||
val script = ArrayDeque<AssAtlasFrame?>()
|
||||
var generation = 0L
|
||||
var glTaken = -1
|
||||
|
||||
val engine = SpecRenderEngine(
|
||||
slotCount = slotCount,
|
||||
speculationEnabled = speculationEnabled,
|
||||
renderAt = { timeMs, slot ->
|
||||
calls.add(RenderCall(timeMs, slot))
|
||||
if (script.isEmpty()) changed() else script.removeFirst()
|
||||
},
|
||||
stateGeneration = { generation },
|
||||
glTakenSlot = { glTaken }
|
||||
)
|
||||
|
||||
/** Primes the cadence estimator with [n] steady requests; returns last pts. */
|
||||
fun prime(n: Int = 5, startUs: Long = 0L, deltaUs: Long = 42_000L): Long {
|
||||
var pts = startUs
|
||||
repeat(n) {
|
||||
engine.service(pts, pinned = true)
|
||||
engine.speculateAfter(pts, pinned = true, hasPending = false)
|
||||
pts += deltaUs
|
||||
}
|
||||
return pts - deltaUs
|
||||
}
|
||||
}
|
||||
|
||||
private companion object {
|
||||
const val DELTA = 42_000L
|
||||
|
||||
fun changed(quads: Int = 5) = AssAtlasFrame(2048, 100, quads, 2, 0)
|
||||
|
||||
fun unchanged() = AssAtlasFrame(0, 0, 0, 0, 0)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `steady state hits serve without rendering`() {
|
||||
val h = Harness()
|
||||
val last = h.prime()
|
||||
|
||||
// Cadence is confident by now: the last speculateAfter pre-rendered last+Δ.
|
||||
val before = h.calls.size
|
||||
val outcome = h.engine.service(last + DELTA, pinned = true)
|
||||
|
||||
assertTrue(outcome is SpecRenderEngine.Outcome.Post)
|
||||
outcome as SpecRenderEngine.Outcome.Post
|
||||
assertTrue(outcome.specHit)
|
||||
assertFalse(outcome.newContent) // content was written (and seq-bumped) at spec time
|
||||
assertEquals("service must not render on a hit", before, h.calls.size)
|
||||
assertEquals(1L, h.engine.specHits)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `hit tolerates jitter within epsilon`() {
|
||||
val h = Harness()
|
||||
val last = h.prime()
|
||||
|
||||
val outcome = h.engine.service(last + DELTA + 3_000, pinned = true)
|
||||
|
||||
assertTrue((outcome as SpecRenderEngine.Outcome.Post).specHit)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `seek misses and renders on demand into the spec slot`() {
|
||||
val h = Harness()
|
||||
val last = h.prime()
|
||||
val specSlot = h.calls.last().slot // where the speculative content went
|
||||
|
||||
val before = h.calls.size
|
||||
h.script.add(changed())
|
||||
val outcome = h.engine.service(last + 1_000_000, pinned = true)
|
||||
|
||||
assertTrue(outcome is SpecRenderEngine.Outcome.Post)
|
||||
outcome as SpecRenderEngine.Outcome.Post
|
||||
assertFalse(outcome.specHit)
|
||||
assertTrue(outcome.newContent)
|
||||
assertEquals(before + 1, h.calls.size)
|
||||
assertEquals("miss must render into the slot holding libass's last content", specSlot, h.calls.last().slot)
|
||||
assertEquals(1L, h.engine.specMisses)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `miss with changed 0 posts the spec slot not stale screen content`() {
|
||||
// The trap: spec rendered NEW content (never posted); a mismatched request
|
||||
// then returns changed == 0 (identical to libass's LAST render = the spec
|
||||
// content). Posting "last posted" content would be stale — the engine must
|
||||
// post the slot libass last wrote.
|
||||
val h = Harness()
|
||||
val last = h.prime()
|
||||
val specSlot = h.calls.last().slot
|
||||
|
||||
h.script.add(unchanged())
|
||||
val outcome = h.engine.service(last + 200_000, pinned = true) // > ε, miss; estimator resets too
|
||||
|
||||
assertTrue(outcome is SpecRenderEngine.Outcome.Post)
|
||||
outcome as SpecRenderEngine.Outcome.Post
|
||||
assertFalse(outcome.specHit)
|
||||
assertFalse(outcome.newContent)
|
||||
assertEquals("changed==0 must repost libass's last-rendered slot", specSlot, outcome.slot)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `state generation change invalidates speculation`() {
|
||||
val h = Harness()
|
||||
val last = h.prime()
|
||||
|
||||
h.generation++ // margins/zoom/track switch between spec render and the request
|
||||
h.script.add(changed())
|
||||
val before = h.calls.size
|
||||
val outcome = h.engine.service(last + DELTA, pinned = true)
|
||||
|
||||
assertTrue(outcome is SpecRenderEngine.Outcome.Post)
|
||||
assertFalse((outcome as SpecRenderEngine.Outcome.Post).specHit)
|
||||
assertEquals("stale spec must be re-rendered", before + 1, h.calls.size)
|
||||
assertEquals(1L, h.engine.specMisses)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `unpinned requests never feed the estimator or speculate`() {
|
||||
val h = Harness()
|
||||
val last = h.prime()
|
||||
|
||||
// Paused invalidate-repaint at an arbitrary position.
|
||||
h.script.add(changed())
|
||||
h.engine.service(last, pinned = false)
|
||||
val specBefore = h.calls.size
|
||||
assertNull(h.engine.speculateAfter(last, pinned = false, hasPending = false))
|
||||
assertEquals("unpinned must not speculate", specBefore, h.calls.size)
|
||||
|
||||
// Cadence survives the unpinned request: the next pinned pair still hits.
|
||||
h.engine.service(last + DELTA, pinned = true)
|
||||
h.engine.speculateAfter(last + DELTA, pinned = true, hasPending = false)
|
||||
val outcome = h.engine.service(last + 2 * DELTA, pinned = true)
|
||||
assertTrue((outcome as SpecRenderEngine.Outcome.Post).specHit)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `speculation skipped while a request is pending`() {
|
||||
val h = Harness()
|
||||
val last = h.prime()
|
||||
val before = h.calls.size
|
||||
|
||||
assertNull(h.engine.speculateAfter(last, pinned = true, hasPending = true))
|
||||
assertEquals(before, h.calls.size)
|
||||
assertTrue(h.engine.specSkips > 0)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `no speculation until cadence is confident`() {
|
||||
val h = Harness()
|
||||
h.engine.service(0, pinned = true)
|
||||
// Only one delta sample so far (needs 4).
|
||||
h.engine.service(DELTA, pinned = true)
|
||||
val before = h.calls.size
|
||||
assertNull(h.engine.speculateAfter(DELTA, pinned = true, hasPending = false))
|
||||
assertEquals(before, h.calls.size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `render target avoids posted and gl taken slots`() {
|
||||
val h = Harness()
|
||||
h.script.add(changed())
|
||||
val first = h.engine.service(0, pinned = true) as SpecRenderEngine.Outcome.Post
|
||||
val posted = first.slot
|
||||
|
||||
h.glTaken = (posted + 1) % 3
|
||||
h.script.add(changed())
|
||||
val second = h.engine.service(1_000_000, pinned = true) as SpecRenderEngine.Outcome.Post
|
||||
|
||||
val expected = (0 until 3).first { it != posted && it != h.glTaken }
|
||||
assertEquals(expected, second.slot)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `changed 0 before any content skips`() {
|
||||
val h = Harness()
|
||||
h.script.add(unchanged())
|
||||
assertEquals(SpecRenderEngine.Outcome.Skip, h.engine.service(0, pinned = true))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `renderer gone skips`() {
|
||||
val h = Harness()
|
||||
h.script.add(null)
|
||||
assertEquals(SpecRenderEngine.Outcome.Skip, h.engine.service(0, pinned = true))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `two slot mode alternates and never speculates`() {
|
||||
val h = Harness(slotCount = 2, speculationEnabled = false)
|
||||
h.script.add(changed())
|
||||
val a = h.engine.service(0, pinned = true) as SpecRenderEngine.Outcome.Post
|
||||
assertNull(h.engine.speculateAfter(0, pinned = true, hasPending = false))
|
||||
|
||||
h.script.add(changed())
|
||||
val b = h.engine.service(DELTA, pinned = true) as SpecRenderEngine.Outcome.Post
|
||||
assertTrue(a.slot != b.slot)
|
||||
assertTrue(a.slot in 0..1 && b.slot in 0..1)
|
||||
assertEquals(0, h.calls.count { it.slot >= 2 })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `static dialogue hit reposts the same slot without new content`() {
|
||||
// Spec render returns changed == 0 (nothing moves): a hit must repost the
|
||||
// last-rendered slot so GL skips the upload entirely.
|
||||
val h = Harness()
|
||||
h.script.add(changed())
|
||||
val first = h.engine.service(0, pinned = true) as SpecRenderEngine.Outcome.Post
|
||||
var pts = 0L
|
||||
repeat(4) { // build cadence; renders return changed for simplicity
|
||||
pts += DELTA
|
||||
h.engine.service(pts, pinned = true)
|
||||
}
|
||||
h.script.add(unchanged()) // speculative render: nothing changes at pts+Δ
|
||||
assertNull(h.engine.speculateAfter(pts, pinned = true, hasPending = false))
|
||||
|
||||
val before = h.calls.size
|
||||
val outcome = h.engine.service(pts + DELTA, pinned = true) as SpecRenderEngine.Outcome.Post
|
||||
assertTrue(outcome.specHit)
|
||||
assertFalse(outcome.newContent)
|
||||
assertEquals(before, h.calls.size)
|
||||
assertNotNull(first) // first slot existed; hit reposts whichever slot was last rendered
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `speculative write reports slot for seq bump`() {
|
||||
val h = Harness()
|
||||
val last = h.prime()
|
||||
h.script.add(changed())
|
||||
val write = h.engine.speculateAfter(last, pinned = true, hasPending = false)
|
||||
assertNotNull(write)
|
||||
assertEquals(h.calls.last().slot, write!!.slot)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `spec predicts pts plus median delta`() {
|
||||
val h = Harness()
|
||||
val last = h.prime()
|
||||
h.script.add(changed())
|
||||
h.engine.speculateAfter(last, pinned = true, hasPending = false)
|
||||
assertEquals((last + DELTA) / 1000, h.calls.last().timeMs)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `prefetch renders the requested future pts and reports its slot`() {
|
||||
val h = Harness()
|
||||
h.script.add(changed())
|
||||
h.engine.service(0, pinned = true) // some content on screen
|
||||
|
||||
h.script.add(changed())
|
||||
val write = h.engine.prefetch(5_000_000)
|
||||
|
||||
assertNotNull(write)
|
||||
assertEquals(5_000L, h.calls.last().timeMs)
|
||||
assertEquals(h.calls.last().slot, write!!.slot)
|
||||
assertEquals(1L, h.engine.prefetchCount)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `prefetch invalidates pending speculation`() {
|
||||
// Prefetch rewrites libass's last-rendered content (and possibly the spec
|
||||
// slot itself); a stale spec hit would post future-event content.
|
||||
val h = Harness()
|
||||
val last = h.prime() // leaves a valid spec for last+Δ
|
||||
|
||||
h.script.add(changed())
|
||||
h.engine.prefetch(last + 5_000_000)
|
||||
|
||||
val before = h.calls.size
|
||||
h.script.add(changed())
|
||||
val outcome = h.engine.service(last + DELTA, pinned = true)
|
||||
|
||||
assertTrue(outcome is SpecRenderEngine.Outcome.Post)
|
||||
assertFalse((outcome as SpecRenderEngine.Outcome.Post).specHit)
|
||||
assertEquals("post-prefetch request must render on demand", before + 1, h.calls.size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `render after prefetch with changed 0 posts the prefetched slot`() {
|
||||
// changed==0 after a prefetch means "identical to libass's last render" =
|
||||
// the prefetched content — correct exactly when playback reached the
|
||||
// prefetched event. The engine must post that slot, not older content.
|
||||
val h = Harness()
|
||||
h.script.add(changed())
|
||||
h.engine.service(0, pinned = true)
|
||||
|
||||
h.script.add(changed())
|
||||
val write = h.engine.prefetch(5_000_000)!!
|
||||
|
||||
h.script.add(unchanged())
|
||||
val outcome = h.engine.service(5_000_000, pinned = true) as SpecRenderEngine.Outcome.Post
|
||||
assertEquals(write.slot, outcome.slot)
|
||||
assertFalse(outcome.newContent)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `prefetch avoids posted and gl taken slots`() {
|
||||
val h = Harness()
|
||||
h.script.add(changed())
|
||||
val posted = (h.engine.service(0, pinned = true) as SpecRenderEngine.Outcome.Post).slot
|
||||
h.glTaken = (posted + 1) % 3
|
||||
|
||||
h.script.add(changed())
|
||||
val write = h.engine.prefetch(5_000_000)
|
||||
|
||||
val expected = (0 until 3).first { it != posted && it != h.glTaken }
|
||||
assertEquals(expected, write!!.slot)
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,11 @@ pluginManagement {
|
||||
plugins {
|
||||
id("dev.flutter.flutter-plugin-loader") version "1.0.0"
|
||||
id("com.android.application") version "8.9.1" apply false
|
||||
id("com.android.library") version "8.9.1" apply false
|
||||
id("org.jetbrains.kotlin.android") version "2.1.0" apply false
|
||||
}
|
||||
|
||||
include(":app")
|
||||
// libass subtitle module (Kotlin/JNI bindings + Media3 glue); the native libass
|
||||
// core stays the upstream Maven artifact io.github.peerless2012:ass.
|
||||
include(":libass")
|
||||
|
||||
Executable
+143
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env bash
|
||||
# Generates MKV test assets that prove (or disprove) frame-perfect ASS rendering:
|
||||
# the video has a per-frame counter burned in (top) and the MKV carries an ASS
|
||||
# track flipping the same counter (bottom) on exactly the same frame boundaries.
|
||||
# Any captured frame showing video number N with subtitle number != N is a sync
|
||||
# failure.
|
||||
#
|
||||
# Outputs (in scripts/framesync/):
|
||||
# framesync-2397.mkv 23.976 fps, plain per-frame counter events
|
||||
# framesync-60.mkv 60 fps variant
|
||||
# framesync-stress.mkv 23.976 fps + heavy animated typesetting (blur/scale/
|
||||
# rotation re-rendered every frame) to stress libass
|
||||
#
|
||||
# The burn-in uses, in order of preference: ffmpeg drawtext, ffmpeg subtitles
|
||||
# filter, or mpv's encoding mode (mpv always bundles libass) — all three
|
||||
# evaluate the counter at each frame's exact PTS, so the burned reference is
|
||||
# frame-exact by construction.
|
||||
#
|
||||
# Verification procedure (Android, ExoPlayer path):
|
||||
# 1. Play the file with the ASS track selected (SDR content and the
|
||||
# ASS-tunneling block mean the layers are screen-recordable).
|
||||
# 2. adb shell screenrecord /sdcard/sync.mp4 (record ~30 s, pull it)
|
||||
# 3. Step through frames (e.g. ffmpeg -i sync.mp4 frames/%05d.png, or mpv with
|
||||
# '.' frame-step): top (video) and bottom (player-rendered ASS) counters
|
||||
# must match on EVERY captured frame. Desktop mpv is the reference player.
|
||||
# 4. Cross-check at the compositor: while playing,
|
||||
# adb shell dumpsys SurfaceFlinger --latency
|
||||
# lists per-layer (desired, actual) present times for the video SurfaceView
|
||||
# and the ASS overlay layer — matched content must share vsync timestamps.
|
||||
# 5. Cheap regression: the in-app stats overlay should show subLateSwaps ≈ 0,
|
||||
# subOverflows == 0, subMinLeadMs ≥ 0 and a >95% subSpecHits ratio.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")"
|
||||
mkdir -p framesync
|
||||
cd framesync
|
||||
|
||||
DURATION_S=${DURATION_S:-60}
|
||||
# drawtext resolves the family via fontconfig; override with FONT if needed.
|
||||
FONT=${FONT:-Sans}
|
||||
|
||||
have_filter() { ffmpeg -hide_banner -filters 2>/dev/null | grep -q " $1 "; }
|
||||
|
||||
# Emits an ASS file with one Dialogue event per frame. Frame i starts at
|
||||
# i*den/num seconds; timestamps are floored to ASS centisecond precision, which
|
||||
# is safe because the floor error (<10 ms) is smaller than any frame interval
|
||||
# generated here (16.7 / 41.7 ms) — an event can never leak onto the previous
|
||||
# frame. burn=1 emits the top-aligned white reference style (for burn-in),
|
||||
# burn=0 the bottom-aligned yellow player style; heavy=1 adds per-frame
|
||||
# animated blur/scale/rotation events.
|
||||
gen_ass() { # $1=fps_num $2=fps_den $3=frames $4=burn(0/1) $5=heavy(0/1) $6=outfile
|
||||
awk -v num="$1" -v den="$2" -v frames="$3" -v burn="$4" -v heavy="$5" '
|
||||
function ts(cs, h, m, s) {
|
||||
h = int(cs / 360000); cs -= h * 360000
|
||||
m = int(cs / 6000); cs -= m * 6000
|
||||
s = int(cs / 100); cs -= s * 100
|
||||
return sprintf("%d:%02d:%02d.%02d", h, m, s, cs)
|
||||
}
|
||||
BEGIN {
|
||||
print "[Script Info]"
|
||||
print "ScriptType: v4.00+"
|
||||
print "PlayResX: 1280"
|
||||
print "PlayResY: 720"
|
||||
print ""
|
||||
print "[V4+ Styles]"
|
||||
print "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding"
|
||||
if (burn)
|
||||
print "Style: Counter,Arial,150,&H00FFFFFF,&H000000FF,&H00000000,&H00000000,-1,0,0,0,100,100,0,0,1,5,0,8,10,10,30,1"
|
||||
else
|
||||
print "Style: Counter,Arial,150,&H0000FFFF,&H000000FF,&H00000000,&H00000000,-1,0,0,0,100,100,0,0,1,5,0,2,10,10,30,1"
|
||||
print "Style: Stress,Arial,80,&H40FF8800,&H000000FF,&H00000000,&H00000000,-1,0,0,0,100,100,0,0,1,2,0,5,10,10,10,1"
|
||||
print ""
|
||||
print "[Events]"
|
||||
print "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text"
|
||||
for (i = 0; i < frames; i++) {
|
||||
start = int(i * den * 100 / num)
|
||||
end = int((i + 1) * den * 100 / num)
|
||||
if (end <= start) end = start + 1
|
||||
printf "Dialogue: 0,%s,%s,Counter,,0,0,0,,%d\n", ts(start), ts(end), i
|
||||
if (heavy) {
|
||||
# Re-emitted every frame with a frame-dependent rotation/blur so libass
|
||||
# reports changed content and re-rasterizes large blurred glyphs —
|
||||
# approximates typesetting storms (the measured 100-400 ms frames).
|
||||
printf "Dialogue: 1,%s,%s,Stress,,0,0,0,,{\\an5\\pos(640,260)\\blur12\\fscx320\\fscy320\\frz%d}#\n", ts(start), ts(end), (i * 7) % 360
|
||||
printf "Dialogue: 1,%s,%s,Stress,,0,0,0,,{\\an5\\pos(640,500)\\blur18\\fscx500\\fscy220\\frz%d\\alpha&H80&}@\n", ts(start), ts(end), 359 - (i * 11) % 360
|
||||
}
|
||||
}
|
||||
}
|
||||
' /dev/null > "$6"
|
||||
}
|
||||
|
||||
# Produces the reference video: gray frames with the frame counter burned in at
|
||||
# the top. Tries drawtext, then the subtitles filter, then mpv encoding.
|
||||
gen_video() { # $1=rate $2=burn_ass $3=outfile
|
||||
if have_filter drawtext; then
|
||||
ffmpeg -y -v error -f lavfi -i "color=c=0x202020:s=1280x720:r=$1" \
|
||||
-vf "drawtext=font='${FONT}':text='%{n}':fontsize=150:fontcolor=white:borderw=5:bordercolor=black:x=(w-text_w)/2:y=30" \
|
||||
-t "$DURATION_S" -c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p "$3"
|
||||
elif have_filter subtitles; then
|
||||
ffmpeg -y -v error -f lavfi -i "color=c=0x202020:s=1280x720:r=$1" \
|
||||
-vf "subtitles=$2" \
|
||||
-t "$DURATION_S" -c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p "$3"
|
||||
elif command -v mpv > /dev/null; then
|
||||
ffmpeg -y -v error -f lavfi -i "color=c=0x202020:s=1280x720:r=$1" \
|
||||
-t "$DURATION_S" -c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p blank.mp4
|
||||
mpv blank.mp4 --sub-files="$2" --vf=sub --no-audio \
|
||||
--o="$3" --of=mp4 --ovc=libx264 --ovcopts=preset=veryfast,crf=18 \
|
||||
--msg-level=all=error
|
||||
rm -f blank.mp4
|
||||
else
|
||||
echo "error: need ffmpeg with drawtext or subtitles filter, or mpv (for the burn-in)" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Muxes the burned video + silent audio + the player-rendered ASS track.
|
||||
mux() { # $1=video $2=ass $3=outfile
|
||||
ffmpeg -y -v error -i "$1" -f lavfi -i "anullsrc=r=48000:cl=stereo" -i "$2" \
|
||||
-map 0:v -map 1:a -map 2 -c:v copy -c:a aac -b:a 64k -c:s copy -shortest \
|
||||
-metadata:s:s:0 language=eng -disposition:s:0 default "$3"
|
||||
}
|
||||
|
||||
frames_2397=$(awk -v d="$DURATION_S" 'BEGIN { print int(d * 24000 / 1001) }')
|
||||
frames_60=$(awk -v d="$DURATION_S" 'BEGIN { print d * 60 }')
|
||||
|
||||
echo "Generating ${DURATION_S}s assets..."
|
||||
gen_ass 24000 1001 "$frames_2397" 1 0 burn-2397.ass
|
||||
gen_ass 60 1 "$frames_60" 1 0 burn-60.ass
|
||||
gen_ass 24000 1001 "$frames_2397" 0 0 counter-2397.ass
|
||||
gen_ass 60 1 "$frames_60" 0 0 counter-60.ass
|
||||
gen_ass 24000 1001 "$frames_2397" 0 1 counter-stress.ass
|
||||
gen_video "24000/1001" burn-2397.ass video-2397.mp4
|
||||
gen_video "60" burn-60.ass video-60.mp4
|
||||
mux video-2397.mp4 counter-2397.ass framesync-2397.mkv
|
||||
mux video-60.mp4 counter-60.ass framesync-60.mkv
|
||||
mux video-2397.mp4 counter-stress.ass framesync-stress.mkv
|
||||
rm -f video-2397.mp4 video-60.mp4 burn-2397.ass burn-60.ass
|
||||
|
||||
echo "Done:"
|
||||
ls -la framesync-*.mkv
|
||||
echo
|
||||
echo "Reference check: open framesync-2397.mkv in mpv, frame-step with '.' —"
|
||||
echo "top (video) and bottom (subtitle) counters must match on every frame."
|
||||
echo "Then repeat in Plezy on-device and screenrecord (see header comments)."
|
||||
Reference in New Issue
Block a user