diff --git a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerCore.kt b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerCore.kt index 4c0b5de0..739f50a5 100644 --- a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerCore.kt +++ b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/ExoPlayerCore.kt @@ -165,6 +165,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { private var surfaceContainer: FrameLayout? = null private var videoAspectContainer: AspectRatioFrameLayout? = null private var subtitleView: SubtitleView? = null + private var bitmapSubtitleView: SubtitleView? = null private var videoZoomScale: Float = 1.0f private var assHandler: AssHandler? = null private var assSubtitleView: AssSubtitleSurfaceView? = null @@ -474,9 +475,19 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { videoAspectContainer!!.addView(surfaceView) surfaceContainer!!.addView(videoAspectContainer) + // Separate bitmap subtitles from text subtitles. Media3 scales PGS/VOB + // bitmap cue width and height against the SubtitleView bounds, so putting + // image cues in the screen-sized text view deforms them on stretch/zoom. + bitmapSubtitleView = SubtitleView(activity).apply { + layoutParams = FrameLayout.LayoutParams( + FrameLayout.LayoutParams.MATCH_PARENT, + FrameLayout.LayoutParams.MATCH_PARENT + ) + } + // Create SubtitleView - added to surfaceContainer above video. Hosts only // the built-in CanvasSubtitleOutput for non-ASS text cues; the ASS overlay - // lives inside videoAspectContainer so it tracks the video rect. + // is screen-sized and tracks the video rect via libass margins. subtitleView = SubtitleView(activity).apply { layoutParams = FrameLayout.LayoutParams( FrameLayout.LayoutParams.MATCH_PARENT, @@ -485,8 +496,9 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { } // Add SubtitleView to surfaceContainer (above video SurfaceView) // Flutter renders on top of entire surfaceContainer, keeping subtitles below UI + surfaceContainer!!.addView(bitmapSubtitleView) surfaceContainer!!.addView(subtitleView) - Log.d(TAG, "SubtitleView created and added to surfaceContainer") + Log.d(TAG, "SubtitleViews created and added to surfaceContainer") val contentView = activity.findViewById(android.R.id.content) contentView.addView(surfaceContainer, 0) @@ -676,14 +688,21 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { // eglPresentationTimeANDROID can vsync-pin to the video frame. // Z-order: video SurfaceView (-2) < this MediaOverlay-flagged // SurfaceView (-1) < parent canvas < Flutter SurfaceView (+1) in the window. - // Inserted before subtitleView so both punches run before SRT/VTT cues draw - // on the parent canvas. + // Inserted before the Media3 subtitle views so both punches run before + // PGS/VOB/SRT/VTT cues draw on the parent canvas. surfaceContainer?.let { container -> val assView = AssSubtitleSurfaceView(container.context, handler) assSubtitleView = assView // Pre-36 sublayer is already set by the view's own setZOrderMediaOverlay(true). FlutterOverlayHelper.applyCompositionOrder(assView, -1) - val subtitleIndex = container.indexOfChild(subtitleView) + val bitmapIndex = container.indexOfChild(bitmapSubtitleView) + val textIndex = container.indexOfChild(subtitleView) + val subtitleIndex = when { + bitmapIndex >= 0 && textIndex >= 0 -> minOf(bitmapIndex, textIndex) + bitmapIndex >= 0 -> bitmapIndex + textIndex >= 0 -> textIndex + else -> -1 + } container.addView( assView, if (subtitleIndex >= 0) subtitleIndex else container.childCount, @@ -804,12 +823,15 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { // Debug: Log SubtitleView child hierarchy subtitleView?.post { - Log.d(TAG, "SubtitleView post-layout: width=${subtitleView?.width}, height=${subtitleView?.height}, childCount=${subtitleView?.childCount}") + Log.d(TAG, "Text SubtitleView post-layout: width=${subtitleView?.width}, height=${subtitleView?.height}, childCount=${subtitleView?.childCount}") for (i in 0 until (subtitleView?.childCount ?: 0)) { val child = subtitleView?.getChildAt(i) Log.d(TAG, " Child $i: ${child?.javaClass?.simpleName}, w=${child?.width}, h=${child?.height}, visibility=${child?.visibility}") } } + bitmapSubtitleView?.post { + Log.d(TAG, "Bitmap SubtitleView post-layout: width=${bitmapSubtitleView?.width}, height=${bitmapSubtitleView?.height}, childCount=${bitmapSubtitleView?.childCount}") + } // Start position update loop startPositionUpdates() @@ -911,21 +933,28 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { // Player.Listener override fun onCues(cueGroup: CueGroup) { - // ASS subtitles are rendered by the libass overlay surface. - // This callback is for non-ASS subtitles (SRT, VTT, etc.) + // ASS subtitles are rendered by the libass overlay surface. This callback + // handles non-ASS text cues plus bitmap image cues such as PGS/VOB. val incoming = cueGroup.cues lastSubtitleCues = incoming - val stacked = stackUnpositionedCues(incoming) - val outgoing = applySubtitlePosition(stacked) if (incoming.isNotEmpty()) { + val textCount = incoming.count { it.bitmap == null } + val bitmapCount = incoming.size - textCount Log.d( TAG, - "onCues: received ${incoming.size} cues (non-ASS)" + - (if (stacked !== incoming) " - stacked" else "") + - (if (outgoing !== stacked) " - positioned" else "") + "onCues: received ${incoming.size} cues (non-ASS, text=$textCount, bitmap=$bitmapCount)" ) } + renderSubtitleCues(incoming) + } + + private fun renderSubtitleCues(cues: List) { + val textCues = cues.filter { it.bitmap == null } + val bitmapCues = cues.filter { it.bitmap != null } + val stacked = stackUnpositionedCues(textCues) + val outgoing = applySubtitlePosition(stacked) subtitleView?.setCues(outgoing) + bitmapSubtitleView?.setCues(bitmapCues) } // SRT carries no per-cue positioning, so SubripParser emits cues with @@ -1101,7 +1130,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { // Disabling the text track produces no trailing empty CueGroup, and no new // video frame re-renders the libass overlay while paused, so the last SRT/VTT - // line stays painted on the SubtitleView and the last ASS frame stays on the + // cue stays painted on the SubtitleViews and the last ASS frame stays on the // overlay. AssHandler (registered before this listener) has already nulled the // libass track by now, so re-rendering the last position clears it. Gate on the // transition to avoid redundant clears on every track change. (#1387) @@ -1109,6 +1138,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { if (!hasSelectedText && hadSelectedTextTrack) { lastSubtitleCues = emptyList() subtitleView?.setCues(emptyList()) + bitmapSubtitleView?.setCues(emptyList()) assSubtitleView?.invalidateSubtitles() } hadSelectedTextTrack = hasSelectedText @@ -1441,56 +1471,60 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { if (disposing) return if (videoWidth == 0 || videoHeight == 0) return - val subtitle = subtitleView ?: return val contentView = activity.findViewById(android.R.id.content) - val containerWidth = contentView.width - val containerHeight = contentView.height + val containerWidth = surfaceContainer?.width?.takeIf { it > 0 } ?: contentView.width + val containerHeight = surfaceContainer?.height?.takeIf { it > 0 } ?: contentView.height if (containerWidth == 0 || containerHeight == 0) return - // Sizes the non-ASS SubtitleView only (the ASS overlay is screen-sized and - // tracks the video rect via libass margins — see updateAssMargins()). - // In cover/stretch/zoomed-in modes text cues stay at container size so they - // never get cropped. In letterbox mode they follow the visible video rect. - val isLetterbox = videoAspectContainer?.resizeMode == AspectRatioFrameLayout.RESIZE_MODE_FIT - val (subWidth, subHeight) = if (isLetterbox) { - val videoAspect = (videoWidth * pixelRatio) / videoHeight - val containerAspect = containerWidth.toFloat() / containerHeight - val (baseWidth, baseHeight) = if (videoAspect > containerAspect) { - containerWidth to (containerWidth / videoAspect).toInt() - } else { - (containerHeight * videoAspect).toInt() to containerHeight - } - if (videoZoomScale < 0.999f) { - val zoomedWidth = ((baseWidth * videoZoomScale).toInt()).coerceAtLeast(1) - val zoomedHeight = ((baseHeight * videoZoomScale).toInt()).coerceAtLeast(1) - zoomedWidth to zoomedHeight - } else if (videoZoomScale > 1.001f) { - containerWidth to containerHeight - } else { - baseWidth to baseHeight - } - } else { - containerWidth to containerHeight - } + val resizeMode = videoAspectContainer?.resizeMode ?: AspectRatioFrameLayout.RESIZE_MODE_FIT + val textDimensions = SubtitleViewLayout.textDimensions( + containerWidth, + containerHeight, + videoWidth, + videoHeight, + pixelRatio, + resizeMode, + videoZoomScale + ) + val bitmapDimensions = SubtitleViewLayout.bitmapDimensions( + containerWidth, + containerHeight, + videoWidth, + videoHeight, + pixelRatio, + resizeMode, + videoZoomScale + ) activity.runOnUiThread { - // Skip when already at the target size: setLayoutParams always schedules a - // layout pass, and this runs from the global-layout listener — re-applying - // equal params would keep the UI thread laying out every frame (#1261). - val current = subtitle.layoutParams as? FrameLayout.LayoutParams - if (current != null && - current.width == subWidth && - current.height == subHeight && - current.gravity == Gravity.CENTER - ) { - return@runOnUiThread + val textView = subtitleView + if (textView != null && textDimensions != null) { + applySubtitleViewSize(textView, textDimensions) } - subtitle.layoutParams = FrameLayout.LayoutParams(subWidth, subHeight).apply { - gravity = Gravity.CENTER + val bitmapView = bitmapSubtitleView + if (bitmapView != null && bitmapDimensions != null) { + applySubtitleViewSize(bitmapView, bitmapDimensions) } } } + private fun applySubtitleViewSize(view: View, dimensions: SubtitleViewDimensions) { + // Skip when already at the target size: setLayoutParams always schedules a + // layout pass, and this runs from the global-layout listener — re-applying + // equal params would keep the UI thread laying out every frame (#1261). + val current = view.layoutParams as? FrameLayout.LayoutParams + if (current != null && + current.width == dimensions.width && + current.height == dimensions.height && + current.gravity == Gravity.CENTER + ) { + return + } + view.layoutParams = FrameLayout.LayoutParams(dimensions.width, dimensions.height).apply { + gravity = Gravity.CENTER + } + } + // Pushes mpv-style libass margins: the offsets of the video dst rect within the // full-screen container (= the libass frame). Negative when the video extends past // the screen (cover mode, zoom > 1) — libass supports that explicitly. Pure math + @@ -1529,12 +1563,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { AspectRatioFrameLayout.RESIZE_MODE_FILL -> containerWidth.toFloat() to containerHeight.toFloat() // Fit: letterbox within the container - else -> - if (videoAspect > containerAspect) { - containerWidth.toFloat() to containerWidth / videoAspect - } else { - containerHeight * videoAspect to containerHeight.toFloat() - } + else -> SubtitleViewLayout.letterbox(containerWidth, containerHeight, videoAspect) } } @@ -3162,7 +3191,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { activity.runOnUiThread { if (disposing) return@runOnUiThread surfaceContainer?.visibility = if (visible) View.VISIBLE else View.INVISIBLE - // subtitleView is inside surfaceContainer, inherits visibility + // Subtitle views are inside surfaceContainer, so they inherit visibility. Log.d(TAG, "setVisible($visible)") } } @@ -3233,9 +3262,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { } else { subtitleView?.setBottomPaddingFraction(0f) } - if (lastSubtitleCues.isNotEmpty()) { - subtitleView?.setCues(applySubtitlePosition(stackUnpositionedCues(lastSubtitleCues))) - } + if (lastSubtitleCues.isNotEmpty()) renderSubtitleCues(lastSubtitleCues) // 2. ASS subtitles: font scale via libass // MPV default sub-font-size is 38 @@ -3575,6 +3602,7 @@ class ExoPlayerCore(private val activity: Activity) : Player.Listener { videoAspectContainer = null surfaceView = null subtitleView = null + bitmapSubtitleView = null assSubtitleView = null // Remove layout listener synchronously diff --git a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/SubtitleViewLayout.kt b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/SubtitleViewLayout.kt new file mode 100644 index 00000000..f56084e9 --- /dev/null +++ b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/SubtitleViewLayout.kt @@ -0,0 +1,85 @@ +package com.edde746.plezy.exoplayer + +import androidx.media3.ui.AspectRatioFrameLayout +import kotlin.math.roundToInt + +internal data class SubtitleViewDimensions(val width: Int, val height: Int) + +internal object SubtitleViewLayout { + fun textDimensions( + containerWidth: Int, + containerHeight: Int, + videoWidth: Int, + videoHeight: Int, + pixelRatio: Float, + resizeMode: Int, + zoomScale: Float + ): SubtitleViewDimensions? { + val videoAspect = videoAspect(videoWidth, videoHeight, pixelRatio) ?: return null + if (containerWidth <= 0 || containerHeight <= 0) return null + + if (resizeMode != AspectRatioFrameLayout.RESIZE_MODE_FIT) { + return SubtitleViewDimensions(containerWidth, containerHeight) + } + + val base = fit(containerWidth, containerHeight, videoAspect) + return when { + zoomScale < 0.999f -> scale(base, zoomScale) + zoomScale > 1.001f -> SubtitleViewDimensions(containerWidth, containerHeight) + else -> base + } + } + + fun bitmapDimensions( + containerWidth: Int, + containerHeight: Int, + videoWidth: Int, + videoHeight: Int, + pixelRatio: Float, + resizeMode: Int, + zoomScale: Float + ): SubtitleViewDimensions? { + val videoAspect = videoAspect(videoWidth, videoHeight, pixelRatio) ?: return null + if (containerWidth <= 0 || containerHeight <= 0) return null + + // PGS/VOB cues are authored in video coordinates, but users expect them to + // remain readable and fully visible when the video is cropped or zoomed. + // Fit to the visible container instead of following the cropped video rect. + return fit(containerWidth, containerHeight, videoAspect) + } + + private fun videoAspect(videoWidth: Int, videoHeight: Int, pixelRatio: Float): Float? { + if (videoWidth <= 0 || videoHeight <= 0 || pixelRatio <= 0f) return null + val aspect = (videoWidth * pixelRatio) / videoHeight + return if (aspect.isFinite() && aspect > 0f) aspect else null + } + + // Largest [videoAspect] rectangle that fits inside the container, as (width, height) + // in float pixels. Shared by the SubtitleView sizing (rounded to ints in [fit]) and + // the libass FIT-mode margins in updateAssMargins() (kept as floats for the zoom + // multiply) so both agree on the video dst rect. + fun letterbox(containerWidth: Int, containerHeight: Int, videoAspect: Float): Pair { + val containerAspect = containerWidth.toFloat() / containerHeight + return if (videoAspect > containerAspect) { + containerWidth.toFloat() to containerWidth / videoAspect + } else { + containerHeight * videoAspect to containerHeight.toFloat() + } + } + + private fun fit(containerWidth: Int, containerHeight: Int, videoAspect: Float): SubtitleViewDimensions { + val (width, height) = letterbox(containerWidth, containerHeight, videoAspect) + return SubtitleViewDimensions( + width.roundToInt().coerceAtLeast(1), + height.roundToInt().coerceAtLeast(1) + ) + } + + private fun scale(dimensions: SubtitleViewDimensions, scale: Float): SubtitleViewDimensions { + val safeScale = scale.coerceAtLeast(0.001f) + return SubtitleViewDimensions( + (dimensions.width * safeScale).roundToInt().coerceAtLeast(1), + (dimensions.height * safeScale).roundToInt().coerceAtLeast(1) + ) + } +} diff --git a/android/app/src/test/kotlin/com/edde746/plezy/exoplayer/SubtitleViewLayoutTest.kt b/android/app/src/test/kotlin/com/edde746/plezy/exoplayer/SubtitleViewLayoutTest.kt new file mode 100644 index 00000000..9edf8266 --- /dev/null +++ b/android/app/src/test/kotlin/com/edde746/plezy/exoplayer/SubtitleViewLayoutTest.kt @@ -0,0 +1,78 @@ +package com.edde746.plezy.exoplayer + +import androidx.media3.ui.AspectRatioFrameLayout +import org.junit.Assert.assertEquals +import org.junit.Test + +class SubtitleViewLayoutTest { + + @Test + fun letterboxUsesVisibleVideoRectForTextAndBitmapSubtitles() { + val text = textDimensions(resizeMode = AspectRatioFrameLayout.RESIZE_MODE_FIT) + val bitmap = bitmapDimensions(resizeMode = AspectRatioFrameLayout.RESIZE_MODE_FIT) + + assertEquals(SubtitleViewDimensions(1920, 1080), text) + assertEquals(SubtitleViewDimensions(1920, 1080), bitmap) + } + + @Test + fun coverKeepsTextOnScreenAndBitmapSubtitlesFullyVisible() { + val text = textDimensions(resizeMode = AspectRatioFrameLayout.RESIZE_MODE_ZOOM) + val bitmap = bitmapDimensions(resizeMode = AspectRatioFrameLayout.RESIZE_MODE_ZOOM) + + assertEquals(SubtitleViewDimensions(2424, 1080), text) + assertEquals(SubtitleViewDimensions(1920, 1080), bitmap) + assertAspectCloseTo16By9(bitmap!!) + } + + @Test + fun manualZoomKeepsBitmapSubtitlesFullyVisible() { + val text = textDimensions(resizeMode = AspectRatioFrameLayout.RESIZE_MODE_FIT, zoomScale = 1.5f) + val bitmap = bitmapDimensions(resizeMode = AspectRatioFrameLayout.RESIZE_MODE_FIT, zoomScale = 1.5f) + + assertEquals(SubtitleViewDimensions(2424, 1080), text) + assertEquals(SubtitleViewDimensions(1920, 1080), bitmap) + assertAspectCloseTo16By9(bitmap!!) + } + + @Test + fun stretchDoesNotStretchBitmapSubtitlesToScreenAspect() { + val text = textDimensions(resizeMode = AspectRatioFrameLayout.RESIZE_MODE_FILL) + val bitmap = bitmapDimensions(resizeMode = AspectRatioFrameLayout.RESIZE_MODE_FILL) + + assertEquals(SubtitleViewDimensions(2424, 1080), text) + assertEquals(SubtitleViewDimensions(1920, 1080), bitmap) + assertAspectCloseTo16By9(bitmap!!) + } + + private fun textDimensions( + resizeMode: Int, + zoomScale: Float = 1f + ): SubtitleViewDimensions? = SubtitleViewLayout.textDimensions( + containerWidth = 2424, + containerHeight = 1080, + videoWidth = 1920, + videoHeight = 1080, + pixelRatio = 1f, + resizeMode = resizeMode, + zoomScale = zoomScale + ) + + private fun bitmapDimensions( + resizeMode: Int, + zoomScale: Float = 1f + ): SubtitleViewDimensions? = SubtitleViewLayout.bitmapDimensions( + containerWidth = 2424, + containerHeight = 1080, + videoWidth = 1920, + videoHeight = 1080, + pixelRatio = 1f, + resizeMode = resizeMode, + zoomScale = zoomScale + ) + + private fun assertAspectCloseTo16By9(dimensions: SubtitleViewDimensions) { + val aspect = dimensions.width.toDouble() / dimensions.height + assertEquals(16.0 / 9.0, aspect, 0.001) + } +}