diff --git a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/DvBitstreamSanitizer.kt b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/DvBitstreamSanitizer.kt index 12a60a12..eb1c0b39 100644 --- a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/DvBitstreamSanitizer.kt +++ b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/DvBitstreamSanitizer.kt @@ -13,40 +13,64 @@ import java.nio.ByteBuffer * - HEVC fallback for a DV format: strip DV RPU/EL NALs (types 62/63) instead, * leaving HDR10+ for the display. * + * Runs on the playback thread against MediaCodec's direct input buffers, where + * per-byte ByteBuffer accessor calls are far too slow for high-bitrate UHD remuxes + * (#1313). Each sample is therefore staged through a reusable heap array with a + * single bulk get, scanned and compacted with array ops, and only the modified tail + * is written back. Instances are not thread-safe: use one per renderer. + * * Pure JVM (no android/media3 imports) so it stays unit-testable on the host. */ -object DvBitstreamSanitizer { +class DvBitstreamSanitizer { - private const val NAL_TYPE_PREFIX_SEI = 39 - private const val NAL_TYPE_SUFFIX_SEI = 40 - private const val NAL_TYPE_UNSPEC62 = 62 // DV RPU - private const val NAL_TYPE_UNSPEC63 = 63 // DV Enhancement Layer + private companion object { + const val NAL_TYPE_PREFIX_SEI = 39 + const val NAL_TYPE_SUFFIX_SEI = 40 + const val NAL_TYPE_UNSPEC62 = 62 // DV RPU + const val NAL_TYPE_UNSPEC63 = 63 // DV Enhancement Layer - private const val SEI_PAYLOAD_TYPE_ITU_T_T35 = 4 + const val SEI_PAYLOAD_TYPE_ITU_T_T35 = 4 + + const val INITIAL_SCRATCH_SIZE = 256 * 1024 + } + + /** Reusable staging buffer — grown as needed, never shrunk. */ + private var scratch = ByteArray(INITIAL_SCRATCH_SIZE) /** * Scans `[position, limit)` of [data] for Annex B NAL units and removes the selected * metadata NALs by compacting the buffer in place and reducing its limit. The position - * is left unchanged. + * is left unchanged. Returns the number of NAL units stripped. + * + * A sample with no start codes at all is left untouched: it cannot contain the + * targeted NALs, and emptying it would only confuse the decoder. */ - fun sanitize(data: ByteBuffer, stripHdr10PlusSei: Boolean, stripDvRpu: Boolean) { + fun sanitize(data: ByteBuffer, stripHdr10PlusSei: Boolean, stripDvRpu: Boolean): Int { val startPos = data.position() - val limit = data.limit() - var writePos = startPos + val len = data.limit() - startPos + if (len == 0) return 0 + if (scratch.size < len) scratch = ByteArray(maxOf(len, scratch.size * 2)) + val buf = scratch + data.get(buf, 0, len) + data.position(startPos) + + var writeLen = 0 + var firstModified = -1 + var strippedCount = 0 var nalStartIndex = -1 var startCodeLen = 0 - var i = startPos - while (i <= limit) { + var i = 0 + while (i <= len) { // Find next start code or end of buffer. - val atEnd = i == limit + val atEnd = i == len var foundStartCode = false var nextStartCodeLen = 0 - if (!atEnd && i + 2 < limit && data.get(i).toInt() == 0 && data.get(i + 1).toInt() == 0) { - if (data.get(i + 2).toInt() == 1) { + if (!atEnd && i + 2 < len && buf[i].toInt() == 0 && buf[i + 1].toInt() == 0) { + if (buf[i + 2].toInt() == 1) { foundStartCode = true nextStartCodeLen = 3 - } else if (data.get(i + 2).toInt() == 0 && i + 3 < limit && data.get(i + 3).toInt() == 1) { + } else if (buf[i + 2].toInt() == 0 && i + 3 < len && buf[i + 3].toInt() == 1) { foundStartCode = true nextStartCodeLen = 4 } @@ -61,23 +85,25 @@ object DvBitstreamSanitizer { if (nalEnd - nalDataStart >= 2) { // HEVC NAL header: forbidden_zero_bit(1) + nal_unit_type(6) + nuh_layer_id MSB(1). - val nalUnitType = (data.get(nalDataStart).toInt() and 0x7E) shr 1 + val nalUnitType = (buf[nalDataStart].toInt() and 0x7E) shr 1 strip = when (nalUnitType) { NAL_TYPE_UNSPEC62, NAL_TYPE_UNSPEC63 -> stripDvRpu NAL_TYPE_PREFIX_SEI, NAL_TYPE_SUFFIX_SEI -> - stripHdr10PlusSei && isHdr10PlusSeiNalUnit(data, nalDataStart + 2, nalEnd) + stripHdr10PlusSei && isHdr10PlusSeiNalUnit(buf, nalDataStart + 2, nalEnd) else -> false } } - if (!strip) { - if (writePos != nalStartIndex) { - for (j in nalStartIndex until nalEnd) { - data.put(writePos++, data.get(j)) - } - } else { - writePos = nalEnd + if (strip) { + strippedCount++ + if (firstModified < 0) firstModified = writeLen + } else { + if (writeLen != nalStartIndex) { + // Also reached with zero strips when bytes precede the first start code. + if (firstModified < 0) firstModified = writeLen + System.arraycopy(buf, nalStartIndex, buf, writeLen, nalEnd - nalStartIndex) } + writeLen += nalEnd - nalStartIndex } } nalStartIndex = i @@ -88,8 +114,14 @@ object DvBitstreamSanitizer { } } - data.limit(writePos) + if (writeLen == len || firstModified < 0) return 0 + + // Bytes before firstModified are byte-identical to the buffer; write back only the tail. + data.position(startPos + firstModified) + data.put(buf, firstModified, writeLen - firstModified) + data.limit(startPos + writeLen) data.position(startPos) + return strippedCount } /** @@ -99,14 +131,14 @@ object DvBitstreamSanitizer { * (ST 2094-40), application version 0 or 1. Malformed/truncated data returns false so the * NAL is kept. */ - private fun isHdr10PlusSeiNalUnit(data: ByteBuffer, rbspStart: Int, nalEnd: Int): Boolean { + private fun isHdr10PlusSeiNalUnit(buf: ByteArray, rbspStart: Int, nalEnd: Int): Boolean { var pos = rbspStart if (pos >= nalEnd) return false // SEI payload type: accumulated 0xFF bytes plus the final byte. var payloadType = 0 while (pos < nalEnd) { - val b = data.get(pos++).toInt() and 0xFF + val b = buf[pos++].toInt() and 0xFF payloadType += b if (b != 0xFF) break } @@ -114,7 +146,7 @@ object DvBitstreamSanitizer { // SEI payload size, same encoding. var payloadSize = 0 while (pos < nalEnd) { - val b = data.get(pos++).toInt() and 0xFF + val b = buf[pos++].toInt() and 0xFF payloadSize += b if (b != 0xFF) break } @@ -125,11 +157,11 @@ object DvBitstreamSanitizer { // The identifier bytes (B5 00 3C 00 01 04 00/01) cannot contain the 0x000003 emulation // prevention pattern, so they can be read without RBSP unescaping. - val countryCode = data.get(pos).toInt() and 0xFF - val providerCode = ((data.get(pos + 1).toInt() and 0xFF) shl 8) or (data.get(pos + 2).toInt() and 0xFF) - val orientedCode = ((data.get(pos + 3).toInt() and 0xFF) shl 8) or (data.get(pos + 4).toInt() and 0xFF) - val appIdentifier = data.get(pos + 5).toInt() and 0xFF - val appVersion = data.get(pos + 6).toInt() and 0xFF + val countryCode = buf[pos].toInt() and 0xFF + val providerCode = ((buf[pos + 1].toInt() and 0xFF) shl 8) or (buf[pos + 2].toInt() and 0xFF) + val orientedCode = ((buf[pos + 3].toInt() and 0xFF) shl 8) or (buf[pos + 4].toInt() and 0xFF) + val appIdentifier = buf[pos + 5].toInt() and 0xFF + val appVersion = buf[pos + 6].toInt() and 0xFF return countryCode == 0xB5 && providerCode == 0x003C && diff --git a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/PlezyRenderersFactory.kt b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/PlezyRenderersFactory.kt index 651833ef..14b24917 100644 --- a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/PlezyRenderersFactory.kt +++ b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/PlezyRenderersFactory.kt @@ -344,9 +344,18 @@ internal class DvSanitizingVideoRenderer( private val log: ((String, String, String) -> Unit)? ) : MediaCodecVideoRenderer(builder) { + private val sanitizer = DvBitstreamSanitizer() + private var stripHdr10PlusSei = false private var stripDvRpu = false + // Sanitize diagnostics — playback-thread only, cumulative for the renderer's lifetime. + private var sanitizedSampleCount = 0L + private var totalSanitizeTimeUs = 0L + private var maxSanitizeTimeUs = 0L + private var totalStrippedNals = 0L + private var totalStrippedBytes = 0L + override fun onCodecInitialized( name: String, configuration: MediaCodecAdapter.Configuration, @@ -378,7 +387,25 @@ internal class DvSanitizingVideoRenderer( if (stripHdr10PlusSei || stripDvRpu) { val data = buffer.data if (data != null && data.hasRemaining() && !buffer.isEncrypted) { - DvBitstreamSanitizer.sanitize(data, stripHdr10PlusSei, stripDvRpu) + val sizeBefore = data.remaining() + val startNs = System.nanoTime() + val stripped = sanitizer.sanitize(data, stripHdr10PlusSei, stripDvRpu) + val elapsedUs = (System.nanoTime() - startNs) / 1_000 + sanitizedSampleCount++ + totalSanitizeTimeUs += elapsedUs + if (elapsedUs > maxSanitizeTimeUs) maxSanitizeTimeUs = elapsedUs + totalStrippedNals += stripped + totalStrippedBytes += sizeBefore - data.remaining() + if (sanitizedSampleCount <= 3 || sanitizedSampleCount % 500 == 0L) { + log?.invoke( + "debug", + "dv-sanitize", + "Sample #$sanitizedSampleCount: ${sizeBefore}B -> ${data.remaining()}B, " + + "stripped=$stripped, took=${elapsedUs}us " + + "(avg=${totalSanitizeTimeUs / sanitizedSampleCount}us, max=${maxSanitizeTimeUs}us, " + + "totalNals=$totalStrippedNals, totalBytes=${totalStrippedBytes}B)" + ) + } } } super.onQueueInputBuffer(buffer) diff --git a/android/app/src/test/kotlin/com/edde746/plezy/exoplayer/DvBitstreamSanitizerTest.kt b/android/app/src/test/kotlin/com/edde746/plezy/exoplayer/DvBitstreamSanitizerTest.kt index f1c1868d..cd4bef74 100644 --- a/android/app/src/test/kotlin/com/edde746/plezy/exoplayer/DvBitstreamSanitizerTest.kt +++ b/android/app/src/test/kotlin/com/edde746/plezy/exoplayer/DvBitstreamSanitizerTest.kt @@ -8,6 +8,8 @@ import org.junit.Test class DvBitstreamSanitizerTest { + private val sanitizer = DvBitstreamSanitizer() + // --- HDR10+ SEI stripping (native DV codec path) --- @Test @@ -17,7 +19,7 @@ class DvBitstreamSanitizerTest { val buffer = bufferOf(vcl1, hdr10PlusSei(), vcl2) val originalLimit = buffer.limit() - DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false) + sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false) assertArrayEquals(concat(vcl1, vcl2), remainingBytes(buffer)) assertTrue(buffer.limit() < originalLimit) @@ -30,7 +32,7 @@ class DvBitstreamSanitizerTest { val suffixSei = annexBNal(40, hdr10PlusSeiPayload()) val buffer = bufferOf(vcl, suffixSei) - DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false) + sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false) assertArrayEquals(vcl, remainingBytes(buffer)) } @@ -42,7 +44,7 @@ class DvBitstreamSanitizerTest { val vcl2 = annexBNal(1, byteArrayOf(0x03), startCodeLen = 3) val buffer = bufferOf(vcl1, sei, vcl2) - DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false) + sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false) assertArrayEquals(concat(vcl1, vcl2), remainingBytes(buffer)) } @@ -57,7 +59,7 @@ class DvBitstreamSanitizerTest { val buffer = bufferOf(annexBNal(1, byteArrayOf(0x01)), sei, annexBNal(1, byteArrayOf(0x02))) val original = remainingBytes(buffer) - DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false) + sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false) assertArrayEquals(original, remainingBytes(buffer)) } @@ -67,7 +69,7 @@ class DvBitstreamSanitizerTest { val buffer = bufferOf(annexBNal(1, byteArrayOf(0x01, 0x02)), annexBNal(1, byteArrayOf(0x03))) val original = remainingBytes(buffer) - DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = true) + sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = true) assertArrayEquals(original, remainingBytes(buffer)) } @@ -79,7 +81,7 @@ class DvBitstreamSanitizerTest { val buffer = bufferOf(annexBNal(1, byteArrayOf(0x01)), truncated) val original = remainingBytes(buffer) - DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false) + sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false) assertArrayEquals(original, remainingBytes(buffer)) } @@ -89,7 +91,7 @@ class DvBitstreamSanitizerTest { val buffer = bufferOf(annexBNal(1, byteArrayOf(0x01)), hdr10PlusSei()) val original = remainingBytes(buffer) - DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = false, stripDvRpu = false) + sanitizer.sanitize(buffer, stripHdr10PlusSei = false, stripDvRpu = false) assertArrayEquals(original, remainingBytes(buffer)) } @@ -104,7 +106,7 @@ class DvBitstreamSanitizerTest { val sei = hdr10PlusSei() val buffer = bufferOf(vcl, rpu, sei, el) - DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = false, stripDvRpu = true) + sanitizer.sanitize(buffer, stripHdr10PlusSei = false, stripDvRpu = true) assertArrayEquals(concat(vcl, sei), remainingBytes(buffer)) } @@ -115,7 +117,7 @@ class DvBitstreamSanitizerTest { val vcl2 = annexBNal(1, byteArrayOf(0x05)) val buffer = bufferOf(vcl1, annexBNal(62, byteArrayOf(0x19)), hdr10PlusSei(), vcl2) - DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = true) + sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = true) assertArrayEquals(concat(vcl1, vcl2), remainingBytes(buffer)) } @@ -130,7 +132,7 @@ class DvBitstreamSanitizerTest { val buffer = ByteBuffer.wrap(content.copyOf()) buffer.position(prefix.size) - DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false) + sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false) assertEquals(prefix.size, buffer.position()) assertArrayEquals(vcl, remainingBytes(buffer)) @@ -142,12 +144,9 @@ class DvBitstreamSanitizerTest { @Test fun worksOnDirectBuffers() { val vcl = annexBNal(1, byteArrayOf(0x01, 0x02)) - val content = concat(vcl, hdr10PlusSei()) - val buffer = ByteBuffer.allocateDirect(content.size) - buffer.put(content) - buffer.flip() + val buffer = directBufferOf(vcl, hdr10PlusSei()) - DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false) + sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false) assertArrayEquals(vcl, remainingBytes(buffer)) } @@ -156,12 +155,117 @@ class DvBitstreamSanitizerTest { fun emptyBufferIsNoOp() { val buffer = ByteBuffer.allocate(0) - DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = true) + sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = true) assertEquals(0, buffer.position()) assertEquals(0, buffer.limit()) } + @Test + fun stripsOnDirectBufferWithNonZeroPosition() { + val vcl1 = annexBNal(1, byteArrayOf(0x01, 0x02)) + val vcl2 = annexBNal(1, byteArrayOf(0x03)) + val buffer = directBufferOf(byteArrayOf(0xAA.toByte(), 0xBB.toByte()), vcl1, hdr10PlusSei(), vcl2) + buffer.position(2) + + val stripped = sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false) + + assertEquals(1, stripped) + assertEquals(2, buffer.position()) + assertArrayEquals(concat(vcl1, vcl2), remainingBytes(buffer)) + // Bytes before the position are untouched. + assertEquals(0xAA.toByte(), buffer.get(0)) + assertEquals(0xBB.toByte(), buffer.get(1)) + } + + @Test + fun noStripLeavesDirectBufferStateUntouched() { + val vcl1 = annexBNal(1, byteArrayOf(0x01, 0x02)) + val vcl2 = annexBNal(1, byteArrayOf(0x03)) + val buffer = directBufferOf(vcl1, vcl2) + val originalLimit = buffer.limit() + + val stripped = sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = true) + + assertEquals(0, stripped) + assertEquals(0, buffer.position()) + assertEquals(originalLimit, buffer.limit()) + assertArrayEquals(concat(vcl1, vcl2), remainingBytes(buffer)) + } + + @Test + fun stripsTrailingRpuWithZeroByteTailWriteBack() { + val vcl1 = annexBNal(1, byteArrayOf(0x01, 0x02)) + val vcl2 = annexBNal(1, byteArrayOf(0x03)) + val rpu = annexBNal(62, byteArrayOf(0x19, 0x08)) + val buffer = directBufferOf(vcl1, vcl2, rpu) + val originalLimit = buffer.limit() + + val stripped = sanitizer.sanitize(buffer, stripHdr10PlusSei = false, stripDvRpu = true) + + assertEquals(1, stripped) + assertEquals(originalLimit - rpu.size, buffer.limit()) + assertArrayEquals(concat(vcl1, vcl2), remainingBytes(buffer)) + } + + @Test + fun growsScratchForLargeSamples() { + // Exceeds the initial 256KB scratch and one doubling; non-zero filler so no + // accidental start codes appear in the payloads. + val vcl1 = annexBNal(1, ByteArray(800_000) { 0xAB.toByte() }) + val vcl2 = annexBNal(1, ByteArray(700_000) { 0xCD.toByte() }) + val buffer = bufferOf(vcl1, hdr10PlusSei(), vcl2) + + val stripped = sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false) + + assertEquals(1, stripped) + assertArrayEquals(concat(vcl1, vcl2), remainingBytes(buffer)) + } + + @Test + fun reusesSanitizerAcrossSequentialBuffers() { + val vcl1 = annexBNal(1, byteArrayOf(0x01, 0x02)) + val vcl2 = annexBNal(1, byteArrayOf(0x03, 0x04)) + + val first = bufferOf(vcl1, hdr10PlusSei(), vcl2) + assertEquals(1, sanitizer.sanitize(first, stripHdr10PlusSei = true, stripDvRpu = false)) + assertArrayEquals(concat(vcl1, vcl2), remainingBytes(first)) + + val second = bufferOf(vcl1, vcl2) + assertEquals(0, sanitizer.sanitize(second, stripHdr10PlusSei = true, stripDvRpu = false)) + assertArrayEquals(concat(vcl1, vcl2), remainingBytes(second)) + + val third = bufferOf(hdr10PlusSei(), vcl1, annexBNal(62, byteArrayOf(0x19)), vcl2, hdr10PlusSei()) + assertEquals(3, sanitizer.sanitize(third, stripHdr10PlusSei = true, stripDvRpu = true)) + assertArrayEquals(concat(vcl1, vcl2), remainingBytes(third)) + } + + @Test + fun keepsBufferWithoutStartCodes() { + val content = byteArrayOf(0x12, 0x34, 0x56, 0x78) + val buffer = ByteBuffer.wrap(content.copyOf()) + + val stripped = sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = true) + + assertEquals(0, stripped) + assertEquals(0, buffer.position()) + assertEquals(content.size, buffer.limit()) + assertArrayEquals(content, remainingBytes(buffer)) + } + + @Test + fun stripsMultipleSeisInOneAccessUnit() { + val vcl1 = annexBNal(1, byteArrayOf(0x01, 0x02)) + val vcl2 = annexBNal(1, byteArrayOf(0x03)) + val suffixSei = annexBNal(40, hdr10PlusSeiPayload()) + val buffer = bufferOf(vcl1, hdr10PlusSei(), vcl2, suffixSei) + + val stripped = sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false) + + assertEquals(2, stripped) + assertArrayEquals(concat(vcl1, vcl2), remainingBytes(buffer)) + } + // --- Helpers --- /** Builds an HEVC NAL unit: start code + 2-byte NAL header encoding [nalUnitType] + payload. */ @@ -193,6 +297,14 @@ class DvBitstreamSanitizerTest { private fun bufferOf(vararg parts: ByteArray): ByteBuffer = ByteBuffer.wrap(concat(*parts)) + private fun directBufferOf(vararg parts: ByteArray): ByteBuffer { + val content = concat(*parts) + val buffer = ByteBuffer.allocateDirect(content.size) + buffer.put(content) + buffer.flip() + return buffer + } + private fun remainingBytes(buffer: ByteBuffer): ByteArray { val copy = ByteArray(buffer.remaining()) buffer.duplicate().get(copy)