yorickr/plezy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(android): eliminate per-byte buffer ops in DV bitstream sanitizer

Browse Source
This commit is contained in:
edde746
2026-06-12 04:05:35 +02:00
parent 62abd3c30a
commit a03d30fa45
3 changed files with 222 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
android/app/src/main/kotlin/com/edde746/plezy/exoplayer/DvBitstreamSanitizer.kt
+66 -34
View File
@@ -13,40 +13,64 @@ import java.nio.ByteBuffer
* - HEVC fallback for a DV format: strip DV RPU/EL NALs (types 62/63) instead,
* leaving HDR10+ for the display.
*
* Runs on the playback thread against MediaCodec's direct input buffers, where
* per-byte ByteBuffer accessor calls are far too slow for high-bitrate UHD remuxes
* (#1313). Each sample is therefore staged through a reusable heap array with a
* single bulk get, scanned and compacted with array ops, and only the modified tail
* is written back. Instances are not thread-safe: use one per renderer.
*
* Pure JVM (no android/media3 imports) so it stays unit-testable on the host.
*/
object DvBitstreamSanitizer {
class DvBitstreamSanitizer {
private const val NAL_TYPE_PREFIX_SEI = 39
private const val NAL_TYPE_SUFFIX_SEI = 40
private const val NAL_TYPE_UNSPEC62 = 62 // DV RPU
private const val NAL_TYPE_UNSPEC63 = 63 // DV Enhancement Layer
private companion object {
const val NAL_TYPE_PREFIX_SEI = 39
const val NAL_TYPE_SUFFIX_SEI = 40
const val NAL_TYPE_UNSPEC62 = 62 // DV RPU
const val NAL_TYPE_UNSPEC63 = 63 // DV Enhancement Layer
private const val SEI_PAYLOAD_TYPE_ITU_T_T35 = 4
const val SEI_PAYLOAD_TYPE_ITU_T_T35 = 4
const val INITIAL_SCRATCH_SIZE = 256 * 1024
}
/** Reusable staging buffer — grown as needed, never shrunk. */
private var scratch = ByteArray(INITIAL_SCRATCH_SIZE)
/**
* Scans `[position, limit)` of [data] for Annex B NAL units and removes the selected
* metadata NALs by compacting the buffer in place and reducing its limit. The position
* is left unchanged.
* is left unchanged. Returns the number of NAL units stripped.
*
* A sample with no start codes at all is left untouched: it cannot contain the
* targeted NALs, and emptying it would only confuse the decoder.
*/
fun sanitize(data: ByteBuffer, stripHdr10PlusSei: Boolean, stripDvRpu: Boolean) {
fun sanitize(data: ByteBuffer, stripHdr10PlusSei: Boolean, stripDvRpu: Boolean): Int {
val startPos = data.position()
val limit = data.limit()
var writePos = startPos
val len = data.limit() - startPos
if (len == 0) return 0
if (scratch.size < len) scratch = ByteArray(maxOf(len, scratch.size * 2))
val buf = scratch
data.get(buf, 0, len)
data.position(startPos)
var writeLen = 0
var firstModified = -1
var strippedCount = 0
var nalStartIndex = -1
var startCodeLen = 0
var i = startPos
while (i <= limit) {
var i = 0
while (i <= len) {
// Find next start code or end of buffer.
val atEnd = i == limit
val atEnd = i == len
var foundStartCode = false
var nextStartCodeLen = 0
if (!atEnd && i + 2 < limit && data.get(i).toInt() == 0 && data.get(i + 1).toInt() == 0) {
if (data.get(i + 2).toInt() == 1) {
if (!atEnd && i + 2 < len && buf[i].toInt() == 0 && buf[i + 1].toInt() == 0) {
if (buf[i + 2].toInt() == 1) {
foundStartCode = true
nextStartCodeLen = 3
} else if (data.get(i + 2).toInt() == 0 && i + 3 < limit && data.get(i + 3).toInt() == 1) {
} else if (buf[i + 2].toInt() == 0 && i + 3 < len && buf[i + 3].toInt() == 1) {
foundStartCode = true
nextStartCodeLen = 4
}
@@ -61,23 +85,25 @@ object DvBitstreamSanitizer {
if (nalEnd - nalDataStart >= 2) {
// HEVC NAL header: forbidden_zero_bit(1) + nal_unit_type(6) + nuh_layer_id MSB(1).
val nalUnitType = (data.get(nalDataStart).toInt() and 0x7E) shr 1
val nalUnitType = (buf[nalDataStart].toInt() and 0x7E) shr 1
strip = when (nalUnitType) {
NAL_TYPE_UNSPEC62, NAL_TYPE_UNSPEC63 -> stripDvRpu
NAL_TYPE_PREFIX_SEI, NAL_TYPE_SUFFIX_SEI ->
stripHdr10PlusSei && isHdr10PlusSeiNalUnit(data, nalDataStart + 2, nalEnd)
stripHdr10PlusSei && isHdr10PlusSeiNalUnit(buf, nalDataStart + 2, nalEnd)
else -> false
}
}
if (!strip) {
if (writePos != nalStartIndex) {
for (j in nalStartIndex until nalEnd) {
data.put(writePos++, data.get(j))
}
} else {
writePos = nalEnd
if (strip) {
strippedCount++
if (firstModified < 0) firstModified = writeLen
} else {
if (writeLen != nalStartIndex) {
// Also reached with zero strips when bytes precede the first start code.
if (firstModified < 0) firstModified = writeLen
System.arraycopy(buf, nalStartIndex, buf, writeLen, nalEnd - nalStartIndex)
}
writeLen += nalEnd - nalStartIndex
}
}
nalStartIndex = i
@@ -88,8 +114,14 @@ object DvBitstreamSanitizer {
}
}
data.limit(writePos)
if (writeLen == len || firstModified < 0) return 0
// Bytes before firstModified are byte-identical to the buffer; write back only the tail.
data.position(startPos + firstModified)
data.put(buf, firstModified, writeLen - firstModified)
data.limit(startPos + writeLen)
data.position(startPos)
return strippedCount
}
/**
@@ -99,14 +131,14 @@ object DvBitstreamSanitizer {
* (ST 2094-40), application version 0 or 1. Malformed/truncated data returns false so the
* NAL is kept.
*/
private fun isHdr10PlusSeiNalUnit(data: ByteBuffer, rbspStart: Int, nalEnd: Int): Boolean {
private fun isHdr10PlusSeiNalUnit(buf: ByteArray, rbspStart: Int, nalEnd: Int): Boolean {
var pos = rbspStart
if (pos >= nalEnd) return false
// SEI payload type: accumulated 0xFF bytes plus the final byte.
var payloadType = 0
while (pos < nalEnd) {
val b = data.get(pos++).toInt() and 0xFF
val b = buf[pos++].toInt() and 0xFF
payloadType += b
if (b != 0xFF) break
}
@@ -114,7 +146,7 @@ object DvBitstreamSanitizer {
// SEI payload size, same encoding.
var payloadSize = 0
while (pos < nalEnd) {
val b = data.get(pos++).toInt() and 0xFF
val b = buf[pos++].toInt() and 0xFF
payloadSize += b
if (b != 0xFF) break
}
@@ -125,11 +157,11 @@ object DvBitstreamSanitizer {
// The identifier bytes (B5 00 3C 00 01 04 00/01) cannot contain the 0x000003 emulation
// prevention pattern, so they can be read without RBSP unescaping.
val countryCode = data.get(pos).toInt() and 0xFF
val providerCode = ((data.get(pos + 1).toInt() and 0xFF) shl 8) or (data.get(pos + 2).toInt() and 0xFF)
val orientedCode = ((data.get(pos + 3).toInt() and 0xFF) shl 8) or (data.get(pos + 4).toInt() and 0xFF)
val appIdentifier = data.get(pos + 5).toInt() and 0xFF
val appVersion = data.get(pos + 6).toInt() and 0xFF
val countryCode = buf[pos].toInt() and 0xFF
val providerCode = ((buf[pos + 1].toInt() and 0xFF) shl 8) or (buf[pos + 2].toInt() and 0xFF)
val orientedCode = ((buf[pos + 3].toInt() and 0xFF) shl 8) or (buf[pos + 4].toInt() and 0xFF)
val appIdentifier = buf[pos + 5].toInt() and 0xFF
val appVersion = buf[pos + 6].toInt() and 0xFF
return countryCode == 0xB5 &&
providerCode == 0x003C &&
android/app/src/main/kotlin/com/edde746/plezy/exoplayer/PlezyRenderersFactory.kt
+28 -1
View File
@@ -344,9 +344,18 @@ internal class DvSanitizingVideoRenderer(
private val log: ((String, String, String) -> Unit)?
) : MediaCodecVideoRenderer(builder) {
private val sanitizer = DvBitstreamSanitizer()
private var stripHdr10PlusSei = false
private var stripDvRpu = false
// Sanitize diagnostics — playback-thread only, cumulative for the renderer's lifetime.
private var sanitizedSampleCount = 0L
private var totalSanitizeTimeUs = 0L
private var maxSanitizeTimeUs = 0L
private var totalStrippedNals = 0L
private var totalStrippedBytes = 0L
override fun onCodecInitialized(
name: String,
configuration: MediaCodecAdapter.Configuration,
@@ -378,7 +387,25 @@ internal class DvSanitizingVideoRenderer(
if (stripHdr10PlusSei || stripDvRpu) {
val data = buffer.data
if (data != null && data.hasRemaining() && !buffer.isEncrypted) {
DvBitstreamSanitizer.sanitize(data, stripHdr10PlusSei, stripDvRpu)
val sizeBefore = data.remaining()
val startNs = System.nanoTime()
val stripped = sanitizer.sanitize(data, stripHdr10PlusSei, stripDvRpu)
val elapsedUs = (System.nanoTime() - startNs) / 1_000
sanitizedSampleCount++
totalSanitizeTimeUs += elapsedUs
if (elapsedUs > maxSanitizeTimeUs) maxSanitizeTimeUs = elapsedUs
totalStrippedNals += stripped
totalStrippedBytes += sizeBefore - data.remaining()
if (sanitizedSampleCount <= 3 || sanitizedSampleCount % 500 == 0L) {
log?.invoke(
"debug",
"dv-sanitize",
"Sample #$sanitizedSampleCount: ${sizeBefore}B -> ${data.remaining()}B, " +
"stripped=$stripped, took=${elapsedUs}us " +
"(avg=${totalSanitizeTimeUs / sanitizedSampleCount}us, max=${maxSanitizeTimeUs}us, " +
"totalNals=$totalStrippedNals, totalBytes=${totalStrippedBytes}B)"
)
}
}
}
super.onQueueInputBuffer(buffer)
android/app/src/test/kotlin/com/edde746/plezy/exoplayer/DvBitstreamSanitizerTest.kt
+128 -16
View File
@@ -8,6 +8,8 @@ import org.junit.Test
class DvBitstreamSanitizerTest {
private val sanitizer = DvBitstreamSanitizer()
// --- HDR10+ SEI stripping (native DV codec path) ---
@Test
@@ -17,7 +19,7 @@ class DvBitstreamSanitizerTest {
val buffer = bufferOf(vcl1, hdr10PlusSei(), vcl2)
val originalLimit = buffer.limit()
DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
assertArrayEquals(concat(vcl1, vcl2), remainingBytes(buffer))
assertTrue(buffer.limit() < originalLimit)
@@ -30,7 +32,7 @@ class DvBitstreamSanitizerTest {
val suffixSei = annexBNal(40, hdr10PlusSeiPayload())
val buffer = bufferOf(vcl, suffixSei)
DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
assertArrayEquals(vcl, remainingBytes(buffer))
}
@@ -42,7 +44,7 @@ class DvBitstreamSanitizerTest {
val vcl2 = annexBNal(1, byteArrayOf(0x03), startCodeLen = 3)
val buffer = bufferOf(vcl1, sei, vcl2)
DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
assertArrayEquals(concat(vcl1, vcl2), remainingBytes(buffer))
}
@@ -57,7 +59,7 @@ class DvBitstreamSanitizerTest {
val buffer = bufferOf(annexBNal(1, byteArrayOf(0x01)), sei, annexBNal(1, byteArrayOf(0x02)))
val original = remainingBytes(buffer)
DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
assertArrayEquals(original, remainingBytes(buffer))
}
@@ -67,7 +69,7 @@ class DvBitstreamSanitizerTest {
val buffer = bufferOf(annexBNal(1, byteArrayOf(0x01, 0x02)), annexBNal(1, byteArrayOf(0x03)))
val original = remainingBytes(buffer)
DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = true)
sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = true)
assertArrayEquals(original, remainingBytes(buffer))
}
@@ -79,7 +81,7 @@ class DvBitstreamSanitizerTest {
val buffer = bufferOf(annexBNal(1, byteArrayOf(0x01)), truncated)
val original = remainingBytes(buffer)
DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
assertArrayEquals(original, remainingBytes(buffer))
}
@@ -89,7 +91,7 @@ class DvBitstreamSanitizerTest {
val buffer = bufferOf(annexBNal(1, byteArrayOf(0x01)), hdr10PlusSei())
val original = remainingBytes(buffer)
DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = false, stripDvRpu = false)
sanitizer.sanitize(buffer, stripHdr10PlusSei = false, stripDvRpu = false)
assertArrayEquals(original, remainingBytes(buffer))
}
@@ -104,7 +106,7 @@ class DvBitstreamSanitizerTest {
val sei = hdr10PlusSei()
val buffer = bufferOf(vcl, rpu, sei, el)
DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = false, stripDvRpu = true)
sanitizer.sanitize(buffer, stripHdr10PlusSei = false, stripDvRpu = true)
assertArrayEquals(concat(vcl, sei), remainingBytes(buffer))
}
@@ -115,7 +117,7 @@ class DvBitstreamSanitizerTest {
val vcl2 = annexBNal(1, byteArrayOf(0x05))
val buffer = bufferOf(vcl1, annexBNal(62, byteArrayOf(0x19)), hdr10PlusSei(), vcl2)
DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = true)
sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = true)
assertArrayEquals(concat(vcl1, vcl2), remainingBytes(buffer))
}
@@ -130,7 +132,7 @@ class DvBitstreamSanitizerTest {
val buffer = ByteBuffer.wrap(content.copyOf())
buffer.position(prefix.size)
DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
assertEquals(prefix.size, buffer.position())
assertArrayEquals(vcl, remainingBytes(buffer))
@@ -142,12 +144,9 @@ class DvBitstreamSanitizerTest {
@Test
fun worksOnDirectBuffers() {
val vcl = annexBNal(1, byteArrayOf(0x01, 0x02))
val content = concat(vcl, hdr10PlusSei())
val buffer = ByteBuffer.allocateDirect(content.size)
buffer.put(content)
buffer.flip()
val buffer = directBufferOf(vcl, hdr10PlusSei())
DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
assertArrayEquals(vcl, remainingBytes(buffer))
}
@@ -156,12 +155,117 @@ class DvBitstreamSanitizerTest {
fun emptyBufferIsNoOp() {
val buffer = ByteBuffer.allocate(0)
DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = true)
sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = true)
assertEquals(0, buffer.position())
assertEquals(0, buffer.limit())
}
@Test
fun stripsOnDirectBufferWithNonZeroPosition() {
val vcl1 = annexBNal(1, byteArrayOf(0x01, 0x02))
val vcl2 = annexBNal(1, byteArrayOf(0x03))
val buffer = directBufferOf(byteArrayOf(0xAA.toByte(), 0xBB.toByte()), vcl1, hdr10PlusSei(), vcl2)
buffer.position(2)
val stripped = sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
assertEquals(1, stripped)
assertEquals(2, buffer.position())
assertArrayEquals(concat(vcl1, vcl2), remainingBytes(buffer))
// Bytes before the position are untouched.
assertEquals(0xAA.toByte(), buffer.get(0))
assertEquals(0xBB.toByte(), buffer.get(1))
}
@Test
fun noStripLeavesDirectBufferStateUntouched() {
val vcl1 = annexBNal(1, byteArrayOf(0x01, 0x02))
val vcl2 = annexBNal(1, byteArrayOf(0x03))
val buffer = directBufferOf(vcl1, vcl2)
val originalLimit = buffer.limit()
val stripped = sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = true)
assertEquals(0, stripped)
assertEquals(0, buffer.position())
assertEquals(originalLimit, buffer.limit())
assertArrayEquals(concat(vcl1, vcl2), remainingBytes(buffer))
}
@Test
fun stripsTrailingRpuWithZeroByteTailWriteBack() {
val vcl1 = annexBNal(1, byteArrayOf(0x01, 0x02))
val vcl2 = annexBNal(1, byteArrayOf(0x03))
val rpu = annexBNal(62, byteArrayOf(0x19, 0x08))
val buffer = directBufferOf(vcl1, vcl2, rpu)
val originalLimit = buffer.limit()
val stripped = sanitizer.sanitize(buffer, stripHdr10PlusSei = false, stripDvRpu = true)
assertEquals(1, stripped)
assertEquals(originalLimit - rpu.size, buffer.limit())
assertArrayEquals(concat(vcl1, vcl2), remainingBytes(buffer))
}
@Test
fun growsScratchForLargeSamples() {
// Exceeds the initial 256KB scratch and one doubling; non-zero filler so no
// accidental start codes appear in the payloads.
val vcl1 = annexBNal(1, ByteArray(800_000) { 0xAB.toByte() })
val vcl2 = annexBNal(1, ByteArray(700_000) { 0xCD.toByte() })
val buffer = bufferOf(vcl1, hdr10PlusSei(), vcl2)
val stripped = sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
assertEquals(1, stripped)
assertArrayEquals(concat(vcl1, vcl2), remainingBytes(buffer))
}
@Test
fun reusesSanitizerAcrossSequentialBuffers() {
val vcl1 = annexBNal(1, byteArrayOf(0x01, 0x02))
val vcl2 = annexBNal(1, byteArrayOf(0x03, 0x04))
val first = bufferOf(vcl1, hdr10PlusSei(), vcl2)
assertEquals(1, sanitizer.sanitize(first, stripHdr10PlusSei = true, stripDvRpu = false))
assertArrayEquals(concat(vcl1, vcl2), remainingBytes(first))
val second = bufferOf(vcl1, vcl2)
assertEquals(0, sanitizer.sanitize(second, stripHdr10PlusSei = true, stripDvRpu = false))
assertArrayEquals(concat(vcl1, vcl2), remainingBytes(second))
val third = bufferOf(hdr10PlusSei(), vcl1, annexBNal(62, byteArrayOf(0x19)), vcl2, hdr10PlusSei())
assertEquals(3, sanitizer.sanitize(third, stripHdr10PlusSei = true, stripDvRpu = true))
assertArrayEquals(concat(vcl1, vcl2), remainingBytes(third))
}
@Test
fun keepsBufferWithoutStartCodes() {
val content = byteArrayOf(0x12, 0x34, 0x56, 0x78)
val buffer = ByteBuffer.wrap(content.copyOf())
val stripped = sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = true)
assertEquals(0, stripped)
assertEquals(0, buffer.position())
assertEquals(content.size, buffer.limit())
assertArrayEquals(content, remainingBytes(buffer))
}
@Test
fun stripsMultipleSeisInOneAccessUnit() {
val vcl1 = annexBNal(1, byteArrayOf(0x01, 0x02))
val vcl2 = annexBNal(1, byteArrayOf(0x03))
val suffixSei = annexBNal(40, hdr10PlusSeiPayload())
val buffer = bufferOf(vcl1, hdr10PlusSei(), vcl2, suffixSei)
val stripped = sanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
assertEquals(2, stripped)
assertArrayEquals(concat(vcl1, vcl2), remainingBytes(buffer))
}
// --- Helpers ---
/** Builds an HEVC NAL unit: start code + 2-byte NAL header encoding [nalUnitType] + payload. */
@@ -193,6 +297,14 @@ class DvBitstreamSanitizerTest {
private fun bufferOf(vararg parts: ByteArray): ByteBuffer = ByteBuffer.wrap(concat(*parts))
private fun directBufferOf(vararg parts: ByteArray): ByteBuffer {
val content = concat(*parts)
val buffer = ByteBuffer.allocateDirect(content.size)
buffer.put(content)
buffer.flip()
return buffer
}
private fun remainingBytes(buffer: ByteBuffer): ByteArray {
val copy = ByteArray(buffer.remaining())
buffer.duplicate().get(copy)