From 6c96830033cb81c0671cdfee18e67fa3b1f97ed7 Mon Sep 17 00:00:00 2001 From: edde746 <86283021+edde746@users.noreply.github.com> Date: Mon, 29 Jun 2026 02:08:24 +0200 Subject: [PATCH] fix(android): never-drop multi-page subtitle atlas --- android/libass/consumer-rules.pro | 2 - android/libass/src/main/cpp/AssKt.c | 239 +++++++++++++----- .../com/edde746/plezy/libass/AssAtlasFrame.kt | 57 ++++- .../com/edde746/plezy/libass/AssRender.kt | 68 +++-- .../media/widget/AssSubtitleAtlasPipeline.kt | 178 ++++++++----- 5 files changed, 389 insertions(+), 155 deletions(-) diff --git a/android/libass/consumer-rules.pro b/android/libass/consumer-rules.pro index 9ab5d082..16ee3382 100644 --- a/android/libass/consumer-rules.pro +++ b/android/libass/consumer-rules.pro @@ -1,5 +1,3 @@ -# Constructed from JNI via FindClass/NewObject (AssKt.c). --keep class com.edde746.plezy.libass.AssAtlasFrame { *; } # JNI exports bind by name (Java_com_edde746_plezy_libass_*); keep the names stable. -keepclasseswithmembernames class com.edde746.plezy.libass.* { native ; diff --git a/android/libass/src/main/cpp/AssKt.c b/android/libass/src/main/cpp/AssKt.c index c62fefc3..25dea8fd 100644 --- a/android/libass/src/main/cpp/AssKt.c +++ b/android/libass/src/main/cpp/AssKt.c @@ -237,17 +237,25 @@ Java_com_edde746_plezy_libass_AssRender_nativeAssRenderDeinit(JNIEnv* env, jclas } } -// A tile is a <= atlasMaxW x atlasMaxH sub-rect of an ASS_Image. Splitting wide -// or tall images into tiles lets a full-screen sign whose line bitmaps exceed -// the atlas pack completely instead of being dropped (issue #1436: a 4K-rendered -// sign produces line bitmaps wider than a 2048 atlas). Tiles are built in list -// order (= libass blend/painter order, preserved for pass 2); packing runs -// height-sorted via a separate key array so emission order is untouched. +// Hard cap on atlas pages (see the packing comment below). 4 pages of a GL-max +// texture is far above the worst real frame measured (a 4K-rendered full-screen +// typeset letter needs 3); beyond it tiles are dropped and counted in truncated. +#define MAX_ATLAS_PAGES 4 + +// A tile is a <= atlasMaxW x atlasMaxH sub-rect of an ASS_Image. A single image +// can exceed one atlas page only when the render frame is larger than a page +// (>4K, or a GPU whose max texture is below the frame) — multi-page can't split +// one image across pages (a quad samples one texture), so tiling does, keeping +// the never-drop guarantee. (#1436 itself was atlas-AREA overflow, fixed by the +// multi-page pack below, not oversized single images.) Tiles are built in list +// order (= libass blend/painter order, preserved for emission); the single-page +// pack runs height-sorted via a separate key array so emission order is untouched. typedef struct { ASS_Image* img; // source image (for bitmap/stride/color/dst_x/dst_y) int ox, oy; // tile offset within the source bitmap int tw, th; // tile size (<= atlasMaxW x atlasMaxH) - int sx, sy; // packed slot in the atlas; -1 if dropped for capacity + int page; // atlas page the tile is packed into; -1 if dropped for capacity + int sx, sy; // packed slot within the page; valid when page >= 0 } PackTile; typedef struct { @@ -269,34 +277,69 @@ static int imageListHasOutput(ASS_Image* image) { // Throttle for truncation warnings (shared across renderers; logging only). static int truncationLogCounter = 0; +// Frame metadata crosses to Kotlin through a fixed-layout int[] header (filled here, +// read + turned into an AssAtlasFrame by AssRender.kt) instead of constructing the +// object in JNI. A NewObject on an overloaded constructor is fragile under R8: the +// minified release build stripped/rewrote the (I[I[IIIIZ)V ctor the lookup bound by, +// crashing with NoSuchMethodError (#1436 follow-up). Binding a native method by name +// + populating a primitive array has no such reflective dependency. Layout: +// [0]=atlasWidth [1]=quadCount [2]=changed [3]=truncated [4]=requiredPages +// [5]=hasOutput [6]=pageCount +// [7 .. 7+MAX-1] = pageHeights[pageCount] +// [7+MAX .. 7+2*MAX-1] = pageQuadCounts[pageCount] +#define ASS_HEADER_INTS (7 + 2 * MAX_ATLAS_PAGES) + +static jint writeAtlasHeader( + JNIEnv* env, jintArray headerBuf, int atlasWidth, int quadCount, int changed, int truncated, int requiredPages, + int hasOutput, int pageCount, const int* pageHeights, const int* pageQuads) { + int hdr[ASS_HEADER_INTS]; + memset(hdr, 0, sizeof(hdr)); + hdr[0] = atlasWidth; + hdr[1] = quadCount; + hdr[2] = changed; + hdr[3] = truncated; + hdr[4] = requiredPages; + hdr[5] = hasOutput; + hdr[6] = pageCount; + for (int i = 0; i < pageCount && i < MAX_ATLAS_PAGES; i++) { + hdr[7 + i] = pageHeights ? pageHeights[i] : 0; + hdr[7 + MAX_ATLAS_PAGES + i] = pageQuads ? pageQuads[i] : 0; + } + (*env)->SetIntArrayRegion(env, headerBuf, 0, ASS_HEADER_INTS, hdr); + return 1; +} + // Renders a frame into the provided atlas + vertex direct ByteBuffers. // -// - atlasBuf holds packed ALPHA_8 pixels with row stride atlasMaxW. Only the first -// atlasHeight rows are written; the caller uploads that region to a texture -// allocated once at atlasMaxW × atlasMaxH. +// - atlasBuf holds one or more vertically-stacked ALPHA_8 *pages*, each atlasMaxW × +// atlasMaxH (row stride atlasMaxW); page p starts at byte offset p*atlasMaxW*atlasMaxH. +// The buffer's capacity bounds how many pages this render may fill; AssAtlasFrame +// reports pageHeights (rows worth uploading per page) and requiredPages. // - vertexBuf holds a per-quad vertex stream (6 vertices × (2 pos + 2 uv + 4 color) // floats = 48 floats = 192 bytes per quad). Must match BYTES_PER_QUAD/VERTEX in -// AssSubtitleAtlasPipeline.kt. Ready for a single glDrawArrays(GL_TRIANGLES, 0, N * 6). -// - UVs are normalized against atlasMaxW × atlasMaxH (the allocated texture dims), -// not the packed region, so the texture never needs reallocation. -// - Images are packed in height-sorted rows (minimizes packed height) but vertices -// are emitted in original list order — the list order is libass's painter order. +// AssSubtitleAtlasPipeline.kt. UVs are page-local, normalized against atlasMaxW × +// atlasMaxH (the per-page texture dims). +// - The common case packs everything into a single height-sorted page (minimizes +// packed height, byte-identical to the prior single-page packer). When that +// overflows (a 4K full-screen sign can exceed one GL-max texture), the packer +// spills into additional pages in list order: page assignment is monotonic in +// libass's painter order, so each page's quads are one contiguous run in the +// vertex stream and drawing the pages in turn reproduces the blend order. +// - Vertices are always emitted in original list order (= libass's painter order). // -// Never fails on content size: images that don't fit the remaining atlas/vertex -// capacity are dropped and counted in AssAtlasFrame.truncated, so a heavy frame -// degrades instead of going stale. Returns NULL only for missing buffers/handles. -// On changed == 0, returns (0, 0, 0, changed, 0, hasOutput) without touching the -// buffers. hasOutput lets Kotlin distinguish "reuse the previous atlas" from -// "the current frame is blank and the GL surface must be cleared." -JNIEXPORT jobject JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderFrameAtlas( +// Never fails on content size: when the frame needs more pages than the buffer holds +// (requiredPages > pageHeights.size) the caller grows the buffer and re-renders; any +// genuinely undrawable tiles (past MAX_ATLAS_PAGES / the vertex budget) are dropped +// and counted in truncated. +// +// Returns 0 for missing buffers/handles (the caller maps that to a null frame); 1 when +// the header was written. On changed == 0 the header carries (atlasWidth=0, quadCount=0, +// changed, hasOutput) without touching the atlas/vertex buffers — hasOutput lets Kotlin +// distinguish "reuse the previous atlas" from "blank, clear the GL surface." +JNIEXPORT jint JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRenderFrameAtlas( JNIEnv* env, jclass clazz, jlong render, jlong track, jlong time, jobject atlasBuf, jint atlasMaxW, jint atlasMaxH, - jobject vertexBuf) { - if (!render || !track || !atlasBuf || !vertexBuf || atlasMaxW <= 0 || atlasMaxH <= 0) return NULL; - - jclass atlasFrameClass = (*env)->FindClass(env, "com/edde746/plezy/libass/AssAtlasFrame"); - if (!atlasFrameClass) return NULL; - jmethodID ctor = (*env)->GetMethodID(env, atlasFrameClass, "", "(IIIIIZ)V"); - if (!ctor) return NULL; + jobject vertexBuf, jintArray headerBuf) { + if (!render || !track || !atlasBuf || !vertexBuf || !headerBuf || atlasMaxW <= 0 || atlasMaxH <= 0) return 0; const long long t0 = nowMs(); int changed; @@ -304,13 +347,13 @@ JNIEXPORT jobject JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRende const long long tAss = nowMs(); if (changed == 0) { - const jboolean hasOutput = imageListHasOutput(image) ? JNI_TRUE : JNI_FALSE; + const int hasOutput = imageListHasOutput(image) ? 1 : 0; if (tAss - t0 > 40) { __android_log_print( ANDROID_LOG_WARN, LOG_TAG, "slow render t=%lldms: ass=%lldms (changed=%d, hasOutput=%d)", (long long)time, - tAss - t0, changed, hasOutput == JNI_TRUE); + tAss - t0, changed, hasOutput); } - return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, 0, hasOutput); + return writeAtlasHeader(env, headerBuf, 0, 0, changed, 0, 1, hasOutput, 1, NULL, NULL); } if (image == NULL) { @@ -319,26 +362,30 @@ JNIEXPORT jobject JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRende ANDROID_LOG_WARN, LOG_TAG, "slow render t=%lldms: ass=%lldms (changed=%d, no output)", (long long)time, tAss - t0, changed); } - return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, 0, JNI_FALSE); + return writeAtlasHeader(env, headerBuf, 0, 0, changed, 0, 1, 0, 1, NULL, NULL); } uint8_t* atlasPixels = (uint8_t*)(*env)->GetDirectBufferAddress(env, atlasBuf); jlong atlasCap = (*env)->GetDirectBufferCapacity(env, atlasBuf); float* vertices = (float*)(*env)->GetDirectBufferAddress(env, vertexBuf); jlong vertexCap = (*env)->GetDirectBufferCapacity(env, vertexBuf); - if (!atlasPixels || !vertices) return NULL; + if (!atlasPixels || !vertices) return 0; if ((jlong)atlasMaxW * atlasMaxH > atlasCap) { __android_log_print( ANDROID_LOG_ERROR, LOG_TAG, "atlas buffer smaller than %dx%d (capacity %lld bytes)", atlasMaxW, atlasMaxH, (long long)atlasCap); - return NULL; + return 0; } // 48 floats per quad × 4 bytes = 192 bytes/quad const int maxQuads = (int)(vertexCap / 192); + const size_t pageBytes = (size_t)atlasMaxW * atlasMaxH; + int providedPages = (int)(atlasCap / (jlong)pageBytes); + if (providedPages < 1) return 0; // one page is guaranteed above; keep the page math safe + if (providedPages > MAX_ATLAS_PAGES) providedPages = MAX_ATLAS_PAGES; // Split every image into <= atlasMaxW x atlasMaxH tiles, then pack the tiles. - // tiles[] stays in list order (= blend/painter order for pass 2); keys[] is - // sorted by height so packing produces tight rows without disturbing it. + // tiles[] stays in list order (= blend/painter order for emission); keys[] is + // sorted by height for the single-page pack so it produces tight rows. int total = 0; for (ASS_Image* img = image; img != NULL; img = img->next) { if (img->w > 0 && img->h > 0) { @@ -348,7 +395,7 @@ JNIEXPORT jobject JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRende } } if (total == 0) { - return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, 0, JNI_FALSE); + return writeAtlasHeader(env, headerBuf, 0, 0, changed, 0, 1, 0, 1, NULL, NULL); } PackTile* tiles = (PackTile*)malloc(sizeof(PackTile) * (size_t)total); @@ -356,7 +403,7 @@ JNIEXPORT jobject JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRende if (!tiles || !keys) { free(tiles); free(keys); - return NULL; + return 0; } int n = 0; long long srcPixels = 0; @@ -369,65 +416,121 @@ JNIEXPORT jobject JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRende for (int ox = 0; ox < img->w; ox += atlasMaxW) { int tw = img->w - ox; if (tw > atlasMaxW) tw = atlasMaxW; - tiles[n] = (PackTile){.img = img, .ox = ox, .oy = oy, .tw = tw, .th = th, .sx = -1, .sy = -1}; + tiles[n] = (PackTile){.img = img, .ox = ox, .oy = oy, .tw = tw, .th = th, .page = -1, .sx = -1, .sy = -1}; keys[n] = (TileSortKey){.th = th, .idx = n}; n++; } } } - qsort(keys, (size_t)n, sizeof(TileSortKey), compareTileKeysByHeightDesc); - - int cursorX = 0, cursorY = 0, rowH = 0; + int pageHeights[MAX_ATLAS_PAGES] = {0}; + int pageQuads[MAX_ATLAS_PAGES] = {0}; + int pageCount = 1; + int requiredPages = 1; int truncated = 0; - int packedH = 0; - int accepted = 0; + + // Pass 1a: height-sorted single page — the common case, minimal packed height + // (byte-identical to the prior single-page packer when the frame fits one page). + qsort(keys, (size_t)n, sizeof(TileSortKey), compareTileKeysByHeightDesc); + int cursorX = 0, cursorY = 0, rowH = 0, packedH = 0, accepted = 0; for (int i = 0; i < n; i++) { PackTile* t = &tiles[keys[i].idx]; - if (t->tw <= atlasMaxW && accepted < maxQuads) { - int cx = cursorX, cy = cursorY, rh = rowH; + if (accepted >= maxQuads) break; + int cx = cursorX, cy = cursorY, rh = rowH; + if (cx + t->tw > atlasMaxW) { + cy += rh; + cx = 0; + rh = 0; + } + if (cy + t->th > atlasMaxH) continue; // doesn't fit a single page + t->page = 0; + t->sx = cx; + t->sy = cy; + cursorX = cx + t->tw; + cursorY = cy; + rowH = (t->th > rh) ? t->th : rh; + if (cy + t->th > packedH) packedH = cy + t->th; + accepted++; + } + + if (accepted == n) { + pageHeights[0] = packedH; + pageQuads[0] = accepted; + } else { + // Pass 1b: the frame overflows one page. Re-pack in list order, starting a new + // page whenever a tile won't fit the current one. List order keeps the page + // index monotonic in painter order, so each page's quads stay one contiguous run. + for (int i = 0; i < n; i++) { + tiles[i].page = -1; + tiles[i].sx = -1; + tiles[i].sy = -1; + } + int page = 0, cx = 0, cy = 0, rh = 0, placed = 0; + for (int i = 0; i < n; i++) { + PackTile* t = &tiles[i]; if (cx + t->tw > atlasMaxW) { cy += rh; cx = 0; rh = 0; } - if (cy + t->th <= atlasMaxH) { + if (cy + t->th > atlasMaxH) { + page++; + cx = 0; + cy = 0; + rh = 0; + } + if (page + 1 > requiredPages) requiredPages = page + 1; + if (page < providedPages && placed < maxQuads) { + t->page = page; t->sx = cx; t->sy = cy; - cursorX = cx + t->tw; - cursorY = cy; - rowH = (t->th > rh) ? t->th : rh; - if (cy + t->th > packedH) packedH = cy + t->th; - accepted++; + if (cy + t->th > pageHeights[page]) pageHeights[page] = cy + t->th; + pageQuads[page]++; + placed++; } + cx += t->tw; + rh = (t->th > rh) ? t->th : rh; } - if (t->sx < 0) truncated++; + pageCount = (requiredPages < providedPages) ? requiredPages : providedPages; + accepted = placed; + truncated = n - placed; } - if (truncated > 0 && (truncationLogCounter++ & 63) == 0) { + // Warn only for genuinely-unrecoverable loss. A frame that needs more pages than + // the buffer currently holds, yet fits within MAX_ATLAS_PAGES and the vertex + // budget, is recoverable: the caller grows the buffer and re-renders, so the + // first (discarded) render's truncated > 0 is a false alarm, not data loss. Tiles + // are only truly lost past the page cap or the vertex budget. + const int recoverableGrow = requiredPages <= MAX_ATLAS_PAGES && n <= maxQuads; + if (truncated > 0 && !recoverableGrow && (truncationLogCounter++ & 63) == 0) { __android_log_print( - ANDROID_LOG_WARN, LOG_TAG, "atlas truncation: %d of %d tiles dropped (atlas %dx%d, %d quads max)", truncated, n, - atlasMaxW, atlasMaxH, maxQuads); + ANDROID_LOG_WARN, LOG_TAG, "atlas truncation: %d of %d tiles dropped (atlas %dx%d, need %d pages have %d)", + truncated, n, atlasMaxW, atlasMaxH, requiredPages, providedPages); } if (accepted == 0) { free(tiles); free(keys); - return (*env)->NewObject(env, atlasFrameClass, ctor, 0, 0, 0, changed, truncated, JNI_TRUE); + return writeAtlasHeader(env, headerBuf, 0, 0, changed, truncated, requiredPages, 1, 1, NULL, NULL); } - memset(atlasPixels, 0, (size_t)atlasMaxW * packedH); + // Clear only the packed rows of each written page. + for (int p = 0; p < pageCount; p++) { + memset(atlasPixels + (size_t)p * pageBytes, 0, (size_t)atlasMaxW * pageHeights[p]); + } - // Pass 2: emit tiles in build order (= libass's painter/blend order), copying - // each placed tile into its slot and emitting its quad. + // Emit tiles in list order (= libass's painter/blend order), copying each placed + // tile into its page slot and emitting its quad. Monotonic page assignment makes + // each page's quads a contiguous run, matching pageQuads[] for the per-page draw. int qi = 0; for (int i = 0; i < n; i++) { PackTile* t = &tiles[i]; - if (t->sx < 0) continue; + if (t->page < 0) continue; ASS_Image* img = t->img; const int px = t->sx; const int py = t->sy; + uint8_t* pageBase = atlasPixels + (size_t)t->page * pageBytes; for (int y = 0; y < t->th; y++) { - uint8_t* dst = atlasPixels + (size_t)(py + y) * atlasMaxW + px; + uint8_t* dst = pageBase + (size_t)(py + y) * atlasMaxW + px; const uint8_t* src = img->bitmap + (size_t)(t->oy + y) * img->stride + t->ox; memcpy(dst, src, (size_t)t->tw); } @@ -511,13 +614,15 @@ JNIEXPORT jobject JNICALL Java_com_edde746_plezy_libass_AssRender_nativeAssRende if (tEnd - t0 > 40) { __android_log_print( ANDROID_LOG_WARN, LOG_TAG, - "slow render t=%lldms: total=%lldms ass=%lldms pack+copy=%lldms images=%d srcPx=%lldk atlas=%dx%d quads=%d", - (long long)time, tEnd - t0, tAss - t0, tEnd - tAss, n, srcPixels / 1000, atlasMaxW, packedH, qi); + "slow render t=%lldms: total=%lldms ass=%lldms pack+copy=%lldms images=%d srcPx=%lldk " + "atlas=%dx%d pages=%d quads=%d", + (long long)time, tEnd - t0, tAss - t0, tEnd - tAss, n, srcPixels / 1000, atlasMaxW, atlasMaxH, pageCount, qi); } // atlasWidth is the full row stride (GLES2 can't upload with stride ≠ width); - // atlasHeight is the packed height — the only rows worth uploading. - return (*env)->NewObject(env, atlasFrameClass, ctor, atlasMaxW, packedH, qi, changed, truncated, JNI_TRUE); + // pageHeights/pageQuadCounts describe the per-page upload + draw ranges. + return writeAtlasHeader( + env, headerBuf, atlasMaxW, qi, changed, truncated, requiredPages, 1, pageCount, pageHeights, pageQuads); } // --- AssFrameTimestamps (EGL_ANDROID_get_frame_timestamps) --- diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/AssAtlasFrame.kt b/android/libass/src/main/java/com/edde746/plezy/libass/AssAtlasFrame.kt index 8d259636..18993c65 100644 --- a/android/libass/src/main/java/com/edde746/plezy/libass/AssAtlasFrame.kt +++ b/android/libass/src/main/java/com/edde746/plezy/libass/AssAtlasFrame.kt @@ -4,25 +4,60 @@ package com.edde746.plezy.libass * Result of a packed-atlas render. The atlas pixel data is stored in the direct ByteBuffer * that was passed into [AssRender.renderFrameAtlas]; the vertex stream is in the other. * - * @param atlasWidth atlas row stride in pixels (= the allocated width; 0 when [changed] == 0) - * @param atlasHeight packed atlas height in pixels — the rows worth uploading - * @param quadCount number of quads; the vertex buffer holds [quadCount] * 6 vertices - * @param changed libass change flag (0 = no change, 1 = positions, 2 = content) - * @param truncated images dropped because they exceeded the atlas/vertex capacity; - * the frame is incomplete but never stale (> 0 should be rare — - * it means even the GL-max-sized atlas couldn't fit the frame) - * @param hasOutput true when libass reported at least one visible image for this - * timestamp, even when [changed] is 0 and the buffers were not - * rewritten. false means this timestamp should be blank. + * The atlas may span more than one *page* — a heavy full-screen sign can produce more + * sub-pixels than a single GL-max texture holds. Pages are vertically stacked in the + * atlas ByteBuffer (page `p` at byte offset `p * atlasWidth * atlasMaxHeight`), each its + * own texture, and are drawn in turn. Quads are emitted in libass painter order and page + * assignment is monotonic in that order, so each page's quads form one contiguous run in + * the vertex stream ([pageQuadCounts]); the runner uploads page `p`, then draws its run. + * + * Built in Kotlin by [AssRender.renderFrameAtlas] from the int[] header the native + * renderer fills (see `writeAtlasHeader` in AssKt.c) — never constructed from JNI, so + * the minifier may obfuscate it freely without breaking the native boundary. + * + * @param atlasWidth atlas row stride in pixels (= the allocated width; same for every + * page; 0 when [changed] == 0) + * @param pageHeights packed height (rows worth uploading) of each page; `size` = page count + * @param pageQuadCounts quads on each page, contiguous in the vertex stream in this order; + * `size` = page count, `sum` = [quadCount] + * @param quadCount total quads; the vertex buffer holds [quadCount] * 6 vertices + * @param changed libass change flag (0 = no change, 1 = positions, 2 = content) + * @param truncated images dropped because the frame needed more than [requiredPages] + * pages of capacity or exceeded the vertex budget; the frame is + * incomplete but never stale (should be unreachable for real content) + * @param requiredPages pages this frame needs to render completely. When it exceeds + * [pageHeights].size the caller must grow the atlas buffer and + * re-render; the rendered pages are still valid in the meantime. + * @param hasOutput true when libass reported at least one visible image for this + * timestamp, even when [changed] is 0 and the buffers were not + * rewritten. false means this timestamp should be blank. */ class AssAtlasFrame( val atlasWidth: Int, - val atlasHeight: Int, + val pageHeights: IntArray, + val pageQuadCounts: IntArray, val quadCount: Int, val changed: Int, val truncated: Int, + val requiredPages: Int, val hasOutput: Boolean ) { + /** Number of atlas pages this frame occupies. */ + val pageCount: Int get() = pageHeights.size + + /** Packed height of the first page; the only page in the common single-page case. */ + val atlasHeight: Int get() = if (pageHeights.isNotEmpty()) pageHeights[0] else 0 + + /** Single-page convenience: blank/unchanged frames and tests. */ + constructor( + atlasWidth: Int, + atlasHeight: Int, + quadCount: Int, + changed: Int, + truncated: Int, + hasOutput: Boolean + ) : this(atlasWidth, intArrayOf(atlasHeight), intArrayOf(quadCount), quadCount, changed, truncated, 1, hasOutput) + constructor( atlasWidth: Int, atlasHeight: Int, diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/AssRender.kt b/android/libass/src/main/java/com/edde746/plezy/libass/AssRender.kt index c5d06a2a..35fac0e9 100644 --- a/android/libass/src/main/java/com/edde746/plezy/libass/AssRender.kt +++ b/android/libass/src/main/java/com/edde746/plezy/libass/AssRender.kt @@ -8,6 +8,10 @@ class AssRender(nativeAss: Long, private val lock: ReentrantLock) { companion object { + /** Must match MAX_ATLAS_PAGES + the header layout in AssKt.c (`writeAtlasHeader`). */ + private const val MAX_ATLAS_PAGES = 4 + private const val HEADER_INTS = 7 + 2 * MAX_ATLAS_PAGES + @JvmStatic external fun nativeAssRenderInit(ass: Long): Long @@ -29,6 +33,12 @@ class AssRender(nativeAss: Long, private val lock: ReentrantLock) { @JvmStatic external fun nativeAssRenderSetUseMargins(render: Long, use: Boolean) + /** + * Renders into [atlasBuf]/[vertexBuf] and writes frame metadata into [header] + * (layout per `writeAtlasHeader` in AssKt.c). Returns 1 when the header was written, + * 0 for missing buffers/handles. The frame object is built on the Kotlin side from + * the header so the JNI boundary never constructs it (R8-safe; see [renderFrameAtlas]). + */ @JvmStatic external fun nativeAssRenderFrameAtlas( render: Long, @@ -37,8 +47,9 @@ class AssRender(nativeAss: Long, private val lock: ReentrantLock) { atlasBuf: ByteBuffer, atlasMaxWidth: Int, atlasMaxHeight: Int, - vertexBuf: ByteBuffer - ): AssAtlasFrame? + vertexBuf: ByteBuffer, + header: IntArray + ): Int @JvmStatic external fun nativeAssRenderDeinit(render: Long) @@ -46,6 +57,10 @@ class AssRender(nativeAss: Long, private val lock: ReentrantLock) { private var nativeRender: Long = nativeAssRenderInit(nativeAss) + /** Reusable JNI frame-metadata header (see `writeAtlasHeader` in AssKt.c). Calls to + * [renderFrameAtlas] are serialized by [lock], so one buffer is safe to reuse. */ + private val frameHeader = IntArray(HEADER_INTS) + @Volatile var released = false private set @@ -109,26 +124,31 @@ class AssRender(nativeAss: Long, private val lock: ReentrantLock) { withNative { nativeAssRenderSetUseMargins(it, use) } } - /** - * Renders a frame into a packed ALPHA_8 texture atlas plus a single vertex stream - * ready for `glDrawArrays(GL_TRIANGLES, 0, quadCount * 6)`. - * - * UVs are normalized against ([atlasMaxW], [atlasMaxH]) — the allocated texture - * dims — so the caller can allocate the texture once and `glTexSubImage2D` only - * the packed rows. Images that exceed the capacity are dropped and counted in - * [AssAtlasFrame.truncated]; the render never fails on content size. - * - * @param atlasBuf direct ByteBuffer receiving the packed pixels (≥ atlasMaxW × atlasMaxH) - * @param atlasMaxW atlas row stride in pixels (bound by `GL_MAX_TEXTURE_SIZE`) - * @param atlasMaxH atlas height in pixels (bound by `GL_MAX_TEXTURE_SIZE`) - * @param vertexBuf direct ByteBuffer receiving the vertex stream (192 bytes per quad) - */ /** How long the most recent [renderFrameAtlas] waited to acquire the shared * libass lock (contended by track dialogue/font feeding), in milliseconds. */ @Volatile var lastLockWaitMs: Long = 0 private set + /** + * Renders a frame into a packed ALPHA_8 texture atlas plus a vertex stream. + * + * The atlas may span one or more vertically-stacked pages (a dense full-screen + * sign can exceed a single GL-max texture); vertices stay in libass painter order, + * grouped per page ([AssAtlasFrame.pageQuadCounts]). The caller uploads each page + * to its own texture and draws it with its own `glDrawArrays`, reproducing the + * blend order. UVs are page-local, normalized against ([atlasMaxW], [atlasMaxH]). + * + * Never fails on content size: when the frame needs more pages than [atlasBuf] + * holds, [AssAtlasFrame.requiredPages] signals the caller to grow the buffer and + * re-render; only tiles past `MAX_ATLAS_PAGES` or the vertex budget are dropped + * and counted in [AssAtlasFrame.truncated]. + * + * @param atlasBuf direct ByteBuffer receiving the stacked pages (≥ atlasMaxW × atlasMaxH per page) + * @param atlasMaxW per-page atlas row stride in pixels (bound by `GL_MAX_TEXTURE_SIZE`) + * @param atlasMaxH per-page atlas height in pixels (bound by `GL_MAX_TEXTURE_SIZE`) + * @param vertexBuf direct ByteBuffer receiving the vertex stream (192 bytes per quad) + */ fun renderFrameAtlas( time: Long, atlasBuf: ByteBuffer, @@ -142,7 +162,21 @@ class AssRender(nativeAss: Long, private val lock: ReentrantLock) { if (released || nativeRender == 0L) return null val t = track ?: return null if (t.released || t.nativeAssTrack == 0L) return null - return nativeAssRenderFrameAtlas(nativeRender, t.nativeAssTrack, time, atlasBuf, atlasMaxW, atlasMaxH, vertexBuf) + val header = frameHeader + val status = + nativeAssRenderFrameAtlas(nativeRender, t.nativeAssTrack, time, atlasBuf, atlasMaxW, atlasMaxH, vertexBuf, header) + if (status == 0) return null + val pageCount = header[6] + return AssAtlasFrame( + atlasWidth = header[0], + pageHeights = IntArray(pageCount) { header[7 + it] }, + pageQuadCounts = IntArray(pageCount) { header[7 + MAX_ATLAS_PAGES + it] }, + quadCount = header[1], + changed = header[2], + truncated = header[3], + requiredPages = header[4], + hasOutput = header[5] != 0 + ) } } diff --git a/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/AssSubtitleAtlasPipeline.kt b/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/AssSubtitleAtlasPipeline.kt index 898d1d6d..7f78235e 100644 --- a/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/AssSubtitleAtlasPipeline.kt +++ b/android/libass/src/main/java/com/edde746/plezy/libass/media/widget/AssSubtitleAtlasPipeline.kt @@ -28,10 +28,10 @@ import java.util.concurrent.locks.LockSupport /** * Atlas-rendering pipeline behind [AssSubtitleSurfaceView]. - * Runs libass on its own [HandlerThread] into a packed - * ALPHA_8 texture atlas plus a single vertex stream, and a GL thread that uploads - * both and issues one `glDrawArrays` per frame. Each timed swap is pinned to the - * corresponding video frame via [EGLExt.eglPresentationTimeANDROID]. + * Runs libass on its own [HandlerThread] into a packed ALPHA_8 atlas of one or more + * pages plus a vertex stream, and a GL thread that uploads both and issues one + * `glDrawArrays` per atlas page. Each timed swap is pinned to the corresponding + * video frame via [EGLExt.eglPresentationTimeANDROID]. */ @UnstableApi internal object AssAtlasPipelineConfig { @@ -54,6 +54,16 @@ internal object AssAtlasPipelineConfig { /** Preallocated vertex-stream capacity (192 bytes × 16384 = 3 MB per buffer). */ internal const val MAX_QUADS = 16384 + /** + * Hard cap on vertically-stacked atlas pages per slot. A frame whose packed + * sub-pixels exceed one [ATLAS_PIXEL_BUDGET] texture (a 4K-rendered full-screen + * typeset sign) spills into extra pages so nothing is dropped (#1436); the atlas + * buffer grows on demand toward this cap. 4 covers the worst frame measured (a 4K + * letter needs 3); past it tiles are dropped and counted in `truncated`. Must match + * MAX_ATLAS_PAGES in AssKt.c. + */ + internal const val MAX_ATLAS_PAGES = 4 + /** Must match the byte layout produced by `nativeAssRenderFrameAtlas` in AssKt.c. */ internal const val BYTES_PER_VERTEX = 32 internal const val BYTES_PER_QUAD = BYTES_PER_VERTEX * 6 @@ -98,8 +108,12 @@ internal object AssAtlasPipelineConfig { internal class AtlasPayload( val slotIndex: Int, - val atlasBuf: ByteBuffer, + /** Vertically-stacked atlas pages (page p at byte offset p·atlasW·atlasH). Starts + * one page; [growAtlas] reallocates it larger when a dense frame needs more. */ + var atlasBuf: ByteBuffer, val vertexBuf: ByteBuffer, + /** How many pages [atlasBuf] currently holds — the high-water mark for this slot. */ + var pageCapacity: Int, var frame: AssAtlasFrame, var presentationTimeUs: Long, var releaseTimeNs: Long, @@ -109,7 +123,14 @@ internal class AtlasPayload( var contentSeq: Long = 0L, var requestSeq: Long = 0L, var stateGeneration: Long = 0L -) +) { + /** Reallocates [atlasBuf] to hold [pages] stacked atlasW×atlasH pages. Runs on the + * libass thread before hand-off, so no GL reader can be looking at the old buffer. */ + fun growAtlas(pages: Int, atlasW: Int, atlasH: Int) { + atlasBuf = ByteBuffer.allocateDirect(atlasW * atlasH * pages).order(ByteOrder.nativeOrder()) + pageCapacity = pages + } +} private class AtlasDrawSnapshot( val atlasBuf: ByteBuffer, @@ -181,7 +202,7 @@ internal class AssAtlasPipeline( // GL_MAX_TEXTURE_SIZE right after EGL init; by the libass thread's 1 s fallback // if GL never comes up. Both threads then agree on the dims, which matters // because the C side bakes UV denominators = these dims into the vertex stream - // and the GL side allocates the texture once at these dims. + // and the GL side allocates each atlas-page texture at these dims. private val dimsResolved = java.util.concurrent.atomic.AtomicBoolean(false) private val dimsLatch = java.util.concurrent.CountDownLatch(1) @@ -227,10 +248,12 @@ internal class AssAtlasPipeline( val payloads = Array(slotCount) { index -> AtlasPayload( slotIndex = index, + // One page up front (the common case); grows on demand for dense frames. atlasBuf = ByteBuffer.allocateDirect(w * h).order(ByteOrder.nativeOrder()), vertexBuf = ByteBuffer.allocateDirect( AssAtlasPipelineConfig.MAX_QUADS * AssAtlasPipelineConfig.BYTES_PER_QUAD ).order(ByteOrder.nativeOrder()), + pageCapacity = 1, frame = AssAtlasFrame(0, 0, 0, 0, 0), presentationTimeUs = 0L, releaseTimeNs = C.TIME_UNSET @@ -692,8 +715,21 @@ private class AtlasLibassThread( val render = assHandler.render ?: return null val payload = slots.payloads[slot] val t0 = System.nanoTime() - val frame = render.renderFrameAtlas(timeMs, payload.atlasBuf, slots.atlasW, slots.atlasH, payload.vertexBuf) + var frame = render.renderFrameAtlas(timeMs, payload.atlasBuf, slots.atlasW, slots.atlasH, payload.vertexBuf) ?: return null + // A frame overflows one atlas page only on dense full-screen typesetting. When it + // does, grow this slot's buffer to the pages it needs (capped) and render once more + // — libass's caches make the re-render cheap, and the slot keeps the larger buffer + // so the same density never re-grows. The truncated first result is never handed off. + if (frame.requiredPages > payload.pageCapacity && payload.pageCapacity < AssAtlasPipelineConfig.MAX_ATLAS_PAGES) { + payload.growAtlas( + minOf(frame.requiredPages, AssAtlasPipelineConfig.MAX_ATLAS_PAGES), + slots.atlasW, + slots.atlasH + ) + frame = render.renderFrameAtlas(timeMs, payload.atlasBuf, slots.atlasW, slots.atlasH, payload.vertexBuf) + ?: return null + } val libassMs = (System.nanoTime() - t0) / 1_000_000 renderCount++ lastLibassMs = libassMs @@ -1083,8 +1119,8 @@ private class AtlasGlThread( EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext) renderer.onSurfaceCreated() // Resolve atlas dims from real GL caps (first-wins against the libass - // thread's fallback) and allocate the texture once at those dims — uploads - // are glTexSubImage2D of the packed rows from then on. + // thread's fallback) and allocate the page-0 texture at those dims (extra + // pages lazily) — uploads are glTexSubImage2D of the packed rows from then on. val maxTexture = IntArray(1) GLES20.glGetIntegerv(GLES20.GL_MAX_TEXTURE_SIZE, maxTexture, 0) val (atlasW, atlasH) = resolveAtlasDims(maxTexture[0]) @@ -1342,9 +1378,11 @@ private class AtlasGlThread( } /** - * GL-side work for the atlas-based path. Maintains a single atlas texture and a - * single vertex buffer; uploads them per frame (unless the payload identity - * matches the last upload) and issues one `glDrawArrays` for the whole frame. + * GL-side work for the atlas-based path. Maintains one ALPHA_8 atlas texture per + * page (allocated lazily, up to MAX_ATLAS_PAGES) plus a single vertex buffer; + * uploads them per frame (unless the payload identity matches the last upload) and + * issues one `glDrawArrays` per page, drawing pages in turn to reproduce libass's + * blend order. */ @UnstableApi private class AtlasRenderer(private val assHandler: AssHandler) { @@ -1380,7 +1418,9 @@ private class AtlasRenderer(private val assHandler: AssHandler) { private var surfaceSize = Size.ZERO private lateinit var glProgram: GlProgram - private var atlasTexId = 0 + // One texture per atlas page; allocated lazily as the page count grows. + private val atlasTexIds = IntArray(AssAtlasPipelineConfig.MAX_ATLAS_PAGES) + private var allocatedPages = 0 private var vertexBufferId = 0 private var aPosition = 0 @@ -1393,20 +1433,38 @@ private class AtlasRenderer(private val assHandler: AssHandler) { private var atlasAllocatedH = 0 /** - * Allocates the atlas texture once at the resolved dims. The C side bakes UV - * denominators = these dims into the vertex stream, so per-frame uploads can be - * partial ([uploadAtlas]) without ever reallocating — drivers keep one stable - * texture allocation instead of churning on packed-height changes. + * Records the per-page texture dims and allocates the first page's texture. The C + * side bakes UV denominators = these dims into the vertex stream and stacks pages + * at byte multiples of width×height, so per-frame uploads stay partial + * ([uploadPage]) — drivers keep stable texture allocations instead of churning on + * packed-height changes — and extra pages allocate lazily ([ensurePageTexture]). */ fun allocateAtlasTexture(width: Int, height: Int) { - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, atlasTexId) - GLES20.glTexImage2D( - GLES20.GL_TEXTURE_2D, 0, GLES20.GL_ALPHA, - width, height, 0, - GLES20.GL_ALPHA, GLES20.GL_UNSIGNED_BYTE, null - ) atlasAllocatedW = width atlasAllocatedH = height + allocatedPages = 0 + ensurePageTexture(0) + } + + /** Lazily allocates atlas-page textures through [page] at the recorded dims. */ + private fun ensurePageTexture(page: Int) { + while (allocatedPages <= page && allocatedPages < atlasTexIds.size) { + val p = allocatedPages + val tex = IntArray(1) + GLES20.glGenTextures(1, tex, 0) + atlasTexIds[p] = tex[0] + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, atlasTexIds[p]) + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE) + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE) + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR) + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR) + GLES20.glTexImage2D( + GLES20.GL_TEXTURE_2D, 0, GLES20.GL_ALPHA, + atlasAllocatedW, atlasAllocatedH, 0, + GLES20.GL_ALPHA, GLES20.GL_UNSIGNED_BYTE, null + ) + allocatedPages = p + 1 + } } fun onSurfaceCreated() { @@ -1420,16 +1478,9 @@ private class AtlasRenderer(private val assHandler: AssHandler) { uTexture = glProgram.getUniformLocation("u_Texture") uSurfaceSize = glProgram.getUniformLocation("u_SurfaceSize") - val tex = IntArray(1) - GLES20.glGenTextures(1, tex, 0) - atlasTexId = tex[0] GLES20.glActiveTexture(GLES20.GL_TEXTURE0) - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, atlasTexId) - GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE) - GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE) - GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR) - GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR) GLES20.glUniform1i(uTexture, 0) + // Atlas-page textures are generated lazily in allocateAtlasTexture/ensurePageTexture. val buf = IntArray(1) GLES20.glGenBuffers(1, buf, 0) @@ -1468,7 +1519,6 @@ private class AtlasRenderer(private val assHandler: AssHandler) { if (quadCount == 0) return if (!reuseUploads) { - uploadAtlas(payload.atlasBuf, frame.atlasWidth, frame.atlasHeight) uploadVertices(payload.vertexBuf, quadCount) } @@ -1477,30 +1527,42 @@ private class AtlasRenderer(private val assHandler: AssHandler) { GLES20.glVertexAttribPointer(aPosition, 2, GLES20.GL_FLOAT, false, stride, 0) GLES20.glVertexAttribPointer(aTexCoord, 2, GLES20.GL_FLOAT, false, stride, 8) GLES20.glVertexAttribPointer(aColor, 4, GLES20.GL_FLOAT, false, stride, 16) - GLES20.glDrawArrays(GLES20.GL_TRIANGLES, 0, quadCount * 6) + + // Each atlas page is its own texture; its quads are one contiguous run in the + // stream (page assignment is monotonic in painter order). Upload + draw each in + // turn, which reproduces the libass blend order across pages. + GLES20.glActiveTexture(GLES20.GL_TEXTURE0) + var quadOffset = 0 + for (p in 0 until frame.pageCount) { + val pageQuads = frame.pageQuadCounts[p] + if (pageQuads > 0) { + ensurePageTexture(p) + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, atlasTexIds[p]) + if (!reuseUploads) uploadPage(payload.atlasBuf, p, frame.atlasWidth, frame.pageHeights[p]) + GLES20.glDrawArrays(GLES20.GL_TRIANGLES, quadOffset * 6, pageQuads * 6) + } + quadOffset += pageQuads + } } - private fun uploadAtlas(atlasBuf: ByteBuffer, atlasW: Int, atlasH: Int) { - atlasBuf.position(0).limit(atlasW * atlasH) - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, atlasTexId) - if (atlasW == atlasAllocatedW && atlasH <= atlasAllocatedH) { - // Steady state: packed rows into the once-allocated texture. - GLES20.glTexSubImage2D( - GLES20.GL_TEXTURE_2D, 0, 0, 0, atlasW, atlasH, - GLES20.GL_ALPHA, GLES20.GL_UNSIGNED_BYTE, atlasBuf - ) - } else { - // Defensive: dims disagree with the allocation (shouldn't happen — both - // sides resolve dims through the same first-wins gate). - Log.w("AssAtlasRenderer", "atlas upload ${atlasW}x$atlasH outside allocation ${atlasAllocatedW}x$atlasAllocatedH") - GLES20.glTexImage2D( - GLES20.GL_TEXTURE_2D, 0, GLES20.GL_ALPHA, - atlasW, atlasH, 0, - GLES20.GL_ALPHA, GLES20.GL_UNSIGNED_BYTE, atlasBuf - ) - atlasAllocatedW = atlasW - atlasAllocatedH = atlasH + /** Uploads page [page]'s packed rows from the stacked atlas buffer into the + * currently-bound page texture. */ + private fun uploadPage(atlasBuf: ByteBuffer, page: Int, atlasW: Int, pageH: Int) { + if (pageH <= 0) return + if (atlasW != atlasAllocatedW || pageH > atlasAllocatedH) { + // Defensive: dims disagree with the allocation (shouldn't happen — both sides + // resolve dims through the same first-wins gate). + Log.w("AssAtlasRenderer", "page upload ${atlasW}x$pageH outside allocation ${atlasAllocatedW}x$atlasAllocatedH") + return } + val start = page * atlasW * atlasAllocatedH + atlasBuf.clear() + atlasBuf.limit(start + atlasW * pageH) + atlasBuf.position(start) + GLES20.glTexSubImage2D( + GLES20.GL_TEXTURE_2D, 0, 0, 0, atlasW, pageH, + GLES20.GL_ALPHA, GLES20.GL_UNSIGNED_BYTE, atlasBuf + ) } private fun uploadVertices(vertexBuf: ByteBuffer, quadCount: Int) { @@ -1511,10 +1573,10 @@ private class AtlasRenderer(private val assHandler: AssHandler) { } fun onSurfaceDestroyed() { - if (atlasTexId != 0) { - val tex = intArrayOf(atlasTexId) - GLES20.glDeleteTextures(1, tex, 0) - atlasTexId = 0 + if (allocatedPages > 0) { + GLES20.glDeleteTextures(allocatedPages, atlasTexIds, 0) + atlasTexIds.fill(0) + allocatedPages = 0 } if (vertexBufferId != 0) { val buf = intArrayOf(vertexBufferId)