fix(player): create animated AVIF scenes from MediaCodec output

app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/AndroidSceneCaptureService.kt

@@ -47,25 +47,54 @@ internal class AndroidSceneCaptureService private constructor(
             val lease = inputAcquirer.acquire(input)
                 ?: return@withContext AnkiScreenshotPreparation.Failed(stillFallback = null)
             sceneDirectory.mkdirs()
-            val output = File(sceneDirectory, "${UUID.randomUUID()}.avif")
+            val outputBaseName = UUID.randomUUID().toString()
+            val intermediate = File(sceneDirectory, "$outputBaseName.obu")
+            val output = File(sceneDirectory, "$outputBaseName.avif")
             val inputCleanup = SceneNativeCleanup(lease::close)
-            val outputCleanup = SceneNativeCleanup(output::delete)
+            val intermediateCleanup = SceneNativeCleanup(intermediate::delete)
+            var outputCleanup: SceneNativeCleanup? = null
             var transferred = false
             try {
-                val result = commandExecutor.executeFfmpeg(
-                    SceneFfmpegArguments.animatedAvif(
+                val encodeResult = commandExecutor.executeFfmpeg(
+                    SceneFfmpegArguments.av1MediaCodecPackets(
                         input = input,
                         acquiredInputValue = lease.ffmpegValue,
                         range = range,
-                        outputFile = output.absolutePath,
+                        outputFile = intermediate.absolutePath,
                         encoderName = encoderName,
                         tlsCaFile = lease.tlsCaFile,
                     ),
                 ) {
                     inputCleanup.nativeFinished()
-                    outputCleanup.nativeFinished()
+                    intermediateCleanup.nativeFinished()
+                }
+                inputCleanup.release()
+                when (encodeResult) {
+                    SceneCommandResult.Failed -> {
+                        return@withContext AnkiScreenshotPreparation.Failed(stillFallback = null)
+                    }
+                    is SceneCommandResult.Success -> Unit
                 }
-                when (result) {
+                val normalized = intermediate
+                    .takeIf { it.isFile && it.length() in 1..MAX_INTERMEDIATE_BYTES }
+                    ?.readBytes()
+                    ?.let(MediaCodecAv1StreamNormalizer::normalize)
+                    ?: return@withContext AnkiScreenshotPreparation.Failed(stillFallback = null)
+                intermediate.writeBytes(normalized)
+
+                val currentOutputCleanup = SceneNativeCleanup(output::delete)
+                outputCleanup = currentOutputCleanup
+                val finishIntermediateRemuxUse = intermediateCleanup.retainNativeUse()
+                val remuxResult = commandExecutor.executeFfmpeg(
+                    SceneFfmpegArguments.animatedAvifFromObu(
+                        inputFile = intermediate.absolutePath,
+                        outputFile = output.absolutePath,
+                    ),
+                ) {
+                    finishIntermediateRemuxUse()
+                    currentOutputCleanup.nativeFinished()
+                }
+                when (remuxResult) {
                     SceneCommandResult.Failed -> {
                         return@withContext AnkiScreenshotPreparation.Failed(stillFallback = null)
                     }
@@ -91,7 +120,10 @@ internal class AndroidSceneCaptureService private constructor(
                 AnkiScreenshotPreparation.Failed(stillFallback = null)
             } finally {
                 inputCleanup.release()
-                if (!transferred) outputCleanup.release()
+                intermediateCleanup.release()
+                if (!transferred) {
+                    outputCleanup?.release() ?: output.delete()
+                }
             }
         }
     }
@@ -113,6 +145,7 @@ internal class AndroidSceneCaptureService private constructor(
     internal companion object {
         private const val SCENE_CACHE_DIRECTORY = "chimahon_scene_capture"
         private const val MAX_OUTPUT_DIMENSION = 640
+        private const val MAX_INTERMEDIATE_BYTES = 12L * 1024L * 1024L
 
         fun forTests(
             sceneDirectory: File,

app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/FfmpegKitSceneCommandExecutor.kt

@@ -37,24 +37,52 @@ internal interface SceneCommandExecutor {
 internal class SceneNativeCleanup(
     private val cleanup: () -> Unit,
 ) {
-    private val nativeFinished = AtomicBoolean(false)
-    private val released = AtomicBoolean(false)
-    private val cleaned = AtomicBoolean(false)
+    private val lock = Any()
+    private val initialNativeFinished = AtomicBoolean(false)
+    private var activeNativeUses = 1
+    private var released = false
+    private var cleaned = false
 
     fun nativeFinished() {
-        nativeFinished.set(true)
-        cleanIfReady()
+        finishNativeUse(initialNativeFinished)
+    }
+
+    fun retainNativeUse(): () -> Unit {
+        synchronized(lock) {
+            check(!released) { "Cannot retain a released native resource" }
+            activeNativeUses++
+        }
+        val finished = AtomicBoolean(false)
+        return {
+            finishNativeUse(finished)
+        }
     }
 
     fun release() {
-        released.set(true)
-        cleanIfReady()
+        val shouldClean = synchronized(lock) {
+            released = true
+            markCleanIfReady()
+        }
+        if (shouldClean) runCatching(cleanup)
+    }
+
+    private fun finishNativeUse(finished: AtomicBoolean) {
+        if (finished.compareAndSet(false, true)) {
+            val shouldClean = synchronized(lock) {
+                check(activeNativeUses > 0)
+                activeNativeUses--
+                markCleanIfReady()
+            }
+            if (shouldClean) runCatching(cleanup)
+        }
     }
 
-    private fun cleanIfReady() {
-        if (nativeFinished.get() && released.get() && cleaned.compareAndSet(false, true)) {
-            runCatching(cleanup)
+    private fun markCleanIfReady(): Boolean {
+        if (activeNativeUses == 0 && released && !cleaned) {
+            cleaned = true
+            return true
         }
+        return false
     }
 }
 

app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/MediaCodecAv1StreamNormalizer.kt

@@ -0,0 +1,103 @@
+package eu.kanade.tachiyomi.ui.player.scene
+
+import java.io.ByteArrayOutputStream
+
+/**
+ * Removes Android's AV1CodecConfigurationRecord and restores temporal-unit boundaries that raw
+ * packet output loses. FFmpeg's MediaCodec wrapper incorrectly prepends the record to frame data.
+ */
+internal object MediaCodecAv1StreamNormalizer {
+    fun normalize(input: ByteArray): ByteArray? {
+        if (input.size < AV1C_HEADER_SIZE + 1) return null
+        val start = if (isAv1CodecConfigurationRecord(input)) AV1C_HEADER_SIZE else 0
+        val obus = parseObus(input, start) ?: return null
+        if (obus.none { it.type == OBU_SEQUENCE_HEADER } ||
+            obus.count { it.type == OBU_FRAME || it.type == OBU_FRAME_HEADER } < 2
+        ) {
+            return null
+        }
+
+        val output = ByteArrayOutputStream(input.size + obus.size * TEMPORAL_DELIMITER.size)
+        var frameStarted = false
+        if (obus.first().type != OBU_TEMPORAL_DELIMITER) {
+            output.write(TEMPORAL_DELIMITER)
+        }
+        obus.forEach { obu ->
+            when (obu.type) {
+                OBU_TEMPORAL_DELIMITER -> {
+                    if (!output.endsWithTemporalDelimiter()) {
+                        output.write(TEMPORAL_DELIMITER)
+                    }
+                    frameStarted = false
+                }
+                OBU_FRAME,
+                OBU_FRAME_HEADER,
+                -> {
+                    if (frameStarted) output.write(TEMPORAL_DELIMITER)
+                    output.write(input, obu.offset, obu.length)
+                    frameStarted = true
+                }
+                else -> output.write(input, obu.offset, obu.length)
+            }
+        }
+        return output.toByteArray()
+    }
+
+    private fun isAv1CodecConfigurationRecord(input: ByteArray): Boolean {
+        val first = input[0].toInt() and 0xff
+        return first and 0x80 != 0 && first and 0x7f == 1
+    }
+
+    private fun parseObus(input: ByteArray, start: Int): List<Obu>? {
+        val result = mutableListOf<Obu>()
+        var offset = start
+        while (offset < input.size) {
+            val header = input[offset].toInt() and 0xff
+            if (header and 0x80 != 0 || header and 0x01 != 0 || header and 0x02 == 0) return null
+            val extensionBytes = if (header and 0x04 != 0) 1 else 0
+            val sizeOffset = offset + 1 + extensionBytes
+            if (sizeOffset >= input.size) return null
+            val size = readLeb128(input, sizeOffset) ?: return null
+            val payloadOffset = sizeOffset + size.bytes
+            val end = payloadOffset.toLong() + size.value
+            if (end > input.size || end > Int.MAX_VALUE) return null
+            result += Obu(
+                type = header shr 3 and 0x0f,
+                offset = offset,
+                length = end.toInt() - offset,
+            )
+            offset = end.toInt()
+        }
+        return result.takeIf { it.isNotEmpty() }
+    }
+
+    private fun readLeb128(input: ByteArray, offset: Int): Leb128? {
+        var value = 0L
+        for (index in 0 until MAX_LEB128_BYTES) {
+            val position = offset + index
+            if (position >= input.size) return null
+            val byte = input[position].toInt() and 0xff
+            value = value or ((byte and 0x7f).toLong() shl (index * 7))
+            if (byte and 0x80 == 0) return Leb128(value, index + 1)
+        }
+        return null
+    }
+
+    private fun ByteArrayOutputStream.endsWithTemporalDelimiter(): Boolean {
+        val bytes = toByteArray()
+        return bytes.size >= TEMPORAL_DELIMITER.size &&
+            bytes[bytes.lastIndex - 1] == TEMPORAL_DELIMITER[0] &&
+            bytes[bytes.lastIndex] == TEMPORAL_DELIMITER[1]
+    }
+
+    private data class Obu(val type: Int, val offset: Int, val length: Int)
+    private data class Leb128(val value: Long, val bytes: Int)
+
+    private val TEMPORAL_DELIMITER = byteArrayOf(0x12, 0x00)
+    private const val AV1C_HEADER_SIZE = 4
+    private const val MAX_LEB128_BYTES = 8
+    private const val OBU_SEQUENCE_HEADER = 1
+    private const val OBU_TEMPORAL_DELIMITER = 2
+    private const val OBU_FRAME_HEADER = 3
+    private const val OBU_FRAME = 6
+}

app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneMediaProbe.kt

@@ -24,19 +24,9 @@ internal object SceneMediaProbe {
         if (pixelFormat in setOf("none", "unknown")) {
             return false
         }
-        val rawBits = values.firstOrNull { it.first == "bits_per_raw_sample" }
-            ?.second
-            ?.toIntOrNull()
         val transfer = values.firstOrNull { it.first == "color_transfer" }?.second.orEmpty()
         val primaries = values.firstOrNull { it.first == "color_primaries" }?.second.orEmpty()
-        val profile = values.firstOrNull { it.first == "profile" }?.second.orEmpty()
-        if (
-            rawBits?.let { it > 8 } == true ||
-            TEN_BIT_PIXEL_FORMAT.containsMatchIn(pixelFormat) ||
-            transfer in HDR_TRANSFERS ||
-            primaries == "bt2020" ||
-            profile.contains("main 10")
-        ) {
+        if (transfer in HDR_TRANSFERS || primaries == "bt2020") {
             return false
         }
         return true
@@ -48,6 +38,5 @@ internal object SceneMediaProbe {
     }
 
     private val HDR_TRANSFERS = setOf("smpte2084", "arib-std-b67")
-    private val TEN_BIT_PIXEL_FORMAT = Regex("(p0(?:10|12|16)|p(?:9|10|12|14|16)(?:le|be)?)(?:$|[^0-9])")
     private val PROTECTION_MARKERS = setOf("cenc", "cbcs", "crypto", "encrypted", "encryption", "drm")
 }

app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt

@@ -163,7 +163,7 @@ internal object SceneVideoInputResolver {
 }
 
 internal object SceneFfmpegArguments {
-    fun animatedAvif(
+    fun av1MediaCodecPackets(
         input: SceneVideoInputSpec,
         acquiredInputValue: String,
         range: SceneTimeRange,
@@ -201,15 +201,37 @@ internal object SceneFfmpegArguments {
             add("1")
             add("-pix_fmt")
             add("yuv420p")
-            add("-loop")
-            add("0")
             add("-f")
-            add("avif")
+            add("data")
             add("-y")
             add(outputFile)
         }.toTypedArray()
     }
 
+    fun animatedAvifFromObu(
+        inputFile: String,
+        outputFile: String,
+    ): Array<String> {
+        return arrayOf(
+            "-f",
+            "obu",
+            "-framerate",
+            FRAME_RATE.toInt().toString(),
+            "-i",
+            inputFile,
+            "-map",
+            "0:v:0",
+            "-c:v",
+            "copy",
+            "-loop",
+            "0",
+            "-f",
+            "avif",
+            "-y",
+            outputFile,
+        )
+    }
+
     fun videoProbe(
         input: SceneVideoInputSpec,
         acquiredInputValue: String,