diff --git a/android/app/src/main/kotlin/com/org/yumiparty/GiftMp4VideoPlatformView.kt b/android/app/src/main/kotlin/com/org/yumiparty/GiftMp4VideoPlatformView.kt index 6878569..cd44487 100644 --- a/android/app/src/main/kotlin/com/org/yumiparty/GiftMp4VideoPlatformView.kt +++ b/android/app/src/main/kotlin/com/org/yumiparty/GiftMp4VideoPlatformView.kt @@ -2,25 +2,37 @@ package com.org.yumiparty import android.content.Context import android.graphics.Bitmap -import android.graphics.Color +import android.graphics.SurfaceTexture import android.media.MediaMetadataRetriever -import android.net.Uri +import android.media.MediaPlayer +import android.opengl.EGL14 +import android.opengl.EGLConfig +import android.opengl.EGLContext +import android.opengl.EGLDisplay +import android.opengl.EGLSurface +import android.opengl.GLES11Ext +import android.opengl.GLES20 +import android.opengl.Matrix import android.os.Handler +import android.os.HandlerThread import android.os.Looper -import android.os.SystemClock import android.util.Log import android.view.Surface +import android.view.TextureView import android.view.View -import androidx.media3.common.MediaItem -import androidx.media3.common.PlaybackException -import androidx.media3.common.Player -import androidx.media3.exoplayer.ExoPlayer import io.flutter.FlutterInjector import io.flutter.plugin.common.MethodCall import io.flutter.plugin.common.MethodChannel import io.flutter.plugin.common.MethodChannel.Result import io.flutter.plugin.platform.PlatformView import java.io.File +import java.io.FileOutputStream +import java.nio.ByteBuffer +import java.nio.ByteOrder +import kotlin.math.abs +import kotlin.math.max +import kotlin.math.min +import kotlin.math.sqrt import org.json.JSONArray import org.json.JSONObject @@ -32,114 +44,48 @@ class GiftMp4VideoPlatformView( ) : PlatformView { private val mainHandler = Handler(Looper.getMainLooper()) private val events = MethodChannel(messenger, CHANNEL_NAME) - private val glView = GiftMp4GlVideoView(context) - private val player = ExoPlayer.Builder(context).build() - private val token = (params["token"] as? Number)?.toInt() ?: 0 - private var released = false - private var prepared = false - private var started = false - private var playerSurface: Surface? = null - - init { - val path = params["path"] as? String ?: "" - val sourceType = params["sourceType"] as? String ?: "file" - val muted = params["muted"] as? Boolean ?: false - val source = GiftMp4Source(context, path, sourceType) - Log.d( - "GiftMp4Video", - "create viewId=$viewId token=$token sourceType=$sourceType path=$path", - ) - val sourceLayout = resolveSourceLayout(source) - - registry[viewId] = this - glView.setSourceLayout( - sourceLayout.hasAlphaMask, - sourceLayout.useTextureMatrix, - sourceLayout.flipY, - sourceLayout.contentAspectRatio, - sourceLayout.colorRect.toFloatArray(), - sourceLayout.alphaRect.toFloatArray(), - ) - player.volume = if (muted) 0f else 1f - player.repeatMode = Player.REPEAT_MODE_OFF - player.addListener( - object : Player.Listener { - override fun onPlaybackStateChanged(playbackState: Int) { - when (playbackState) { - Player.STATE_READY -> { - if (!started) { - started = true - Log.d( - "GiftMp4Video", - "started viewId=$viewId token=$token", - ) - postEvent("started") - } - } - Player.STATE_ENDED -> { - Log.d( - "GiftMp4Video", - "completed viewId=$viewId token=$token", - ) - postEvent("completed") - } - } - } - - override fun onPlayerError(error: PlaybackException) { - Log.e( - "GiftMp4Video", - "failed viewId=$viewId token=$token code=${error.errorCode}", - error, - ) - postEvent( - "failed", - mapOf( - "code" to error.errorCode, - "message" to error.message, - ), - ) - } + private val textureView = GiftMp4AlphaTextureView(context).apply { + isOpaque = false + } + private val renderer = + GiftMp4AlphaRenderer( + context.applicationContext, + params, + onStarted = { postEvent("started") }, + onCompleted = { postEvent("completed") }, + onFailed = { code, message -> + postEvent( + "failed", + mapOf("code" to code, "message" to message), + ) }, ) + private var disposed = false - glView.onSurfaceReady = onSurfaceReady@{ surface -> - if (released) { - surface.release() - return@onSurfaceReady - } - playerSurface?.release() - playerSurface = surface - player.setVideoSurface(surface) - if (!prepared) { - prepared = true - player.setMediaItem(source.toMediaItem()) - player.prepare() - player.seekTo(0) - player.play() - } - } + init { + Log.d( + "GiftMp4Video", + "create viewId=$viewId token=${renderer.token} sourceType=${renderer.sourceType} path=${renderer.source}", + ) + registry[viewId] = this + textureView.renderer = renderer } - override fun getView(): View = glView + override fun getView(): View = textureView fun setMute(muted: Boolean) { - player.volume = if (muted) 0f else 1f + renderer.setMuted(muted) } fun stop() { - player.stop() + renderer.stop() } override fun dispose() { - released = true + disposed = true registry.remove(viewId) - glView.onSurfaceReady = null - player.clearVideoSurface() - player.release() - playerSurface?.release() - playerSurface = null - glView.release() + textureView.renderer = null + renderer.release() } private fun postEvent( @@ -147,133 +93,22 @@ class GiftMp4VideoPlatformView( extra: Map = emptyMap(), ) { mainHandler.post { + if (disposed) { + return@post + } events.invokeMethod( method, - mapOf("viewId" to viewId, "token" to token) + extra, + mapOf("viewId" to viewId, "token" to renderer.token) + extra, ) } } - private class GiftMp4Source( - private val context: Context, - val path: String, - sourceType: String, - ) { - val type: SourceType = - if (sourceType == "asset") SourceType.ASSET else SourceType.FILE - val assetKey: String? = - if (type == SourceType.ASSET) { - FlutterInjector - .instance() - .flutterLoader() - .getLookupKeyForAsset(path) - } else { - null - } - val cacheKey: String = "${type.name}:${assetKey ?: path}" - - fun toMediaItem(): MediaItem { - return if (type == SourceType.ASSET) { - MediaItem.fromUri(Uri.parse("asset:///$assetKey")) - } else { - MediaItem.fromUri(Uri.fromFile(File(path))) - } - } - - fun readBytes(): ByteArray { - return if (type == SourceType.ASSET) { - context.assets.open(assetKey ?: path).use { it.readBytes() } - } else { - File(path).readBytes() - } - } - - fun newMetadataRetriever(): MediaMetadataRetriever { - val retriever = MediaMetadataRetriever() - if (type == SourceType.ASSET) { - context.assets.openFd(assetKey ?: path).use { descriptor -> - retriever.setDataSource( - descriptor.fileDescriptor, - descriptor.startOffset, - descriptor.length, - ) - } - } else { - retriever.setDataSource(path) - } - return retriever - } - } - - private enum class SourceType { ASSET, FILE } - - private data class VideoSize( - val width: Int, - val height: Int, - ) { - val aspectRatio: Float - get() = if (height <= 0) 1f else width.toFloat() / height.toFloat() - } - - private data class SourceRect( - val x: Float, - val y: Float, - val width: Float, - val height: Float, - ) { - fun toFloatArray(): FloatArray = floatArrayOf(x, y, width, height) - } - - private data class SourceLayout( - val kind: String, - val hasAlphaMask: Boolean, - val useTextureMatrix: Boolean, - val flipY: Boolean, - val contentAspectRatio: Float, - val colorRect: SourceRect, - val alphaRect: SourceRect, - ) - - private data class HalfStats( - val saturation: Double, - val grayError: Double, - val brightness: Double, - ) - - private enum class SplitOrientation { HORIZONTAL, VERTICAL } - - private data class SplitAlphaCandidate( - val orientation: SplitOrientation, - val alphaFirst: Boolean, - ) - - private data class SplitScore( - var direction: Double = 0.0, - var saturationGap: Double = 0.0, - var maskSaturation: Double = 0.0, - var maskGrayError: Double = 0.0, - var frames: Int = 0, - ) - - private data class PackedAlphaCandidate( - val frameWidth: Int, - val frameHeight: Int, - val alphaStartX: Int, - val alphaEndX: Int, - ) - companion object { const val VIEW_TYPE = "gift_mp4_video_view" const val CHANNEL_NAME = "com.org.yumiparty/gift_mp4_events" const val CONTROL_CHANNEL_NAME = "com.org.yumiparty/gift_mp4_control" - private const val MIN_SPLIT_MATCH_FRAMES = 3 - private const val MIN_USEFUL_FRAME_BRIGHTNESS = 0.035 - private const val MIN_SPLIT_SATURATION_GAP = 0.08 - private const val MAX_SPLIT_MASK_SATURATION = 0.28 - private const val MAX_SPLIT_MASK_GRAY_ERROR = 0.06 private val registry = mutableMapOf() - private val layoutCache = mutableMapOf() fun handleControlMethod(call: MethodCall, result: Result) { val viewId = call.argument("viewId") @@ -294,385 +129,594 @@ class GiftMp4VideoPlatformView( else -> result.notImplemented() } } + } +} - private fun resolveSourceLayout(source: GiftMp4Source): SourceLayout { - layoutCache[source.cacheKey]?.let { return it } - val startedAt = SystemClock.elapsedRealtime() - val videoSize = probeVideoSize(source) - val layout = - parseVapcSourceLayout(source) - // Top-right packed alpha can look like a gray half-frame, - // so classify it before the looser split-alpha heuristic. - ?: detectPackedAlphaTopRightSourceLayout(source) - ?: detectSplitSourceLayout(source, videoSize) - ?: normalSourceLayout(videoSize) - val elapsedMs = SystemClock.elapsedRealtime() - startedAt - Log.d( - "GiftMp4Video", - "classified path=${source.path} mode=${layout.kind} elapsedMs=$elapsedMs", - ) - layoutCache[source.cacheKey] = layout - trimLayoutCache() - return layout - } - - private fun trimLayoutCache() { - while (layoutCache.size > 64) { - layoutCache.remove(layoutCache.keys.first()) +private class GiftMp4AlphaTextureView(context: Context) : + TextureView(context), + TextureView.SurfaceTextureListener { + var renderer: GiftMp4AlphaRenderer? = null + set(value) { + field = value + if (isAvailable && value != null) { + value.attach(surfaceTexture ?: return, width, height) } } - private fun probeVideoSize(source: GiftMp4Source): VideoSize { - val retriever = source.newMetadataRetriever() - return try { - val width = - retriever - .extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH) - ?.toIntOrNull() - ?: 1 - val height = - retriever - .extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT) - ?.toIntOrNull() - ?: 1 - val rotation = - retriever - .extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION) - ?.toIntOrNull() - ?: 0 - if (rotation == 90 || rotation == 270) { - VideoSize(height, width) - } else { - VideoSize(width, height) - } - } finally { - retriever.release() - } + init { + surfaceTextureListener = this + } + + override fun onSurfaceTextureAvailable(surface: SurfaceTexture, width: Int, height: Int) { + renderer?.attach(surface, width, height) + } + + override fun onSurfaceTextureSizeChanged(surface: SurfaceTexture, width: Int, height: Int) { + renderer?.resize(width, height) + } + + override fun onSurfaceTextureDestroyed(surface: SurfaceTexture): Boolean { + renderer?.detach() + return true + } + + override fun onSurfaceTextureUpdated(surface: SurfaceTexture) = Unit +} + +private class GiftMp4AlphaRenderer( + private val context: Context, + params: Map<*, *>, + private val onStarted: () -> Unit, + private val onCompleted: () -> Unit, + private val onFailed: (Int, String?) -> Unit, +) { + val token: Int = (params["token"] as? Number)?.toInt() ?: 0 + val sourceType: String = params["sourceType"]?.toString() ?: "file" + val source: String = params["path"]?.toString().orEmpty() + private val providedLayout = GiftMp4NativeLayout.fromCreationParams(params["vapLayout"]) + + private val renderThread = HandlerThread("GiftMp4AlphaRenderer").apply { start() } + private val handler = Handler(renderThread.looper) + private val textureMatrix = FloatArray(16) + private val identityMatrix = FloatArray(16).apply { Matrix.setIdentityM(this, 0) } + private val mutedVolume = floatArrayOf(if ((params["muted"] as? Boolean) == true) 0f else 1f) + + private var outputSurfaceTexture: SurfaceTexture? = null + private var outputSurface: Surface? = null + private var videoSurfaceTexture: SurfaceTexture? = null + private var videoSurface: Surface? = null + private var mediaPlayer: MediaPlayer? = null + private var sourceData: GiftMp4ResolvedSource? = null + private var layout = GiftMp4NativeLayout.defaultLayout() + private var viewWidth = 1 + private var viewHeight = 1 + private var frameAvailable = false + private var released = false + private var started = false + + private var eglDisplay: EGLDisplay = EGL14.EGL_NO_DISPLAY + private var eglContext: EGLContext = EGL14.EGL_NO_CONTEXT + private var eglSurface: EGLSurface = EGL14.EGL_NO_SURFACE + private var program = 0 + private var videoTextureId = 0 + private var positionHandle = 0 + private var texCoordHandle = 0 + private var textureMatrixHandle = 0 + private var textureHandle = 0 + private var colorRectHandle = 0 + private var alphaRectHandle = 0 + private var hasAlphaHandle = 0 + + fun attach(surfaceTexture: SurfaceTexture, width: Int, height: Int) { + handler.post { + if (released) return@post + outputSurfaceTexture = surfaceTexture + viewWidth = maxOf(1, width) + viewHeight = maxOf(1, height) + setupEgl(surfaceTexture) + setupProgram() + setupVideoTexture() + prepareSourceAndPlayer() } + } - private fun normalSourceLayout(videoSize: VideoSize): SourceLayout { - val fullFrame = SourceRect(0f, 0f, 1f, 1f) - return SourceLayout( - kind = "normal", - hasAlphaMask = false, - useTextureMatrix = true, - flipY = true, - contentAspectRatio = videoSize.aspectRatio, - colorRect = fullFrame, - alphaRect = fullFrame, - ) + fun resize(width: Int, height: Int) { + handler.post { + viewWidth = maxOf(1, width) + viewHeight = maxOf(1, height) + drawFrame(force = true) } + } - private fun splitSourceLayout( - candidate: SplitAlphaCandidate, - videoSize: VideoSize, - ): SourceLayout { - val left = SourceRect(0f, 0f, 0.5f, 1f) - val right = SourceRect(0.5f, 0f, 0.5f, 1f) - val top = SourceRect(0f, 0.5f, 1f, 0.5f) - val bottom = SourceRect(0f, 0f, 1f, 0.5f) - val isHorizontal = candidate.orientation == SplitOrientation.HORIZONTAL - val first = if (isHorizontal) left else top - val second = if (isHorizontal) right else bottom - val outputAspectRatio = - if (isHorizontal) { - videoSize.width.toFloat() / 2f / videoSize.height.toFloat() - } else { - videoSize.width.toFloat() / (videoSize.height.toFloat() / 2f) - } - return SourceLayout( - kind = if (isHorizontal) "splitAlphaHorizontal" else "splitAlphaVertical", - hasAlphaMask = true, - useTextureMatrix = true, - flipY = true, - contentAspectRatio = outputAspectRatio, - colorRect = if (candidate.alphaFirst) second else first, - alphaRect = if (candidate.alphaFirst) first else second, - ) + fun detach() { + handler.post { + releasePlayer() + releaseGl() + outputSurfaceTexture = null } + } - private fun parseVapcSourceLayout(source: GiftMp4Source): SourceLayout? { - return try { - val json = extractVapcJson(source.readBytes()) ?: return null - val info = JSONObject(json).getJSONObject("info") - val videoWidth = info.optDouble("videoW", 0.0).toFloat() - val videoHeight = info.optDouble("videoH", 0.0).toFloat() - val outputWidth = info.optDouble("w", 0.0).toFloat() - val outputHeight = info.optDouble("h", 0.0).toFloat() - if (videoWidth <= 0f || videoHeight <= 0f || outputWidth <= 0f || outputHeight <= 0f) { - return null - } + fun setMuted(muted: Boolean) { + handler.post { + mutedVolume[0] = if (muted) 0f else 1f + mediaPlayer?.setVolume(mutedVolume[0], mutedVolume[0]) + } + } - SourceLayout( - kind = "vapc", - hasAlphaMask = true, - useTextureMatrix = false, - flipY = false, - contentAspectRatio = outputWidth / outputHeight, - colorRect = parseNormalizedRect(info.getJSONArray("rgbFrame"), videoWidth, videoHeight), - alphaRect = parseNormalizedRect(info.getJSONArray("aFrame"), videoWidth, videoHeight), - ) - } catch (_: Exception) { + fun stop() { + handler.post { + releasePlayer() + } + } + + fun release() { + handler.post { + released = true + releasePlayer() + releaseGl() + sourceData?.close() + sourceData = null + renderThread.quitSafely() + } + } + + private fun prepareSourceAndPlayer() { + val resolved = + try { + GiftMp4ResolvedSource.resolve(context, sourceType, source) + } catch (error: Exception) { + Log.e("GiftMp4Video", "resolve source failed path=$source", error) + onFailed(-1, error.message) null + } ?: return + sourceData = resolved + val creationLayout = providedLayout + val detectedLayout by lazy { GiftMp4NativeLayout.detectLayout(resolved) } + val layoutSource: String + layout = if (creationLayout != null) { + layoutSource = "creationParams" + creationLayout + } else { + val parsedLayout = GiftMp4NativeLayout.parseVapcLayout(resolved) + if (parsedLayout != null) { + layoutSource = "vapc" + parsedLayout + } else { + layoutSource = "frame" + detectedLayout } } + Log.d( + "GiftMp4Video", + "layout source=$layoutSource path=$source hasAlpha=${layout.hasAlphaMask} " + + "contentAspect=${layout.contentAspectRatio} " + + "color=${layout.colorRect} alpha=${layout.alphaRect}", + ) - private fun parseNormalizedRect( - values: JSONArray, - videoWidth: Float, - videoHeight: Float, - ): SourceRect { - return SourceRect( - x = (values.getDouble(0).toFloat() / videoWidth).coerceIn(0f, 1f), - y = (values.getDouble(1).toFloat() / videoHeight).coerceIn(0f, 1f), - width = (values.getDouble(2).toFloat() / videoWidth).coerceIn(0f, 1f), - height = (values.getDouble(3).toFloat() / videoHeight).coerceIn(0f, 1f), + val player = MediaPlayer() + mediaPlayer = player + try { + resolved.setMediaPlayerDataSource(player) + player.setSurface(videoSurface) + player.isLooping = false + player.setVolume(mutedVolume[0], mutedVolume[0]) + player.setOnPreparedListener { + if (!released && mediaPlayer === it) { + started = true + onStarted() + it.start() + } + } + player.setOnCompletionListener { + if (!released && mediaPlayer === it) { + onCompleted() + } + } + player.setOnErrorListener { _, what, extra -> + onFailed(what, "extra=$extra") + true + } + player.prepareAsync() + } catch (error: Exception) { + Log.e("GiftMp4Video", "prepare player failed path=$source", error) + onFailed(-2, error.message) + releasePlayer() + } + } + + private fun setupEgl(surfaceTexture: SurfaceTexture) { + outputSurface = Surface(surfaceTexture) + eglDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY) + val version = IntArray(2) + EGL14.eglInitialize(eglDisplay, version, 0, version, 1) + val attribList = + intArrayOf( + EGL14.EGL_RED_SIZE, 8, + EGL14.EGL_GREEN_SIZE, 8, + EGL14.EGL_BLUE_SIZE, 8, + EGL14.EGL_ALPHA_SIZE, 8, + EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT, + EGL14.EGL_NONE, ) - } + val configs = arrayOfNulls(1) + val numConfigs = IntArray(1) + EGL14.eglChooseConfig(eglDisplay, attribList, 0, configs, 0, 1, numConfigs, 0) + val config = configs[0] ?: return + val contextAttribs = intArrayOf(EGL14.EGL_CONTEXT_CLIENT_VERSION, 2, EGL14.EGL_NONE) + eglContext = + EGL14.eglCreateContext( + eglDisplay, + config, + EGL14.EGL_NO_CONTEXT, + contextAttribs, + 0, + ) + eglSurface = + EGL14.eglCreateWindowSurface( + eglDisplay, + config, + outputSurface, + intArrayOf(EGL14.EGL_NONE), + 0, + ) + EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext) + GLES20.glDisable(GLES20.GL_DEPTH_TEST) + GLES20.glEnable(GLES20.GL_BLEND) + GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA) + GLES20.glClearColor(0f, 0f, 0f, 0f) + } - private fun extractVapcJson(bytes: ByteArray): String? { - val marker = byteArrayOf('v'.code.toByte(), 'a'.code.toByte(), 'p'.code.toByte(), 'c'.code.toByte()) - var searchStart = 0 - while (searchStart < bytes.size) { - val markerIndex = indexOf(bytes, marker, searchStart) - if (markerIndex < 0) return null - val scanEnd = minOf(bytes.size, markerIndex + 128) - val jsonStart = - (markerIndex + marker.size until scanEnd) - .firstOrNull { bytes[it] == '{'.code.toByte() } - if (jsonStart != null) { - val jsonEnd = findJsonEnd(bytes, jsonStart) - if (jsonEnd != null) { - return String(bytes, jsonStart, jsonEnd - jsonStart + 1, Charsets.UTF_8) - } + private fun setupProgram() { + program = createProgram(VERTEX_SHADER, FRAGMENT_SHADER) + positionHandle = GLES20.glGetAttribLocation(program, "aPosition") + texCoordHandle = GLES20.glGetAttribLocation(program, "aTexCoord") + textureMatrixHandle = GLES20.glGetUniformLocation(program, "uTextureMatrix") + textureHandle = GLES20.glGetUniformLocation(program, "uTexture") + colorRectHandle = GLES20.glGetUniformLocation(program, "uColorRect") + alphaRectHandle = GLES20.glGetUniformLocation(program, "uAlphaRect") + hasAlphaHandle = GLES20.glGetUniformLocation(program, "uHasAlpha") + } + + private fun setupVideoTexture() { + val textures = IntArray(1) + GLES20.glGenTextures(1, textures, 0) + videoTextureId = textures[0] + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, videoTextureId) + GLES20.glTexParameteri( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, + GLES20.GL_TEXTURE_MIN_FILTER, + GLES20.GL_LINEAR, + ) + GLES20.glTexParameteri( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, + GLES20.GL_TEXTURE_MAG_FILTER, + GLES20.GL_LINEAR, + ) + GLES20.glTexParameteri( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, + GLES20.GL_TEXTURE_WRAP_S, + GLES20.GL_CLAMP_TO_EDGE, + ) + GLES20.glTexParameteri( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, + GLES20.GL_TEXTURE_WRAP_T, + GLES20.GL_CLAMP_TO_EDGE, + ) + + val surfaceTexture = SurfaceTexture(videoTextureId) + videoSurfaceTexture = surfaceTexture + surfaceTexture.setOnFrameAvailableListener( + { frameSurfaceTexture -> + if (released || frameSurfaceTexture !== videoSurfaceTexture) { + return@setOnFrameAvailableListener } - searchStart = markerIndex + marker.size - } - return null + frameAvailable = true + drawFrame() + }, + handler, + ) + videoSurface = Surface(surfaceTexture) + } + + private fun drawFrame(force: Boolean = false) { + if (eglDisplay == EGL14.EGL_NO_DISPLAY || + eglSurface == EGL14.EGL_NO_SURFACE || + program == 0 || + viewWidth <= 0 || + viewHeight <= 0 + ) { + return + } + EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext) + if (frameAvailable) { + videoSurfaceTexture?.updateTexImage() + videoSurfaceTexture?.getTransformMatrix(textureMatrix) + frameAvailable = false + } else if (!force) { + return } - private fun indexOf( - bytes: ByteArray, - marker: ByteArray, - start: Int, - ): Int { - var index = start - while (index <= bytes.size - marker.size) { - var matched = true - for (offset in marker.indices) { - if (bytes[index + offset] != marker[offset]) { - matched = false - break - } - } - if (matched) return index - index++ - } - return -1 - } + GLES20.glViewport(0, 0, viewWidth, viewHeight) + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT) + val viewport = contentViewport(viewWidth, viewHeight, layout.contentAspectRatio) + GLES20.glViewport(viewport.x, viewport.y, viewport.width, viewport.height) + GLES20.glUseProgram(program) + GLES20.glActiveTexture(GLES20.GL_TEXTURE0) + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, videoTextureId) + GLES20.glUniform1i(textureHandle, 0) + GLES20.glUniformMatrix4fv( + textureMatrixHandle, + 1, + false, + if (textureMatrix.any { it != 0f }) textureMatrix else identityMatrix, + 0, + ) + GLES20.glUniform4f( + colorRectHandle, + layout.colorRect.x, + layout.colorRect.y, + layout.colorRect.width, + layout.colorRect.height, + ) + val alpha = layout.alphaRect + GLES20.glUniform4f( + alphaRectHandle, + alpha?.x ?: 0f, + alpha?.y ?: 0f, + alpha?.width ?: 1f, + alpha?.height ?: 1f, + ) + GLES20.glUniform1i(hasAlphaHandle, if (layout.hasAlphaMask) 1 else 0) - private fun findJsonEnd( - bytes: ByteArray, - start: Int, - ): Int? { - var depth = 0 - var inString = false - var escaped = false - var index = start - while (index < bytes.size) { - val char = bytes[index].toInt().toChar() - if (inString) { - when { - escaped -> escaped = false - char == '\\' -> escaped = true - char == '"' -> inString = false - } + QUAD_VERTICES.position(0) + GLES20.glVertexAttribPointer(positionHandle, 2, GLES20.GL_FLOAT, false, 16, QUAD_VERTICES) + GLES20.glEnableVertexAttribArray(positionHandle) + QUAD_VERTICES.position(2) + GLES20.glVertexAttribPointer(texCoordHandle, 2, GLES20.GL_FLOAT, false, 16, QUAD_VERTICES) + GLES20.glEnableVertexAttribArray(texCoordHandle) + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4) + GLES20.glDisableVertexAttribArray(positionHandle) + GLES20.glDisableVertexAttribArray(texCoordHandle) + EGL14.eglSwapBuffers(eglDisplay, eglSurface) + } + + private fun releasePlayer() { + videoSurfaceTexture?.setOnFrameAvailableListener(null) + mediaPlayer?.let { + runCatching { it.setOnPreparedListener(null) } + runCatching { it.setOnCompletionListener(null) } + runCatching { it.setOnErrorListener(null) } + runCatching { it.stop() } + runCatching { it.release() } + } + mediaPlayer = null + videoSurface?.release() + videoSurface = null + videoSurfaceTexture?.release() + videoSurfaceTexture = null + started = false + } + + private fun releaseGl() { + if (eglDisplay == EGL14.EGL_NO_DISPLAY) { + outputSurface?.release() + outputSurface = null + return + } + EGL14.eglMakeCurrent( + eglDisplay, + EGL14.EGL_NO_SURFACE, + EGL14.EGL_NO_SURFACE, + EGL14.EGL_NO_CONTEXT, + ) + if (videoTextureId != 0) { + GLES20.glDeleteTextures(1, intArrayOf(videoTextureId), 0) + videoTextureId = 0 + } + if (program != 0) { + GLES20.glDeleteProgram(program) + program = 0 + } + EGL14.eglDestroySurface(eglDisplay, eglSurface) + EGL14.eglDestroyContext(eglDisplay, eglContext) + EGL14.eglTerminate(eglDisplay) + eglDisplay = EGL14.EGL_NO_DISPLAY + eglContext = EGL14.EGL_NO_CONTEXT + eglSurface = EGL14.EGL_NO_SURFACE + outputSurface?.release() + outputSurface = null + } + + companion object { + private val QUAD_VERTICES = + floatBufferOf( + -1f, -1f, 0f, 1f, + 1f, -1f, 1f, 1f, + -1f, 1f, 0f, 0f, + 1f, 1f, 1f, 0f, + ) + + private const val VERTEX_SHADER = + """ + attribute vec4 aPosition; + attribute vec2 aTexCoord; + varying vec2 vTexCoord; + void main() { + gl_Position = aPosition; + vTexCoord = aTexCoord; + } + """ + + private const val FRAGMENT_SHADER = + """ + #extension GL_OES_EGL_image_external : require + precision mediump float; + uniform samplerExternalOES uTexture; + uniform mat4 uTextureMatrix; + uniform vec4 uColorRect; + uniform vec4 uAlphaRect; + uniform int uHasAlpha; + varying vec2 vTexCoord; + + vec2 mapRect(vec4 rect, vec2 uv) { + return vec2(rect.x + uv.x * rect.z, 1.0 - rect.y - uv.y * rect.w); + } + + void main() { + vec2 colorUv = (uTextureMatrix * vec4(mapRect(uColorRect, vTexCoord), 0.0, 1.0)).xy; + vec4 color = texture2D(uTexture, colorUv); + if (uHasAlpha == 0) { + gl_FragColor = color; + return; + } + vec2 alphaUv = (uTextureMatrix * vec4(mapRect(uAlphaRect, vTexCoord), 0.0, 1.0)).xy; + vec4 mask = texture2D(uTexture, alphaUv); + float alpha = dot(mask.rgb, vec3(0.299, 0.587, 0.114)); + gl_FragColor = vec4(color.rgb, alpha); + } + """ + } +} + +private data class GiftMp4Viewport( + val x: Int, + val y: Int, + val width: Int, + val height: Int, +) + +private data class GiftMp4Rect( + val x: Float, + val y: Float, + val width: Float, + val height: Float, +) + +private data class GiftMp4NativeLayout( + val colorRect: GiftMp4Rect, + val alphaRect: GiftMp4Rect?, + val contentAspectRatio: Float = + if (colorRect.width > 0f && colorRect.height > 0f) { + colorRect.width / colorRect.height + } else { + 1f + }, +) { + val hasAlphaMask: Boolean = alphaRect != null + + companion object { + fun defaultLayout(): GiftMp4NativeLayout = + GiftMp4NativeLayout(GiftMp4Rect(0f, 0f, 1f, 1f), null) + + fun fromCreationParams(data: Any?): GiftMp4NativeLayout? = runCatching { + val map = data as? Map<*, *> ?: return null + val color = readLayoutRect(map["colorRect"]) ?: return null + val hasAlpha = readLayoutBool(map["hasAlphaMask"]) + val alpha = + if (hasAlpha) { + readLayoutRect(map["alphaRect"]) ?: return null } else { - when (char) { - '"' -> inString = true - '{' -> depth++ - '}' -> { - depth-- - if (depth == 0) return index - } - } + null } - index++ - } - return null - } - - private fun detectSplitSourceLayout( - source: GiftMp4Source, - videoSize: VideoSize, - ): SourceLayout? { - val candidate = detectSplitAlpha(source) ?: return null - return splitSourceLayout(candidate, videoSize) - } - - private fun detectSplitAlpha(source: GiftMp4Source): SplitAlphaCandidate? { - val retriever = source.newMetadataRetriever() - return try { - val durationMs = - retriever - .extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION) - ?.toLongOrNull() - ?: 0L - val timesUs = splitSampleTimesUs(durationMs) - - val horizontalScore = SplitScore() - val verticalScore = SplitScore() - for (timeUs in timesUs) { - val frame = - retriever.getFrameAtTime( - timeUs, - MediaMetadataRetriever.OPTION_CLOSEST, - ) ?: continue - try { - val horizontalStats = statsByHorizontalHalves(frame) - val verticalStats = statsByVerticalHalves(frame) - addSplitScore(horizontalScore, horizontalStats.first, horizontalStats.second) - addSplitScore(verticalScore, verticalStats.first, verticalStats.second) - } finally { - frame.recycle() - } + val fallbackAspectRatio = + if (color.width > 0f && color.height > 0f) { + color.width / color.height + } else { + 1f } - val horizontalCandidate = - splitCandidateFromScore(SplitOrientation.HORIZONTAL, horizontalScore) - val verticalCandidate = - splitCandidateFromScore(SplitOrientation.VERTICAL, verticalScore) - listOfNotNull(horizontalCandidate, verticalCandidate) - .maxByOrNull { it.second } - ?.first - } catch (_: RuntimeException) { - null - } finally { - retriever.release() - } - } + val contentAspectRatio = + readLayoutFloat(map["contentAspectRatio"], fallbackAspectRatio) + .takeIf { it > 0f } + ?: fallbackAspectRatio + GiftMp4NativeLayout(color, alpha, contentAspectRatio) + }.getOrNull() - private fun splitSampleTimesUs(durationMs: Long): List { - val durationUs = durationMs * 1000L - val desiredTimesUs = + fun parseVapcLayout(source: GiftMp4ResolvedSource): GiftMp4NativeLayout? = runCatching { + val bytes = source.readBytes() ?: return null + val jsonText = parseVapcJson(bytes) ?: return null + val root = JSONObject(jsonText) + val info = root.optJSONObject("info") ?: return null + val videoWidth = readPositiveInt(info, "videoW") + ?: readPositiveInt(info, "video_width") + ?: return null + val videoHeight = readPositiveInt(info, "videoH") + ?: readPositiveInt(info, "video_height") + ?: return null + val contentWidth = readPositiveInt(info, "w") ?: videoWidth + val contentHeight = readPositiveInt(info, "h") ?: videoHeight + val color = readFrame(info, "rgbFrame") + ?: readFrame(info, "rgb_frame") + ?: return null + val alpha = readFrame(info, "aFrame") + ?: readFrame(info, "a_frame") + ?: return null + GiftMp4NativeLayout( + normalizeRect(color, videoWidth, videoHeight), + normalizeRect(alpha, videoWidth, videoHeight), + contentWidth.toFloat() / contentHeight.toFloat(), + ) + }.getOrNull() + + fun detectLayout(source: GiftMp4ResolvedSource): GiftMp4NativeLayout { + val retriever = MediaMetadataRetriever() + val bitmap = + try { + source.setRetrieverDataSource(retriever) + retriever.getFrameAtTime(1_000_000, MediaMetadataRetriever.OPTION_CLOSEST_SYNC) + ?: retriever.getFrameAtTime(0, MediaMetadataRetriever.OPTION_CLOSEST_SYNC) + } catch (_: Exception) { + null + } finally { + runCatching { retriever.release() } + } ?: return defaultLayout() + + detectPackedAlphaTopRight(bitmap)?.let { packed -> + bitmap.recycle() + return packed + } + + val candidates = listOf( - 300_000L, - 600_000L, - 900_000L, - 1_200_000L, - 1_500_000L, - 1_800_000L, - 2_100_000L, - 2_400_000L, - 2_700_000L, - 3_000_000L, - ) - return desiredTimesUs - .filter { durationMs <= 0L || it <= durationUs } - .ifEmpty { - if (durationUs > 0L) { - listOf(durationUs / 2L) - } else { - listOf(0L) - } - } - } - - private fun detectPackedAlphaTopRightSourceLayout( - source: GiftMp4Source, - ): SourceLayout? { - val candidate = detectPackedAlphaTopRight(source) ?: return null - val frameWidth = candidate.frameWidth.coerceAtLeast(1) - val frameHeight = candidate.frameHeight.coerceAtLeast(1) - val alphaStartX = candidate.alphaStartX.coerceIn(1, frameWidth - 1) - val alphaEndX = candidate.alphaEndX.coerceIn(alphaStartX, frameWidth - 1) - val colorWidth = alphaStartX - val alphaWidth = (alphaEndX - alphaStartX + 1).coerceAtLeast(1) - val alphaHeight = - (frameHeight.toFloat() * alphaWidth.toFloat() / colorWidth.toFloat() + 0.5f) - .toInt() - .coerceIn(1, frameHeight) - val alphaHeightRatio = alphaHeight.toFloat() / frameHeight.toFloat() - - return SourceLayout( - kind = "packedAlphaTopRight", - hasAlphaMask = true, - useTextureMatrix = true, - flipY = true, - contentAspectRatio = colorWidth.toFloat() / frameHeight.toFloat(), - colorRect = - SourceRect( - x = 0f, - y = 0f, - width = colorWidth.toFloat() / frameWidth.toFloat(), - height = 1f, + GiftMp4LayoutCandidate( + GiftMp4Rect(0f, 0f, 0.5f, 1f), + GiftMp4Rect(0.5f, 0f, 0.5f, 1f), ), - alphaRect = - SourceRect( - x = alphaStartX.toFloat() / frameWidth.toFloat(), - y = 1f - alphaHeightRatio, - width = alphaWidth.toFloat() / frameWidth.toFloat(), - height = alphaHeightRatio, + GiftMp4LayoutCandidate( + GiftMp4Rect(0.5f, 0f, 0.5f, 1f), + GiftMp4Rect(0f, 0f, 0.5f, 1f), + ), + GiftMp4LayoutCandidate( + GiftMp4Rect(0f, 0f, 1f, 0.5f), + GiftMp4Rect(0f, 0.5f, 1f, 0.5f), + ), + GiftMp4LayoutCandidate( + GiftMp4Rect(0f, 0.5f, 1f, 0.5f), + GiftMp4Rect(0f, 0f, 1f, 0.5f), ), - ) - } - - private fun detectPackedAlphaTopRight(source: GiftMp4Source): PackedAlphaCandidate? { - val retriever = source.newMetadataRetriever() - return try { - val durationMs = - retriever - .extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION) - ?.toLongOrNull() - ?: 0L - val timesUs = - listOf(0L, 500_000L, 1_000_000L, 2_000_000L, 3_000_000L) - .filter { durationMs == 0L || it <= durationMs * 1000L } - .ifEmpty { listOf(0L) } - - val candidates = mutableListOf() - for (timeUs in timesUs) { - val frame = - retriever.getFrameAtTime( - timeUs, - MediaMetadataRetriever.OPTION_CLOSEST_SYNC, - ) ?: continue - detectPackedAlphaTopRightInFrame(frame)?.let { candidates.add(it) } - frame.recycle() - } - if (candidates.isEmpty()) { - return null - } - - val frameWidth = medianInt(candidates.map { it.frameWidth }) - val frameHeight = medianInt(candidates.map { it.frameHeight }) - val alphaStartX = medianInt(candidates.map { it.alphaStartX }) - val alphaEndX = medianInt(candidates.map { it.alphaEndX }) - val alphaWidth = alphaEndX - alphaStartX + 1 - val alphaStartRatio = alphaStartX.toDouble() / frameWidth.toDouble() - val alphaWidthRatio = alphaWidth.toDouble() / frameWidth.toDouble() - val colorToAlphaRatio = alphaStartX.toDouble() / alphaWidth.toDouble() - - if (alphaStartRatio !in 0.58..0.78 || - alphaWidthRatio !in 0.18..0.42 || - colorToAlphaRatio !in 1.45..2.65 || - alphaEndX < frameWidth * 0.82 - ) { - return null - } - PackedAlphaCandidate( - frameWidth = frameWidth, - frameHeight = frameHeight, - alphaStartX = alphaStartX, - alphaEndX = alphaEndX, ) - } catch (_: RuntimeException) { - null - } finally { - retriever.release() - } + val scoredCandidates = + candidates.map { candidate -> + val colorStats = sampleStats(bitmap, candidate.colorRect) + val alphaStats = sampleStats(bitmap, candidate.alphaRect) + GiftMp4CandidateScore( + candidate, + alphaStats.maskScore - colorStats.maskScore + colorStats.colorScore * 0.4, + ) + } + val sortedCandidates = scoredCandidates.sortedByDescending { it.score } + val best = sortedCandidates.firstOrNull() + val runnerUpScore = sortedCandidates.drop(1).firstOrNull()?.score ?: 0.0 + val selected = + if (best != null && (best.score - runnerUpScore >= 0.12 || best.score >= 0.35)) { + best.candidate + } else { + candidates.first() + } + bitmap.recycle() + return GiftMp4NativeLayout(selected.colorRect, selected.alphaRect) } - private fun detectPackedAlphaTopRightInFrame(bitmap: Bitmap): PackedAlphaCandidate? { + private fun detectPackedAlphaTopRight(bitmap: Bitmap): GiftMp4NativeLayout? { val width = bitmap.width val height = bitmap.height if (width < 12 || height < 12) { @@ -680,7 +724,6 @@ class GiftMp4VideoPlatformView( } val searchStart = (width * 0.54f).toInt().coerceIn(0, width - 1) - val searchEnd = width val sampleHeight = (height * 0.58f).toInt().coerceIn(1, height) val stepY = maxOf(1, sampleHeight / 220) val rowSamples = ((sampleHeight + stepY - 1) / stepY).coerceAtLeast(1) @@ -688,7 +731,7 @@ class GiftMp4VideoPlatformView( val grayColumns = BooleanArray(width) var x = searchStart - while (x < searchEnd) { + while (x < width) { var grayCount = 0 var y = 0 while (y < sampleHeight) { @@ -705,15 +748,15 @@ class GiftMp4VideoPlatformView( var bestStart = -1 var bestEnd = -1 var index = searchStart - while (index < searchEnd) { - while (index < searchEnd && !grayColumns[index]) { + while (index < width) { + while (index < width && !grayColumns[index]) { index++ } - if (index >= searchEnd) { + if (index >= width) { break } val start = index - while (index < searchEnd && grayColumns[index]) { + while (index < width && grayColumns[index]) { index++ } val end = index - 1 @@ -722,146 +765,344 @@ class GiftMp4VideoPlatformView( bestEnd = end } } - if (bestStart < 0 || bestEnd < 0) { return null } - return PackedAlphaCandidate( - frameWidth = width, - frameHeight = height, - alphaStartX = bestStart, - alphaEndX = bestEnd, + + val alphaWidth = bestEnd - bestStart + 1 + val alphaStartRatio = bestStart.toDouble() / width.toDouble() + val alphaWidthRatio = alphaWidth.toDouble() / width.toDouble() + val colorToAlphaRatio = bestStart.toDouble() / alphaWidth.toDouble() + if (alphaStartRatio !in 0.58..0.78 || + alphaWidthRatio !in 0.18..0.42 || + colorToAlphaRatio !in 1.45..2.65 || + bestEnd < width * 0.82 + ) { + return null + } + + val colorWidth = bestStart.coerceAtLeast(1) + val alphaHeight = + (height.toFloat() * alphaWidth.toFloat() / colorWidth.toFloat() + 0.5f) + .toInt() + .coerceIn(1, height) + val alphaHeightRatio = alphaHeight.toFloat() / height.toFloat() + return GiftMp4NativeLayout( + colorRect = + GiftMp4Rect( + x = 0f, + y = 0f, + width = colorWidth.toFloat() / width.toFloat(), + height = 1f, + ), + alphaRect = + GiftMp4Rect( + x = bestStart.toFloat() / width.toFloat(), + y = 1f - alphaHeightRatio, + width = alphaWidth.toFloat() / width.toFloat(), + height = alphaHeightRatio, + ), + contentAspectRatio = colorWidth.toFloat() / height.toFloat(), ) } private fun isLikelyAlphaMaskPixel(pixel: Int): Boolean { - val red = Color.red(pixel) - val green = Color.green(pixel) - val blue = Color.blue(pixel) - val maxChannel = maxOf(red, green, blue) + val red = (pixel shr 16) and 0xff + val green = (pixel shr 8) and 0xff + val blue = pixel and 0xff + val maxChannel = max(red, max(green, blue)) if (maxChannel < 22) { return false } - val minChannel = minOf(red, green, blue) + val minChannel = min(red, min(green, blue)) val saturation = if (maxChannel == 0) 0.0 else (maxChannel - minChannel).toDouble() / maxChannel val grayError = - (kotlin.math.abs(red - green) + - kotlin.math.abs(green - blue) + - kotlin.math.abs(blue - red)).toDouble() / (3.0 * 255.0) + (abs(red - green) + abs(green - blue) + abs(blue - red)).toDouble() / (3.0 * 255.0) return saturation < 0.16 && grayError < 0.055 } - private fun medianInt(values: List): Int { - if (values.isEmpty()) { - return 0 + private fun parseVapcJson(bytes: ByteArray): String? { + val marker = "vapc".toByteArray(Charsets.US_ASCII) + val markerIndex = bytes.indexOf(marker, 0) + if (markerIndex < 0) return null + val searchEnd = min(bytes.size, markerIndex + marker.size + 512 * 1024) + for (index in markerIndex + marker.size until searchEnd) { + if (bytes[index] != '{'.code.toByte()) continue + val end = findJsonObjectEnd(bytes, index, searchEnd) + if (end > index) { + return bytes.copyOfRange(index, end + 1).toString(Charsets.UTF_8) + } } - return values.sorted()[values.size / 2] + return null } - private fun addSplitScore( - score: SplitScore, - first: HalfStats, - second: HalfStats, - ) { - if (isLowBrightnessFrame(first, second)) { - return + private fun findJsonObjectEnd(bytes: ByteArray, start: Int, maxEnd: Int): Int { + var depth = 0 + var inString = false + var escaping = false + for (index in start until maxEnd) { + val byte = bytes[index] + if (inString) { + if (escaping) { + escaping = false + } else if (byte == '\\'.code.toByte()) { + escaping = true + } else if (byte == '"'.code.toByte()) { + inString = false + } + } else if (byte == '"'.code.toByte()) { + inString = true + } else if (byte == '{'.code.toByte()) { + depth += 1 + } else if (byte == '}'.code.toByte()) { + depth -= 1 + if (depth == 0) return index + } } - val alphaStats = if (first.saturation < second.saturation) first else second - val saturationGap = kotlin.math.abs(second.saturation - first.saturation) - if (saturationGap < MIN_SPLIT_SATURATION_GAP || - alphaStats.saturation > MAX_SPLIT_MASK_SATURATION || - alphaStats.grayError > MAX_SPLIT_MASK_GRAY_ERROR - ) { - return - } - score.direction += second.saturation - first.saturation - score.saturationGap += saturationGap - score.maskSaturation += alphaStats.saturation - score.maskGrayError += alphaStats.grayError - score.frames++ + return -1 } - private fun isLowBrightnessFrame( - first: HalfStats, - second: HalfStats, - ): Boolean = maxOf(first.brightness, second.brightness) < MIN_USEFUL_FRAME_BRIGHTNESS + private fun readPositiveInt(info: JSONObject, key: String): Int? { + val value = + when (val raw = info.opt(key)) { + is Number -> raw.toInt() + is String -> raw.trim().toIntOrNull() + else -> null + } + return value?.takeIf { it > 0 } + } - private fun splitCandidateFromScore( - orientation: SplitOrientation, - score: SplitScore, - ): Pair? { - if (score.frames < MIN_SPLIT_MATCH_FRAMES) return null - val averageSaturationGap = score.saturationGap / score.frames - val averageMaskSaturation = score.maskSaturation / score.frames - val averageMaskGrayError = score.maskGrayError / score.frames - if (averageSaturationGap < MIN_SPLIT_SATURATION_GAP || - averageMaskSaturation > MAX_SPLIT_MASK_SATURATION || - averageMaskGrayError > MAX_SPLIT_MASK_GRAY_ERROR - ) { + private fun readFrame(info: JSONObject, key: String): IntArray? { + val values = info.optJSONArray(key)?.toIntList() + ?: info.optString(key, "") + .takeIf { it.isNotBlank() } + ?.split(',') + ?.map { value -> value.trim().toIntOrNull() ?: return null } + ?: return null + if (values.size < 4 || values[2] <= 0 || values[3] <= 0) return null + return intArrayOf(values[0], values[1], values[2], values[3]) + } + + private fun JSONArray.toIntList(): List { + val values = ArrayList(length()) + for (index in 0 until length()) { + when (val value = opt(index)) { + is Number -> values.add(value.toInt()) + is String -> value.trim().toIntOrNull()?.let(values::add) + } + } + return values + } + + private fun normalizeRect(values: IntArray, videoWidth: Int, videoHeight: Int): GiftMp4Rect = + GiftMp4Rect( + values[0].toFloat() / videoWidth, + values[1].toFloat() / videoHeight, + values[2].toFloat() / videoWidth, + values[3].toFloat() / videoHeight, + ) + + private fun readLayoutRect(data: Any?): GiftMp4Rect? { + val values = + (data as? Iterable<*>)?.mapNotNull(::readLayoutFloatOrNull) + ?.toList() + ?: return null + if (values.size < 4 || values[2] <= 0f || values[3] <= 0f) { return null } - return SplitAlphaCandidate( - orientation = orientation, - alphaFirst = score.direction > 0.0, - ) to averageSaturationGap + return GiftMp4Rect(values[0], values[1], values[2], values[3]) } - private fun statsByHorizontalHalves(bitmap: Bitmap): Pair { - val half = bitmap.width / 2 - return statsForRect(bitmap, 0, 0, half, bitmap.height) to - statsForRect(bitmap, half, 0, bitmap.width, bitmap.height) - } + private fun readLayoutBool(data: Any?): Boolean = + when (data) { + is Boolean -> data + is Number -> data.toInt() != 0 + is String -> data.equals("true", ignoreCase = true) + else -> false + } - private fun statsByVerticalHalves(bitmap: Bitmap): Pair { - val half = bitmap.height / 2 - return statsForRect(bitmap, 0, 0, bitmap.width, half) to - statsForRect(bitmap, 0, half, bitmap.width, bitmap.height) - } + private fun readLayoutFloat(data: Any?, fallback: Float): Float = + readLayoutFloatOrNull(data) ?: fallback - private fun statsForRect( - bitmap: Bitmap, - startX: Int, - startY: Int, - endX: Int, - endY: Int, - ): HalfStats { - val stepX = maxOf(1, (endX - startX) / 80) - val stepY = maxOf(1, (endY - startY) / 120) + private fun readLayoutFloatOrNull(data: Any?): Float? = + when (data) { + is Number -> data.toFloat() + is String -> data.trim().toFloatOrNull() + else -> null + } + + private fun sampleStats(bitmap: Bitmap, rect: GiftMp4Rect): GiftMp4FrameStats { + val sampleCount = 12 var saturationSum = 0.0 var grayErrorSum = 0.0 - var brightnessSum = 0.0 - var count = 0 - var y = startY - while (y < endY) { - var x = startX - while (x < endX) { + val brightnessValues = ArrayList(sampleCount * sampleCount) + for (yIndex in 0 until sampleCount) { + for (xIndex in 0 until sampleCount) { + val normalizedX = rect.x + rect.width * (xIndex + 0.5f) / sampleCount + val normalizedY = rect.y + rect.height * (yIndex + 0.5f) / sampleCount + val x = (normalizedX * bitmap.width).toInt().coerceIn(0, bitmap.width - 1) + val y = (normalizedY * bitmap.height).toInt().coerceIn(0, bitmap.height - 1) val pixel = bitmap.getPixel(x, y) - val red = Color.red(pixel) / 255.0 - val green = Color.green(pixel) / 255.0 - val blue = Color.blue(pixel) / 255.0 - val maxChannel = maxOf(red, green, blue) - val minChannel = minOf(red, green, blue) - saturationSum += - if (maxChannel == 0.0) 0.0 else (maxChannel - minChannel) / maxChannel - grayErrorSum += - (kotlin.math.abs(red - green) + - kotlin.math.abs(green - blue) + - kotlin.math.abs(blue - red)) / 3.0 - brightnessSum += red * 0.299 + green * 0.587 + blue * 0.114 - count++ - x += stepX + val r = ((pixel shr 16) and 0xff) / 255.0 + val g = ((pixel shr 8) and 0xff) / 255.0 + val b = (pixel and 0xff) / 255.0 + val maxValue = max(r, max(g, b)) + val minValue = min(r, min(g, b)) + val saturation = if (maxValue > 0.001) (maxValue - minValue) / maxValue else 0.0 + val grayError = (abs(r - g) + abs(r - b) + abs(g - b)) / 3.0 + val brightness = (r + g + b) / 3.0 + saturationSum += saturation + grayErrorSum += grayError + brightnessValues.add(brightness) } - y += stepY } - if (count == 0) { - return HalfStats(saturation = 0.0, grayError = 0.0, brightness = 0.0) - } - return HalfStats( - saturation = saturationSum / count, - grayError = grayErrorSum / count, - brightness = brightnessSum / count, - ) + val count = brightnessValues.size.coerceAtLeast(1).toDouble() + val averageSaturation = saturationSum / count + val averageGrayError = grayErrorSum / count + val averageBrightness = brightnessValues.sum() / count + val variance = + brightnessValues.sumOf { + val delta = it - averageBrightness + delta * delta + } / count + val brightnessSpread = min(1.0, sqrt(variance) * 2.0) + val maskScore = (1.0 - min(1.0, averageSaturation * 1.4)) * 0.58 + + (1.0 - min(1.0, averageGrayError * 1.6)) * 0.24 + + brightnessSpread * 0.18 + val colorScore = min(1.0, averageSaturation * 1.3) * 0.68 + + min(1.0, averageGrayError * 1.3) * 0.22 + + brightnessSpread * 0.10 + return GiftMp4FrameStats(maskScore, colorScore) } } } + +private data class GiftMp4ResolvedSource( + val file: File, +) { + fun setMediaPlayerDataSource(player: MediaPlayer) { + player.setDataSource(file.absolutePath) + } + + fun setRetrieverDataSource(retriever: MediaMetadataRetriever) { + retriever.setDataSource(file.absolutePath) + } + + fun readBytes(): ByteArray? = + runCatching { + file.readBytes() + }.getOrNull() + + fun close() = Unit + + companion object { + fun resolve(context: Context, sourceType: String, source: String): GiftMp4ResolvedSource? { + if (source.isBlank()) return null + return when (sourceType) { + "asset" -> { + val assetKey = FlutterInjector + .instance() + .flutterLoader() + .getLookupKeyForAsset(source) + val extension = source.substringAfterLast('.', "mp4").lowercase() + val cacheFile = File(context.cacheDir, "gift_mp4_asset_${source.stableHash()}.$extension") + if (!cacheFile.exists() || cacheFile.length() <= 0) { + context.assets.open(assetKey).use { input -> + FileOutputStream(cacheFile).use { output -> + input.copyTo(output) + } + } + } + GiftMp4ResolvedSource(cacheFile) + } + else -> { + val file = File(source.removePrefix("file://")) + file.takeIf { it.exists() }?.let { GiftMp4ResolvedSource(it) } + } + } + } + } +} + +private data class GiftMp4LayoutCandidate( + val colorRect: GiftMp4Rect, + val alphaRect: GiftMp4Rect, +) + +private data class GiftMp4CandidateScore( + val candidate: GiftMp4LayoutCandidate, + val score: Double, +) + +private data class GiftMp4FrameStats( + val maskScore: Double, + val colorScore: Double, +) + +private fun contentViewport(width: Int, height: Int, aspectRatio: Float): GiftMp4Viewport { + if (width <= 0 || height <= 0 || aspectRatio <= 0f) { + return GiftMp4Viewport(0, 0, width, height) + } + val viewAspect = width.toFloat() / height + return if (viewAspect > aspectRatio) { + val contentWidth = (height * aspectRatio).toInt().coerceAtLeast(1) + GiftMp4Viewport((width - contentWidth) / 2, 0, contentWidth, height) + } else { + val contentHeight = (width / aspectRatio).toInt().coerceAtLeast(1) + GiftMp4Viewport(0, (height - contentHeight) / 2, width, contentHeight) + } +} + +private fun createProgram(vertexSource: String, fragmentSource: String): Int { + val vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource) + val fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource) + val program = GLES20.glCreateProgram() + GLES20.glAttachShader(program, vertexShader) + GLES20.glAttachShader(program, fragmentShader) + GLES20.glLinkProgram(program) + GLES20.glDeleteShader(vertexShader) + GLES20.glDeleteShader(fragmentShader) + return program +} + +private fun loadShader(type: Int, source: String): Int { + val shader = GLES20.glCreateShader(type) + GLES20.glShaderSource(shader, source) + GLES20.glCompileShader(shader) + return shader +} + +private fun floatBufferOf(vararg values: Float) = + ByteBuffer + .allocateDirect(values.size * 4) + .order(ByteOrder.nativeOrder()) + .asFloatBuffer() + .apply { + put(values) + position(0) + } + +private fun ByteArray.indexOf(pattern: ByteArray, start: Int): Int { + if (pattern.isEmpty() || size < pattern.size) return -1 + for (index in start..(size - pattern.size)) { + var matches = true + for (patternIndex in pattern.indices) { + if (this[index + patternIndex] != pattern[patternIndex]) { + matches = false + break + } + } + if (matches) return index + } + return -1 +} + +private fun String.stableHash(): String { + var hash = -0x340d631b7bdddcdbL + for (char in this) { + hash = hash xor char.code.toLong() + hash *= 0x100000001b3L + } + return hash.toULong().toString(16) +} diff --git a/lib/modules/room_game/views/baishun_game_page.dart b/lib/modules/room_game/views/baishun_game_page.dart index 6ef9b32..00c1a13 100644 --- a/lib/modules/room_game/views/baishun_game_page.dart +++ b/lib/modules/room_game/views/baishun_game_page.dart @@ -7,7 +7,6 @@ import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:flutter_screenutil/flutter_screenutil.dart'; import 'package:webview_flutter/webview_flutter.dart'; -import 'package:webview_flutter_android/webview_flutter_android.dart'; import 'package:yumi/app/routes/sc_fluro_navigator.dart'; import 'package:yumi/modules/room_game/bridge/baishun_js_bridge.dart'; import 'package:yumi/modules/room_game/data/models/room_game_models.dart'; @@ -51,7 +50,6 @@ class _BaishunGamePageState extends State { bool _didReceiveBridgeMessage = false; bool _didFinishPageLoad = false; bool _hasDeliveredLaunchConfig = false; - bool _shouldMountWebView = defaultTargetPlatform != TargetPlatform.android; int _bridgeInjectCount = 0; String? _errorMessage; @@ -107,7 +105,6 @@ class _BaishunGamePageState extends State { onPageFinished: (String url) async { _didFinishPageLoad = true; _log('page_finished url=${_clip(url, 240)}'); - _showWebView(reason: 'page_finished'); await _injectBridge(reason: 'page_finished'); }, onWebResourceError: (WebResourceError error) { @@ -145,7 +142,6 @@ class _BaishunGamePageState extends State { _didReceiveBridgeMessage = false; _didFinishPageLoad = false; _hasDeliveredLaunchConfig = false; - _shouldMountWebView = defaultTargetPlatform != TargetPlatform.android; _bridgeInjectCount = 0; _stopBridgeBootstrap(reason: 'prepare_page_load'); _bridgeBootstrapTimer = Timer.periodic(const Duration(milliseconds: 250), ( @@ -177,7 +173,6 @@ class _BaishunGamePageState extends State { return; } _log('loading_fallback_fire isLoading=$_isLoading'); - _showWebView(reason: 'loading_fallback'); setState(() { _isLoading = false; }); @@ -331,7 +326,6 @@ class _BaishunGamePageState extends State { return; } _log('game_loaded received'); - _showWebView(reason: 'game_loaded'); setState(() { _isLoading = false; _errorMessage = null; @@ -363,16 +357,6 @@ class _BaishunGamePageState extends State { } } - void _showWebView({required String reason}) { - if (_shouldMountWebView || !mounted) { - return; - } - _log('mount_webview reason=$reason'); - setState(() { - _shouldMountWebView = true; - }); - } - Future _reload() async { if (!mounted) { return; @@ -651,26 +635,6 @@ class _BaishunGamePageState extends State { return screenSize.width / ratio; } - Widget _buildWebView() { - final params = PlatformWebViewWidgetCreationParams( - controller: _controller.platform, - gestureRecognizers: _webGestureRecognizers, - ); - if (defaultTargetPlatform == TargetPlatform.android) { - return WebViewWidget.fromPlatformCreationParams( - params: - AndroidWebViewWidgetCreationParams.fromPlatformWebViewWidgetCreationParams( - params, - displayWithHybridComposition: true, - ), - ); - } - return WebViewWidget( - controller: _controller, - gestureRecognizers: _webGestureRecognizers, - ); - } - @override Widget build(BuildContext context) { final topCrop = 28.w; @@ -700,14 +664,16 @@ class _BaishunGamePageState extends State { color: const Color(0xFF081915), child: Stack( children: [ - if (_shouldMountWebView) - Positioned( - top: -topCrop, - left: 0, - right: 0, - bottom: bottomFrameHeight, - child: _buildWebView(), + Positioned( + top: -topCrop, + left: 0, + right: 0, + bottom: bottomFrameHeight, + child: WebViewWidget( + controller: _controller, + gestureRecognizers: _webGestureRecognizers, ), + ), if (bottomFrameHeight > 0) Positioned( left: 0, diff --git a/lib/modules/room_game/views/leader_game_page.dart b/lib/modules/room_game/views/leader_game_page.dart index d46237b..4fc1275 100644 --- a/lib/modules/room_game/views/leader_game_page.dart +++ b/lib/modules/room_game/views/leader_game_page.dart @@ -6,7 +6,6 @@ import 'package:flutter/gestures.dart'; import 'package:flutter/material.dart'; import 'package:flutter_screenutil/flutter_screenutil.dart'; import 'package:webview_flutter/webview_flutter.dart'; -import 'package:webview_flutter_android/webview_flutter_android.dart'; import 'package:yumi/app/routes/sc_fluro_navigator.dart'; import 'package:yumi/modules/room_game/bridge/leader_js_bridge.dart'; import 'package:yumi/modules/room_game/data/models/room_game_models.dart'; @@ -51,7 +50,6 @@ class _LeaderGamePageState extends State { bool _isClosing = false; bool _didReceiveBridgeMessage = false; bool _didFinishPageLoad = false; - bool _shouldMountWebView = defaultTargetPlatform != TargetPlatform.android; int _bridgeInjectCount = 0; String? _errorMessage; @@ -107,7 +105,6 @@ class _LeaderGamePageState extends State { onPageFinished: (String url) async { _didFinishPageLoad = true; _log('page_finished url=${_clip(url, 240)}'); - _showWebView(reason: 'page_finished'); await _injectBridge(reason: 'page_finished'); }, onWebResourceError: (WebResourceError error) { @@ -139,7 +136,6 @@ class _LeaderGamePageState extends State { void _prepareForPageLoad() { _didReceiveBridgeMessage = false; _didFinishPageLoad = false; - _shouldMountWebView = defaultTargetPlatform != TargetPlatform.android; _bridgeInjectCount = 0; _stopBridgeBootstrap(reason: 'prepare_page_load'); _bridgeBootstrapTimer = Timer.periodic(const Duration(milliseconds: 250), ( @@ -159,7 +155,6 @@ class _LeaderGamePageState extends State { return; } _log('loading_fallback_fire'); - _showWebView(reason: 'loading_fallback'); setState(() { _isLoading = false; }); @@ -279,7 +274,6 @@ class _LeaderGamePageState extends State { if (!mounted) { return; } - _showWebView(reason: 'load_complete'); setState(() { _isLoading = false; _errorMessage = null; @@ -300,16 +294,6 @@ class _LeaderGamePageState extends State { } } - void _showWebView({required String reason}) { - if (_shouldMountWebView || !mounted) { - return; - } - _log('mount_webview reason=$reason'); - setState(() { - _shouldMountWebView = true; - }); - } - Future _reload() async { if (!mounted) { return; @@ -401,26 +385,6 @@ class _LeaderGamePageState extends State { return fallback; } - Widget _buildWebView() { - final params = PlatformWebViewWidgetCreationParams( - controller: _controller.platform, - gestureRecognizers: _webGestureRecognizers, - ); - if (defaultTargetPlatform == TargetPlatform.android) { - return WebViewWidget.fromPlatformCreationParams( - params: - AndroidWebViewWidgetCreationParams.fromPlatformWebViewWidgetCreationParams( - params, - displayWithHybridComposition: true, - ), - ); - } - return WebViewWidget( - controller: _controller, - gestureRecognizers: _webGestureRecognizers, - ); - } - void _log(String message) { if (kDebugMode) { debugPrint('$_logPrefix $message'); @@ -606,14 +570,16 @@ class _LeaderGamePageState extends State { color: const Color(0xFF081915), child: Stack( children: [ - if (_shouldMountWebView) - Positioned( - left: 0, - right: 0, - top: 0, - bottom: bottomFrameHeight, - child: _buildWebView(), + Positioned( + left: 0, + right: 0, + top: 0, + bottom: bottomFrameHeight, + child: WebViewWidget( + controller: _controller, + gestureRecognizers: _webGestureRecognizers, ), + ), if (bottomFrameHeight > 0) Positioned( left: 0, diff --git a/lib/services/auth/authentication_manager.dart b/lib/services/auth/authentication_manager.dart index 47fd382..853725d 100644 --- a/lib/services/auth/authentication_manager.dart +++ b/lib/services/auth/authentication_manager.dart @@ -1,6 +1,7 @@ import 'package:firebase_auth/firebase_auth.dart'; import 'package:dio/dio.dart'; import 'package:flutter/cupertino.dart'; +import 'package:flutter/foundation.dart'; import 'package:sign_in_with_apple/sign_in_with_apple.dart'; import 'package:yumi/app/routes/sc_routes.dart'; import 'package:yumi/app/routes/sc_fluro_navigator.dart'; @@ -104,49 +105,60 @@ class SocialChatAuthenticationManager extends ChangeNotifier { } else if (authType == SCAuthType.APPLE.name) { this.authType = authType; try { + _authDebugLog('apple sign-in start'); + final isAvailable = await SignInWithApple.isAvailable(); + _authDebugLog('apple sign-in availability=$isAvailable'); + if (!isAvailable) { + SCTts.show("Sign in with Apple is not available on this device."); + return; + } final appleCredential = await SignInWithApple.getAppleIDCredential( scopes: [ + AppleIDAuthorizationScopes.email, AppleIDAuthorizationScopes.fullName, // 请求姓名权限 ], ); uid = appleCredential.userIdentifier ?? ""; - SCLoadingManager.show(); - if (uid.isNotEmpty) { - await DataPersistence.setPendingChannelAuth(authType, uid); - var user = await SCAccountRepository().loginForChannel(authType, uid); - if (!context.mounted) { - SCLoadingManager.hide(); - return; - } - AccountStorage().setCurrentUser(user); - await DataPersistence.clearPendingChannelAuth(); - if (!context.mounted) { - SCLoadingManager.hide(); - return; - } - final canContinue = await SCVersionUtils.checkReview( - context: context, - ); - if (!context.mounted || !canContinue) { - SCLoadingManager.hide(); - return; - } - await Provider.of( - context, - listen: false, - ).resetLocalRoomState( - fallbackRtmProvider: Provider.of( - context, - listen: false, - ), - ); - if (!context.mounted) { - return; - } - SCLoadingManager.hide(); - notifyListeners(); - SCNavigatorUtils.push(context, SCRoutes.home, replace: true); + _authDebugLog( + 'apple credential success userIdentifier=${_maskForDebug(uid)} ' + 'hasAuthorizationCode=${appleCredential.authorizationCode.isNotEmpty} ' + 'hasIdentityToken=${(appleCredential.identityToken ?? "").isNotEmpty} ' + 'hasEmail=${(appleCredential.email ?? "").isNotEmpty}', + ); + if (uid.isEmpty) { + SCTts.show("Apple authorization failed. Please try again."); + return; } + SCLoadingManager.show(); + await DataPersistence.setPendingChannelAuth(authType, uid); + var user = await SCAccountRepository().loginForChannel(authType, uid); + if (!context.mounted) { + SCLoadingManager.hide(); + return; + } + AccountStorage().setCurrentUser(user); + await DataPersistence.clearPendingChannelAuth(); + if (!context.mounted) { + SCLoadingManager.hide(); + return; + } + final canContinue = await SCVersionUtils.checkReview(context: context); + if (!context.mounted || !canContinue) { + SCLoadingManager.hide(); + return; + } + await Provider.of( + context, + listen: false, + ).resetLocalRoomState( + fallbackRtmProvider: Provider.of(context, listen: false), + ); + if (!context.mounted) { + return; + } + SCLoadingManager.hide(); + notifyListeners(); + SCNavigatorUtils.push(context, SCRoutes.home, replace: true); } catch (e) { SCLoadingManager.hide(); _showAuthError(e); @@ -162,13 +174,56 @@ class SocialChatAuthenticationManager extends ChangeNotifier { } void _showAuthError(Object error) { + _authDebugLog('authenticateUser error: $error'); + if (error is DioException) { return; } + if (error is SignInWithAppleAuthorizationException) { + _showAppleAuthorizationError(error); + return; + } + + if (error is SignInWithAppleNotSupportedException) { + SCTts.show("Sign in with Apple is not available on this device."); + return; + } + final message = error.toString().replaceFirst("Exception: ", "").trim(); if (message.isNotEmpty) { SCTts.show(message); } } + + void _showAppleAuthorizationError( + SignInWithAppleAuthorizationException error, + ) { + if (error.code == AuthorizationErrorCode.canceled) { + return; + } + if (error.code == AuthorizationErrorCode.unknown) { + SCTts.show( + "Apple sign-in is not available. Please check the app signing profile.", + ); + return; + } + SCTts.show("Apple authorization failed. Please try again."); + } + + void _authDebugLog(String message) { + if (kDebugMode) { + debugPrint('[Auth][Apple] $message'); + } + } + + String _maskForDebug(String value) { + if (value.isEmpty) { + return ''; + } + if (value.length <= 8) { + return '***'; + } + return '${value.substring(0, 4)}...${value.substring(value.length - 4)}'; + } } diff --git a/lib/shared/data_sources/sources/repositories/sc_user_repository_impl.dart b/lib/shared/data_sources/sources/repositories/sc_user_repository_impl.dart index 0137aa1..067e704 100644 --- a/lib/shared/data_sources/sources/repositories/sc_user_repository_impl.dart +++ b/lib/shared/data_sources/sources/repositories/sc_user_repository_impl.dart @@ -72,6 +72,45 @@ class SCAccountRepository implements SocialChatUserRepository { } } + void _authDebugLog(String message) { + if (kDebugMode) { + debugPrint('[Auth][API] $message'); + } + } + + Map _sanitizeAuthPayload(Map payload) { + return payload.map((key, value) { + final lowerKey = key.toLowerCase(); + if (lowerKey.contains('openid') || + lowerKey.contains('token') || + lowerKey.contains('sign') || + lowerKey.contains('pwd')) { + return MapEntry(key, _maskForDebug(value?.toString() ?? '')); + } + return MapEntry(key, value); + }); + } + + String _maskForDebug(String value) { + if (value.isEmpty) { + return ''; + } + if (value.length <= 8) { + return '***'; + } + return '${value.substring(0, 4)}...${value.substring(value.length - 4)}'; + } + + String _loginResultSummary(SocialChatLoginRes result) { + return jsonEncode({ + 'hasToken': (result.token ?? '').isNotEmpty, + 'hasUserSig': (result.userSig ?? '').isNotEmpty, + 'userId': result.userProfile?.id, + 'account': result.userProfile?.account, + 'nickname': result.userProfile?.userNickname, + }); + } + String _cpPairSummary(CPRes pair) { return '{relationType=${pair.relationType}, status=${pair.status}, ' 'me=${pair.meUserId}/${pair.meAccount}, ' @@ -198,11 +237,17 @@ class SCAccountRepository implements SocialChatUserRepository { ) async { final data = {"authType": authType, "openId": openId}; data.addAll(await SCMobileLoginContext.loginBodyFields()); + _authDebugLog( + 'request POST /auth/account/login/channel params=${jsonEncode(_sanitizeAuthPayload(data))}', + ); final result = await http.post( "e64f27b9ba6b37881120f4584a5444a5531a33eb137749a5decad584f58aaa44", data: data, fromJson: (json) => SocialChatLoginRes.fromJson(json), ); + _authDebugLog( + 'response POST /auth/account/login/channel result=${_loginResultSummary(result)}', + ); return result; } diff --git a/lib/shared/tools/sc_lk_dialog_util.dart b/lib/shared/tools/sc_lk_dialog_util.dart index 17d88bd..a59b6f3 100644 --- a/lib/shared/tools/sc_lk_dialog_util.dart +++ b/lib/shared/tools/sc_lk_dialog_util.dart @@ -186,7 +186,7 @@ showBottomInBottomDialog( Color? barrierColor, bool barrierDismissible = true, }) { - showGeneralDialog( + return showGeneralDialog( context: context, barrierLabel: '', barrierDismissible: barrierDismissible, diff --git a/lib/ui_kit/widgets/room/effect/gift_mp4_effect_view.dart b/lib/ui_kit/widgets/room/effect/gift_mp4_effect_view.dart index 653caa8..586f6f7 100644 --- a/lib/ui_kit/widgets/room/effect/gift_mp4_effect_view.dart +++ b/lib/ui_kit/widgets/room/effect/gift_mp4_effect_view.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'package:flutter/foundation.dart'; @@ -17,12 +18,14 @@ class SCGiftMp4PlayRequest { required this.sourceType, required this.token, required this.muted, + this.vapLayout, }); final String path; final SCGiftMp4SourceType sourceType; final int token; final bool muted; + final Map? vapLayout; } class SCGiftMp4Controller extends ChangeNotifier { @@ -94,10 +97,10 @@ class SCGiftMp4Controller extends ChangeNotifier { } catch (_) {} } - Future playAsset(String path) => + Future playAsset(String path) async => _play(path: path, sourceType: SCGiftMp4SourceType.asset); - Future playFile(String path) => + Future playFile(String path) async => _play(path: path, sourceType: SCGiftMp4SourceType.file); Future _play({ @@ -110,6 +113,10 @@ class SCGiftMp4Controller extends ChangeNotifier { ); return; } + final layout = await _parseVapLayout(path: path, sourceType: sourceType); + if (_disposed) { + return; + } _unregisterActiveView(); _activeViewId = null; _request = SCGiftMp4PlayRequest( @@ -117,6 +124,7 @@ class SCGiftMp4Controller extends ChangeNotifier { sourceType: sourceType, token: ++_nextToken, muted: _muted, + vapLayout: layout, ); debugPrint( '[GiftFx][mp4] request token=$_nextToken source=${sourceType.name} path=$path', @@ -266,16 +274,21 @@ class _GiftMp4EffectViewState extends State { return const SizedBox.shrink(); } if (!kIsWeb && defaultTargetPlatform == TargetPlatform.android) { + final creationParams = { + 'path': request.path, + 'sourceType': request.sourceType.name, + 'token': request.token, + 'muted': request.muted, + 'renderMode': 'auto', + }; + final vapLayout = request.vapLayout; + if (vapLayout != null) { + creationParams['vapLayout'] = vapLayout; + } return AndroidView( key: ValueKey('gift-mp4-${request.token}-${request.path}'), viewType: 'gift_mp4_video_view', - creationParams: { - 'path': request.path, - 'sourceType': request.sourceType.name, - 'token': request.token, - 'muted': request.muted, - 'renderMode': 'auto', - }, + creationParams: creationParams, creationParamsCodec: const StandardMessageCodec(), onPlatformViewCreated: (viewId) => @@ -283,15 +296,20 @@ class _GiftMp4EffectViewState extends State { ); } if (!kIsWeb && defaultTargetPlatform == TargetPlatform.iOS) { + final creationParams = { + 'path': request.path, + 'sourceType': request.sourceType.name, + 'token': request.token, + 'muted': request.muted, + }; + final vapLayout = request.vapLayout; + if (vapLayout != null) { + creationParams['vapLayout'] = vapLayout; + } return UiKitView( key: ValueKey('gift-mp4-${request.token}-${request.path}'), viewType: 'gift_mp4_video_view', - creationParams: { - 'path': request.path, - 'sourceType': request.sourceType.name, - 'token': request.token, - 'muted': request.muted, - }, + creationParams: creationParams, creationParamsCodec: const StandardMessageCodec(), onPlatformViewCreated: (viewId) => @@ -313,6 +331,176 @@ class _GiftMp4EffectViewState extends State { } } +Future?> _parseVapLayout({ + required String path, + required SCGiftMp4SourceType sourceType, +}) async { + try { + final bytes = + sourceType == SCGiftMp4SourceType.asset + ? (await rootBundle.load(path)).buffer.asUint8List() + : await File(path.removePrefix('file://')).readAsBytes(); + final jsonText = _extractVapcJson(bytes); + if (jsonText == null) { + return null; + } + final root = jsonDecode(jsonText); + if (root is! Map) { + return null; + } + final info = root['info']; + if (info is! Map) { + return null; + } + final videoWidth = + _readPositiveInt(info['videoW']) ?? + _readPositiveInt(info['video_width']); + final videoHeight = + _readPositiveInt(info['videoH']) ?? + _readPositiveInt(info['video_height']); + if (videoWidth == null || videoHeight == null) { + return null; + } + final color = _readFrame(info['rgbFrame']) ?? _readFrame(info['rgb_frame']); + final alpha = _readFrame(info['aFrame']) ?? _readFrame(info['a_frame']); + if (color == null || alpha == null) { + return null; + } + final contentWidth = _readPositiveInt(info['w']) ?? videoWidth; + final contentHeight = _readPositiveInt(info['h']) ?? videoHeight; + final layout = { + 'colorRect': _normalizeFrame(color, videoWidth, videoHeight), + 'alphaRect': _normalizeFrame(alpha, videoWidth, videoHeight), + 'hasAlphaMask': true, + 'contentAspectRatio': contentWidth / contentHeight, + }; + debugPrint('[GiftFx][mp4] parsed vapLayout path=$path layout=$layout'); + return layout; + } catch (error) { + debugPrint('[GiftFx][mp4] parse vapLayout failed path=$path error=$error'); + return null; + } +} + +String? _extractVapcJson(Uint8List bytes) { + const marker = [0x76, 0x61, 0x70, 0x63]; // vapc + final markerIndex = _indexOfBytes(bytes, marker, 0); + if (markerIndex < 0) { + return null; + } + final searchEnd = + (markerIndex + marker.length + 512 * 1024).clamp(0, bytes.length).toInt(); + for (var index = markerIndex + marker.length; index < searchEnd; index++) { + if (bytes[index] != 0x7b) { + continue; + } + final end = _findJsonObjectEnd(bytes, index, searchEnd); + if (end > index) { + return utf8.decode(bytes.sublist(index, end + 1)); + } + } + return null; +} + +int _indexOfBytes(Uint8List bytes, List pattern, int start) { + if (pattern.isEmpty || bytes.length < pattern.length) { + return -1; + } + for (var index = start; index <= bytes.length - pattern.length; index++) { + var matches = true; + for (var patternIndex = 0; patternIndex < pattern.length; patternIndex++) { + if (bytes[index + patternIndex] != pattern[patternIndex]) { + matches = false; + break; + } + } + if (matches) { + return index; + } + } + return -1; +} + +int _findJsonObjectEnd(Uint8List bytes, int start, int maxEnd) { + var depth = 0; + var inString = false; + var escaping = false; + for (var index = start; index < maxEnd; index++) { + final byte = bytes[index]; + if (inString) { + if (escaping) { + escaping = false; + } else if (byte == 0x5c) { + escaping = true; + } else if (byte == 0x22) { + inString = false; + } + } else if (byte == 0x22) { + inString = true; + } else if (byte == 0x7b) { + depth += 1; + } else if (byte == 0x7d) { + depth -= 1; + if (depth == 0) { + return index; + } + } + } + return -1; +} + +int? _readPositiveInt(Object? value) { + final resolved = + value is num + ? value.toInt() + : value is String + ? int.tryParse(value.trim()) + : null; + return resolved != null && resolved > 0 ? resolved : null; +} + +List? _readFrame(Object? value) { + final values = + value is List + ? value.map(_readPositiveInt).toList() + : value is String + ? value.split(',').map((item) => int.tryParse(item.trim())).toList() + : null; + if (values == null || + values.length < 4 || + values[0] == null || + values[1] == null || + values[2] == null || + values[3] == null || + values[2]! <= 0 || + values[3]! <= 0) { + return null; + } + return [values[0]!, values[1]!, values[2]!, values[3]!]; +} + +List _normalizeFrame( + List values, + int videoWidth, + int videoHeight, +) { + return [ + values[0] / videoWidth, + values[1] / videoHeight, + values[2] / videoWidth, + values[3] / videoHeight, + ]; +} + +extension on String { + String removePrefix(String prefix) { + if (startsWith(prefix)) { + return substring(prefix.length); + } + return this; + } +} + class _FallbackMp4EffectVideo extends StatefulWidget { const _FallbackMp4EffectVideo({ required this.request, diff --git a/pubspec.lock b/pubspec.lock index 4e0297e..1659351 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -1781,7 +1781,7 @@ packages: source: hosted version: "4.4.2" webview_flutter_android: - dependency: "direct main" + dependency: transitive description: name: webview_flutter_android sha256: "47a8da40d02befda5b151a26dba71f47df471cddd91dfdb7802d0a87c5442558" diff --git a/pubspec.yaml b/pubspec.yaml index 98fee40..655a755 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -55,10 +55,9 @@ dependencies: flutter_cache_manager: ^3.0.0 #路由 fluro: 2.0.5 - #权限请求 + #权限请求 permission_handler: ^12.0.0+1 webview_flutter: 4.4.2 - webview_flutter_android: 3.16.9 #跳转外链 url_launcher: ^6.3.1 #键值对存储