diff --git a/android/app/src/main/kotlin/com/org/yumiparty/GiftMp4VideoPlatformView.kt b/android/app/src/main/kotlin/com/org/yumiparty/GiftMp4VideoPlatformView.kt index de8b534..6878569 100644 --- a/android/app/src/main/kotlin/com/org/yumiparty/GiftMp4VideoPlatformView.kt +++ b/android/app/src/main/kotlin/com/org/yumiparty/GiftMp4VideoPlatformView.kt @@ -237,6 +237,7 @@ class GiftMp4VideoPlatformView( private data class HalfStats( val saturation: Double, val grayError: Double, + val brightness: Double, ) private enum class SplitOrientation { HORIZONTAL, VERTICAL } @@ -265,6 +266,11 @@ class GiftMp4VideoPlatformView( const val VIEW_TYPE = "gift_mp4_video_view" const val CHANNEL_NAME = "com.org.yumiparty/gift_mp4_events" const val CONTROL_CHANNEL_NAME = "com.org.yumiparty/gift_mp4_control" + private const val MIN_SPLIT_MATCH_FRAMES = 3 + private const val MIN_USEFUL_FRAME_BRIGHTNESS = 0.035 + private const val MIN_SPLIT_SATURATION_GAP = 0.08 + private const val MAX_SPLIT_MASK_SATURATION = 0.28 + private const val MAX_SPLIT_MASK_GRAY_ERROR = 0.06 private val registry = mutableMapOf() private val layoutCache = mutableMapOf() @@ -512,10 +518,7 @@ class GiftMp4VideoPlatformView( .extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION) ?.toLongOrNull() ?: 0L - val timesUs = - listOf(0L, 500_000L, 1_000_000L, 2_000_000L) - .filter { durationMs == 0L || it <= durationMs * 1000L } - .ifEmpty { listOf(0L) } + val timesUs = splitSampleTimesUs(durationMs) val horizontalScore = SplitScore() val verticalScore = SplitScore() @@ -523,13 +526,16 @@ class GiftMp4VideoPlatformView( val frame = retriever.getFrameAtTime( timeUs, - MediaMetadataRetriever.OPTION_CLOSEST_SYNC, + MediaMetadataRetriever.OPTION_CLOSEST, ) ?: continue - val horizontalStats = statsByHorizontalHalves(frame) - val verticalStats = statsByVerticalHalves(frame) - frame.recycle() - addSplitScore(horizontalScore, horizontalStats.first, horizontalStats.second) - addSplitScore(verticalScore, verticalStats.first, verticalStats.second) + try { + val horizontalStats = statsByHorizontalHalves(frame) + val verticalStats = statsByVerticalHalves(frame) + addSplitScore(horizontalScore, horizontalStats.first, horizontalStats.second) + addSplitScore(verticalScore, verticalStats.first, verticalStats.second) + } finally { + frame.recycle() + } } val horizontalCandidate = splitCandidateFromScore(SplitOrientation.HORIZONTAL, horizontalScore) @@ -545,6 +551,32 @@ class GiftMp4VideoPlatformView( } } + private fun splitSampleTimesUs(durationMs: Long): List { + val durationUs = durationMs * 1000L + val desiredTimesUs = + listOf( + 300_000L, + 600_000L, + 900_000L, + 1_200_000L, + 1_500_000L, + 1_800_000L, + 2_100_000L, + 2_400_000L, + 2_700_000L, + 3_000_000L, + ) + return desiredTimesUs + .filter { durationMs <= 0L || it <= durationUs } + .ifEmpty { + if (durationUs > 0L) { + listOf(durationUs / 2L) + } else { + listOf(0L) + } + } + } + private fun detectPackedAlphaTopRightSourceLayout( source: GiftMp4Source, ): SourceLayout? { @@ -732,25 +764,40 @@ class GiftMp4VideoPlatformView( first: HalfStats, second: HalfStats, ) { + if (isLowBrightnessFrame(first, second)) { + return + } val alphaStats = if (first.saturation < second.saturation) first else second + val saturationGap = kotlin.math.abs(second.saturation - first.saturation) + if (saturationGap < MIN_SPLIT_SATURATION_GAP || + alphaStats.saturation > MAX_SPLIT_MASK_SATURATION || + alphaStats.grayError > MAX_SPLIT_MASK_GRAY_ERROR + ) { + return + } score.direction += second.saturation - first.saturation - score.saturationGap += kotlin.math.abs(second.saturation - first.saturation) + score.saturationGap += saturationGap score.maskSaturation += alphaStats.saturation score.maskGrayError += alphaStats.grayError score.frames++ } + private fun isLowBrightnessFrame( + first: HalfStats, + second: HalfStats, + ): Boolean = maxOf(first.brightness, second.brightness) < MIN_USEFUL_FRAME_BRIGHTNESS + private fun splitCandidateFromScore( orientation: SplitOrientation, score: SplitScore, ): Pair? { - if (score.frames == 0) return null + if (score.frames < MIN_SPLIT_MATCH_FRAMES) return null val averageSaturationGap = score.saturationGap / score.frames val averageMaskSaturation = score.maskSaturation / score.frames val averageMaskGrayError = score.maskGrayError / score.frames - if (averageSaturationGap < 0.08 || - averageMaskSaturation > 0.28 || - averageMaskGrayError > 0.06 + if (averageSaturationGap < MIN_SPLIT_SATURATION_GAP || + averageMaskSaturation > MAX_SPLIT_MASK_SATURATION || + averageMaskGrayError > MAX_SPLIT_MASK_GRAY_ERROR ) { return null } @@ -783,6 +830,7 @@ class GiftMp4VideoPlatformView( val stepY = maxOf(1, (endY - startY) / 120) var saturationSum = 0.0 var grayErrorSum = 0.0 + var brightnessSum = 0.0 var count = 0 var y = startY while (y < endY) { @@ -800,15 +848,19 @@ class GiftMp4VideoPlatformView( (kotlin.math.abs(red - green) + kotlin.math.abs(green - blue) + kotlin.math.abs(blue - red)) / 3.0 + brightnessSum += red * 0.299 + green * 0.587 + blue * 0.114 count++ x += stepX } y += stepY } - if (count == 0) return HalfStats(saturation = 0.0, grayError = 0.0) + if (count == 0) { + return HalfStats(saturation = 0.0, grayError = 0.0, brightness = 0.0) + } return HalfStats( saturation = saturationSum / count, grayError = grayErrorSum / count, + brightness = brightnessSum / count, ) } } diff --git a/ios/Runner/GiftMp4VideoPlatformView.swift b/ios/Runner/GiftMp4VideoPlatformView.swift index 6c6b947..39ab217 100644 --- a/ios/Runner/GiftMp4VideoPlatformView.swift +++ b/ios/Runner/GiftMp4VideoPlatformView.swift @@ -482,6 +482,7 @@ private struct GiftMp4SplitAlphaCandidate { private struct GiftMp4HalfStats { let saturation: Double let grayError: Double + let brightness: Double } private struct GiftMp4SplitScore { @@ -500,6 +501,12 @@ private struct GiftMp4PackedAlphaCandidate { } private final class GiftMp4SourceLayoutResolver { + private static let minSplitMatchFrames = 3 + private static let minUsefulFrameBrightness = 0.035 + private static let minSplitSaturationGap = 0.08 + private static let maxSplitMaskSaturation = 0.28 + private static let maxSplitMaskGrayError = 0.06 + private static var layoutCache: [String: GiftMp4SourceLayout] = [:] static func resolve( @@ -888,7 +895,7 @@ private final class GiftMp4SourceLayoutResolver { generator.requestedTimeToleranceAfter = .positiveInfinity let durationSeconds = CMTimeGetSeconds(asset.duration) - let desiredSeconds = [0.0, 0.5, 1.0, 2.0, 3.0] + let desiredSeconds = [0.3, 0.6, 0.9, 1.2, 1.5, 1.8, 2.1, 2.4, 2.7, 3.0] .filter { durationSeconds.isNaN || durationSeconds <= 0 || $0 <= durationSeconds } let times = (desiredSeconds.isEmpty ? [0.0] : desiredSeconds) .map { NSValue(time: CMTime(seconds: $0, preferredTimescale: 600)) } @@ -950,6 +957,7 @@ private final class GiftMp4SourceLayoutResolver { let stepY = max(1, height / 32) var saturationSum = 0.0 var grayErrorSum = 0.0 + var brightnessSum = 0.0 var count = 0.0 var y = startY @@ -967,8 +975,14 @@ private final class GiftMp4SourceLayoutResolver { abs(Double(pixel.g) - Double(pixel.b)) + abs(Double(pixel.b) - Double(pixel.r)) ) / (3.0 * 255.0) + let brightness = ( + Double(pixel.r) * 0.299 + + Double(pixel.g) * 0.587 + + Double(pixel.b) * 0.114 + ) / 255.0 saturationSum += saturation grayErrorSum += grayError + brightnessSum += brightness count += 1 x += stepX } @@ -976,11 +990,12 @@ private final class GiftMp4SourceLayoutResolver { } if count == 0 { - return GiftMp4HalfStats(saturation: 0, grayError: 0) + return GiftMp4HalfStats(saturation: 0, grayError: 0, brightness: 0) } return GiftMp4HalfStats( saturation: saturationSum / count, - grayError: grayErrorSum / count + grayError: grayErrorSum / count, + brightness: brightnessSum / count ) } @@ -989,19 +1004,37 @@ private final class GiftMp4SourceLayoutResolver { first: GiftMp4HalfStats, second: GiftMp4HalfStats ) { + if isLowBrightnessFrame(first: first, second: second) { + return + } let alphaStats = first.saturation < second.saturation ? first : second + let saturationGap = abs(second.saturation - first.saturation) + guard + saturationGap >= minSplitSaturationGap, + alphaStats.saturation <= maxSplitMaskSaturation, + alphaStats.grayError <= maxSplitMaskGrayError + else { + return + } score.direction += second.saturation - first.saturation - score.saturationGap += abs(second.saturation - first.saturation) + score.saturationGap += saturationGap score.maskSaturation += alphaStats.saturation score.maskGrayError += alphaStats.grayError score.frames += 1 } + private static func isLowBrightnessFrame( + first: GiftMp4HalfStats, + second: GiftMp4HalfStats + ) -> Bool { + max(first.brightness, second.brightness) < minUsefulFrameBrightness + } + private static func splitCandidateFromScore( orientation: GiftMp4SplitOrientation, score: GiftMp4SplitScore ) -> (GiftMp4SplitAlphaCandidate, Double)? { - guard score.frames > 0 else { + guard score.frames >= minSplitMatchFrames else { return nil } let frames = Double(score.frames) @@ -1010,9 +1043,9 @@ private final class GiftMp4SourceLayoutResolver { let averageMaskGrayError = score.maskGrayError / frames guard - averageSaturationGap >= 0.08, - averageMaskSaturation <= 0.28, - averageMaskGrayError <= 0.06 + averageSaturationGap >= minSplitSaturationGap, + averageMaskSaturation <= maxSplitMaskSaturation, + averageMaskGrayError <= maxSplitMaskGrayError else { return nil }