1185 lines
32 KiB
Swift
1185 lines
32 KiB
Swift
import AVFoundation
|
|
import CoreImage
|
|
import Flutter
|
|
import UIKit
|
|
|
|
final class GiftMp4VideoViewFactory: NSObject, FlutterPlatformViewFactory {
|
|
private let messenger: FlutterBinaryMessenger
|
|
|
|
init(messenger: FlutterBinaryMessenger) {
|
|
self.messenger = messenger
|
|
super.init()
|
|
}
|
|
|
|
func create(
|
|
withFrame frame: CGRect,
|
|
viewIdentifier viewId: Int64,
|
|
arguments args: Any?
|
|
) -> FlutterPlatformView {
|
|
GiftMp4VideoPlatformView(
|
|
frame: frame,
|
|
viewId: Int(viewId),
|
|
args: args as? [String: Any],
|
|
messenger: messenger
|
|
)
|
|
}
|
|
|
|
func createArgsCodec() -> FlutterMessageCodec & NSObjectProtocol {
|
|
FlutterStandardMessageCodec.sharedInstance()
|
|
}
|
|
}
|
|
|
|
final class GiftMp4VideoPlatformView: NSObject, FlutterPlatformView {
|
|
static let viewType = "gift_mp4_video_view"
|
|
static let eventChannelName = "com.org.yumiparty/gift_mp4_events"
|
|
static let controlChannelName = "com.org.yumiparty/gift_mp4_control"
|
|
|
|
private static var registry: [Int: GiftMp4VideoPlatformView] = [:]
|
|
private static var controlChannel: FlutterMethodChannel?
|
|
|
|
static func installControlChannel(messenger: FlutterBinaryMessenger) {
|
|
if controlChannel != nil {
|
|
return
|
|
}
|
|
let channel = FlutterMethodChannel(
|
|
name: controlChannelName,
|
|
binaryMessenger: messenger
|
|
)
|
|
channel.setMethodCallHandler { call, result in
|
|
let args = call.arguments as? [String: Any]
|
|
guard
|
|
let viewId = args?["viewId"] as? Int,
|
|
let view = registry[viewId]
|
|
else {
|
|
result(nil)
|
|
return
|
|
}
|
|
|
|
switch call.method {
|
|
case "setMute":
|
|
view.setMute(args?["muted"] as? Bool ?? false)
|
|
result(nil)
|
|
case "stop":
|
|
view.stop()
|
|
result(nil)
|
|
default:
|
|
result(FlutterMethodNotImplemented)
|
|
}
|
|
}
|
|
controlChannel = channel
|
|
}
|
|
|
|
private let viewId: Int
|
|
private let token: Int
|
|
private let events: FlutterMethodChannel
|
|
private let videoView = GiftMp4AlphaVideoView()
|
|
|
|
init(
|
|
frame: CGRect,
|
|
viewId: Int,
|
|
args: [String: Any]?,
|
|
messenger: FlutterBinaryMessenger
|
|
) {
|
|
self.viewId = viewId
|
|
token = args?["token"] as? Int ?? 0
|
|
events = FlutterMethodChannel(
|
|
name: Self.eventChannelName,
|
|
binaryMessenger: messenger
|
|
)
|
|
super.init()
|
|
|
|
videoView.frame = frame
|
|
videoView.onStarted = { [weak self] in
|
|
self?.postEvent("started")
|
|
}
|
|
videoView.onCompleted = { [weak self] in
|
|
self?.postEvent("completed")
|
|
}
|
|
videoView.onFailed = { [weak self] message in
|
|
self?.postEvent("failed", extra: ["message": message])
|
|
}
|
|
|
|
Self.registry[viewId] = self
|
|
|
|
let path = args?["path"] as? String ?? ""
|
|
let sourceType = args?["sourceType"] as? String ?? "file"
|
|
let muted = args?["muted"] as? Bool ?? false
|
|
let source = GiftMp4Source(path: path, sourceType: sourceType)
|
|
videoView.play(source: source, muted: muted)
|
|
}
|
|
|
|
func view() -> UIView {
|
|
videoView
|
|
}
|
|
|
|
func setMute(_ muted: Bool) {
|
|
videoView.setMute(muted)
|
|
}
|
|
|
|
func stop() {
|
|
videoView.stop()
|
|
}
|
|
|
|
func dispose() {
|
|
Self.registry.removeValue(forKey: viewId)
|
|
videoView.dispose()
|
|
}
|
|
|
|
private func postEvent(_ method: String, extra: [String: Any?] = [:]) {
|
|
var args: [String: Any?] = [
|
|
"viewId": viewId,
|
|
"token": token
|
|
]
|
|
extra.forEach { args[$0.key] = $0.value }
|
|
DispatchQueue.main.async { [events] in
|
|
events.invokeMethod(method, arguments: args)
|
|
}
|
|
}
|
|
}
|
|
|
|
private final class GiftMp4AlphaVideoView: UIView {
|
|
var onStarted: (() -> Void)?
|
|
var onCompleted: (() -> Void)?
|
|
var onFailed: ((String) -> Void)?
|
|
|
|
private let imageView = UIImageView()
|
|
private let ciContext = CIContext(options: [.cacheIntermediates: false])
|
|
private var player: AVPlayer?
|
|
private var playerItem: AVPlayerItem?
|
|
private var videoOutput: AVPlayerItemVideoOutput?
|
|
private var displayLink: CADisplayLink?
|
|
private var endObserver: NSObjectProtocol?
|
|
private var statusObservation: NSKeyValueObservation?
|
|
private var didStart = false
|
|
private var didFinish = false
|
|
private var sourceLayout = GiftMp4SourceLayout.normal(
|
|
videoSize: CGSize(width: 1, height: 1)
|
|
)
|
|
|
|
override init(frame: CGRect) {
|
|
super.init(frame: frame)
|
|
isOpaque = false
|
|
backgroundColor = .clear
|
|
imageView.isOpaque = false
|
|
imageView.backgroundColor = .clear
|
|
imageView.contentMode = .scaleAspectFit
|
|
imageView.clipsToBounds = true
|
|
addSubview(imageView)
|
|
}
|
|
|
|
required init?(coder: NSCoder) {
|
|
nil
|
|
}
|
|
|
|
override func layoutSubviews() {
|
|
super.layoutSubviews()
|
|
imageView.frame = bounds
|
|
}
|
|
|
|
func play(source: GiftMp4Source, muted: Bool) {
|
|
stopPlayback(notifyComplete: false)
|
|
didStart = false
|
|
didFinish = false
|
|
|
|
guard let url = source.fileURL() else {
|
|
fail("Unable to resolve mp4 source: \(source.path)")
|
|
return
|
|
}
|
|
|
|
let asset = AVURLAsset(url: url)
|
|
sourceLayout = GiftMp4SourceLayoutResolver.resolve(
|
|
asset: asset,
|
|
fileURL: url,
|
|
cacheKey: source.cacheKey
|
|
)
|
|
imageView.contentMode = sourceLayout.hasAlphaMask ? .scaleAspectFit : .scaleAspectFill
|
|
|
|
let item = AVPlayerItem(asset: asset)
|
|
let output = AVPlayerItemVideoOutput(pixelBufferAttributes: [
|
|
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA
|
|
])
|
|
item.add(output)
|
|
|
|
let player = AVPlayer(playerItem: item)
|
|
player.isMuted = muted
|
|
self.player = player
|
|
playerItem = item
|
|
videoOutput = output
|
|
|
|
statusObservation = item.observe(\.status, options: [.new]) {
|
|
[weak self] item, _ in
|
|
DispatchQueue.main.async {
|
|
guard let self else {
|
|
return
|
|
}
|
|
switch item.status {
|
|
case .failed:
|
|
self.fail(item.error?.localizedDescription ?? "AVPlayerItem failed")
|
|
default:
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
endObserver = NotificationCenter.default.addObserver(
|
|
forName: .AVPlayerItemDidPlayToEndTime,
|
|
object: item,
|
|
queue: .main
|
|
) { [weak self] _ in
|
|
self?.complete()
|
|
}
|
|
|
|
let link = CADisplayLink(target: self, selector: #selector(displayLinkTick))
|
|
link.add(to: .main, forMode: .common)
|
|
displayLink = link
|
|
player.play()
|
|
}
|
|
|
|
func setMute(_ muted: Bool) {
|
|
player?.isMuted = muted
|
|
}
|
|
|
|
func stop() {
|
|
stopPlayback(notifyComplete: false)
|
|
}
|
|
|
|
func dispose() {
|
|
stopPlayback(notifyComplete: false)
|
|
onStarted = nil
|
|
onCompleted = nil
|
|
onFailed = nil
|
|
}
|
|
|
|
@objc private func displayLinkTick() {
|
|
guard
|
|
let output = videoOutput,
|
|
let playerItem,
|
|
playerItem.status == .readyToPlay
|
|
else {
|
|
return
|
|
}
|
|
|
|
let itemTime = output.itemTime(forHostTime: CACurrentMediaTime())
|
|
guard output.hasNewPixelBuffer(forItemTime: itemTime) else {
|
|
return
|
|
}
|
|
var presentationTime = CMTime.zero
|
|
guard let pixelBuffer = output.copyPixelBuffer(
|
|
forItemTime: itemTime,
|
|
itemTimeForDisplay: &presentationTime
|
|
) else {
|
|
return
|
|
}
|
|
render(pixelBuffer: pixelBuffer)
|
|
markStarted()
|
|
}
|
|
|
|
private func render(pixelBuffer: CVPixelBuffer) {
|
|
let sourceImage = CIImage(cvPixelBuffer: pixelBuffer)
|
|
let renderedImage: CIImage
|
|
|
|
if sourceLayout.hasAlphaMask {
|
|
renderedImage = renderMaskedFrame(sourceImage)
|
|
} else {
|
|
renderedImage = sourceImage
|
|
}
|
|
|
|
guard let cgImage = ciContext.createCGImage(
|
|
renderedImage,
|
|
from: renderedImage.extent
|
|
) else {
|
|
return
|
|
}
|
|
imageView.image = UIImage(cgImage: cgImage)
|
|
}
|
|
|
|
private func renderMaskedFrame(_ sourceImage: CIImage) -> CIImage {
|
|
let colorCrop = cropRect(sourceLayout.colorRect, in: sourceImage.extent)
|
|
let alphaCrop = cropRect(sourceLayout.alphaRect, in: sourceImage.extent)
|
|
guard
|
|
!colorCrop.isEmpty,
|
|
!alphaCrop.isEmpty
|
|
else {
|
|
return sourceImage
|
|
}
|
|
|
|
let outputExtent = CGRect(origin: .zero, size: colorCrop.size)
|
|
let colorImage = sourceImage
|
|
.cropped(to: colorCrop)
|
|
.transformed(
|
|
by: CGAffineTransform(
|
|
translationX: -colorCrop.origin.x,
|
|
y: -colorCrop.origin.y
|
|
)
|
|
)
|
|
|
|
var maskImage = sourceImage
|
|
.cropped(to: alphaCrop)
|
|
.transformed(
|
|
by: CGAffineTransform(
|
|
translationX: -alphaCrop.origin.x,
|
|
y: -alphaCrop.origin.y
|
|
)
|
|
)
|
|
|
|
let scaleX = outputExtent.width / max(maskImage.extent.width, 1)
|
|
let scaleY = outputExtent.height / max(maskImage.extent.height, 1)
|
|
maskImage = maskImage
|
|
.transformed(by: CGAffineTransform(scaleX: scaleX, y: scaleY))
|
|
.cropped(to: outputExtent)
|
|
.applyingFilter(
|
|
"CIColorControls",
|
|
parameters: [
|
|
kCIInputSaturationKey: 0,
|
|
kCIInputBrightnessKey: 0,
|
|
kCIInputContrastKey: 1
|
|
]
|
|
)
|
|
|
|
let alphaImage = maskImage.applyingFilter("CIMaskToAlpha")
|
|
return colorImage
|
|
.cropped(to: outputExtent)
|
|
.applyingFilter(
|
|
"CISourceInCompositing",
|
|
parameters: [
|
|
kCIInputBackgroundImageKey: alphaImage
|
|
]
|
|
)
|
|
.cropped(to: outputExtent)
|
|
}
|
|
|
|
private func cropRect(_ rect: GiftMp4SourceRect, in extent: CGRect) -> CGRect {
|
|
let width = extent.width * CGFloat(rect.width)
|
|
let height = extent.height * CGFloat(rect.height)
|
|
let x = extent.minX + extent.width * CGFloat(rect.x)
|
|
let topY = extent.height * CGFloat(rect.y)
|
|
let y = extent.maxY - topY - height
|
|
return CGRect(x: x, y: y, width: width, height: height)
|
|
.intersection(extent)
|
|
.integral
|
|
}
|
|
|
|
private func markStarted() {
|
|
guard !didStart else {
|
|
return
|
|
}
|
|
didStart = true
|
|
onStarted?()
|
|
}
|
|
|
|
private func complete() {
|
|
guard !didFinish else {
|
|
return
|
|
}
|
|
didFinish = true
|
|
stopPlayback(notifyComplete: false)
|
|
onCompleted?()
|
|
}
|
|
|
|
private func fail(_ message: String) {
|
|
guard !didFinish else {
|
|
return
|
|
}
|
|
didFinish = true
|
|
stopPlayback(notifyComplete: false)
|
|
onFailed?(message)
|
|
}
|
|
|
|
private func stopPlayback(notifyComplete: Bool) {
|
|
displayLink?.invalidate()
|
|
displayLink = nil
|
|
|
|
if let endObserver {
|
|
NotificationCenter.default.removeObserver(endObserver)
|
|
self.endObserver = nil
|
|
}
|
|
statusObservation?.invalidate()
|
|
statusObservation = nil
|
|
|
|
player?.pause()
|
|
player?.replaceCurrentItem(with: nil)
|
|
player = nil
|
|
playerItem = nil
|
|
videoOutput = nil
|
|
imageView.image = nil
|
|
|
|
if notifyComplete {
|
|
complete()
|
|
}
|
|
}
|
|
}
|
|
|
|
private struct GiftMp4Source {
|
|
let path: String
|
|
let sourceType: String
|
|
|
|
var cacheKey: String {
|
|
"\(sourceType):\(path)"
|
|
}
|
|
|
|
func fileURL() -> URL? {
|
|
if sourceType == "asset" {
|
|
return assetURL()
|
|
}
|
|
return URL(fileURLWithPath: path)
|
|
}
|
|
|
|
private func assetURL() -> URL? {
|
|
let lookupKey = FlutterDartProject.lookupKey(forAsset: path)
|
|
let appFrameworkURL = Bundle.main.privateFrameworksURL?
|
|
.appendingPathComponent("App.framework")
|
|
let appFrameworkBundle = appFrameworkURL.flatMap(Bundle.init(url:))
|
|
|
|
if let url = appFrameworkBundle?.url(forResource: lookupKey, withExtension: nil) {
|
|
return url
|
|
}
|
|
if let url = Bundle.main.url(forResource: lookupKey, withExtension: nil) {
|
|
return url
|
|
}
|
|
if let url = appFrameworkURL?.appendingPathComponent(lookupKey),
|
|
FileManager.default.fileExists(atPath: url.path) {
|
|
return url
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
|
|
private struct GiftMp4SourceRect {
|
|
let x: Double
|
|
let y: Double
|
|
let width: Double
|
|
let height: Double
|
|
}
|
|
|
|
private struct GiftMp4SourceLayout {
|
|
let kind: String
|
|
let hasAlphaMask: Bool
|
|
let contentAspectRatio: Double
|
|
let colorRect: GiftMp4SourceRect
|
|
let alphaRect: GiftMp4SourceRect
|
|
|
|
static func normal(videoSize: CGSize) -> GiftMp4SourceLayout {
|
|
let rect = GiftMp4SourceRect(x: 0, y: 0, width: 1, height: 1)
|
|
let aspect = videoSize.height <= 0 ? 1 : videoSize.width / videoSize.height
|
|
return GiftMp4SourceLayout(
|
|
kind: "normal",
|
|
hasAlphaMask: false,
|
|
contentAspectRatio: aspect,
|
|
colorRect: rect,
|
|
alphaRect: rect
|
|
)
|
|
}
|
|
}
|
|
|
|
private enum GiftMp4SplitOrientation {
|
|
case horizontal
|
|
case vertical
|
|
}
|
|
|
|
private struct GiftMp4SplitAlphaCandidate {
|
|
let orientation: GiftMp4SplitOrientation
|
|
let alphaFirst: Bool
|
|
}
|
|
|
|
private struct GiftMp4HalfStats {
|
|
let saturation: Double
|
|
let grayError: Double
|
|
let brightness: Double
|
|
}
|
|
|
|
private struct GiftMp4SplitScore {
|
|
var direction = 0.0
|
|
var saturationGap = 0.0
|
|
var maskSaturation = 0.0
|
|
var maskGrayError = 0.0
|
|
var frames = 0
|
|
}
|
|
|
|
private struct GiftMp4PackedAlphaCandidate {
|
|
let frameWidth: Int
|
|
let frameHeight: Int
|
|
let alphaStartX: Int
|
|
let alphaEndX: Int
|
|
}
|
|
|
|
private final class GiftMp4SourceLayoutResolver {
|
|
private static let minSplitMatchFrames = 3
|
|
private static let minUsefulFrameBrightness = 0.035
|
|
private static let minSplitSaturationGap = 0.08
|
|
private static let maxSplitMaskSaturation = 0.28
|
|
private static let maxSplitMaskGrayError = 0.06
|
|
|
|
private static var layoutCache: [String: GiftMp4SourceLayout] = [:]
|
|
|
|
static func resolve(
|
|
asset: AVURLAsset,
|
|
fileURL: URL,
|
|
cacheKey: String
|
|
) -> GiftMp4SourceLayout {
|
|
if let cached = layoutCache[cacheKey] {
|
|
return cached
|
|
}
|
|
let videoSize = probeVideoSize(asset: asset)
|
|
let layout = parseVapcSourceLayout(fileURL: fileURL)
|
|
?? detectPackedAlphaTopRightSourceLayout(asset: asset)
|
|
?? detectSplitSourceLayout(asset: asset, videoSize: videoSize)
|
|
?? GiftMp4SourceLayout.normal(videoSize: videoSize)
|
|
#if DEBUG
|
|
NSLog("[GiftMp4Video] classified path=%@ mode=%@", fileURL.lastPathComponent, layout.kind)
|
|
#endif
|
|
layoutCache[cacheKey] = layout
|
|
trimCache()
|
|
return layout
|
|
}
|
|
|
|
private static func trimCache() {
|
|
while layoutCache.count > 64 {
|
|
guard let key = layoutCache.keys.first else {
|
|
return
|
|
}
|
|
layoutCache.removeValue(forKey: key)
|
|
}
|
|
}
|
|
|
|
private static func probeVideoSize(asset: AVAsset) -> CGSize {
|
|
guard let track = asset.tracks(withMediaType: .video).first else {
|
|
return CGSize(width: 1, height: 1)
|
|
}
|
|
let transformed = track.naturalSize.applying(track.preferredTransform)
|
|
let width = abs(transformed.width)
|
|
let height = abs(transformed.height)
|
|
if width > 0, height > 0 {
|
|
return CGSize(width: width, height: height)
|
|
}
|
|
return CGSize(
|
|
width: max(track.naturalSize.width, 1),
|
|
height: max(track.naturalSize.height, 1)
|
|
)
|
|
}
|
|
|
|
private static func parseVapcSourceLayout(fileURL: URL) -> GiftMp4SourceLayout? {
|
|
guard
|
|
let data = try? Data(contentsOf: fileURL, options: [.mappedIfSafe]),
|
|
let jsonData = extractVapcJsonData(data),
|
|
let object = try? JSONSerialization.jsonObject(with: jsonData) as? [String: Any],
|
|
let info = object["info"] as? [String: Any],
|
|
let videoWidth = doubleValue(info["videoW"]),
|
|
let videoHeight = doubleValue(info["videoH"]),
|
|
let outputWidth = doubleValue(info["w"]),
|
|
let outputHeight = doubleValue(info["h"]),
|
|
videoWidth > 0,
|
|
videoHeight > 0,
|
|
outputWidth > 0,
|
|
outputHeight > 0,
|
|
let rgbFrame = info["rgbFrame"] as? [Any],
|
|
let alphaFrame = info["aFrame"] as? [Any],
|
|
rgbFrame.count >= 4,
|
|
alphaFrame.count >= 4
|
|
else {
|
|
return nil
|
|
}
|
|
|
|
return GiftMp4SourceLayout(
|
|
kind: "vapc",
|
|
hasAlphaMask: true,
|
|
contentAspectRatio: outputWidth / outputHeight,
|
|
colorRect: normalizedRect(
|
|
values: rgbFrame,
|
|
videoWidth: videoWidth,
|
|
videoHeight: videoHeight
|
|
),
|
|
alphaRect: normalizedRect(
|
|
values: alphaFrame,
|
|
videoWidth: videoWidth,
|
|
videoHeight: videoHeight
|
|
)
|
|
)
|
|
}
|
|
|
|
private static func extractVapcJsonData(_ data: Data) -> Data? {
|
|
let marker = Array("vapc".utf8)
|
|
var searchStart = 0
|
|
while searchStart < data.count {
|
|
guard let markerIndex = indexOf(data, marker: marker, start: searchStart) else {
|
|
return nil
|
|
}
|
|
let scanEnd = min(data.count, markerIndex + 128)
|
|
var jsonStart: Int?
|
|
if markerIndex + marker.count < scanEnd {
|
|
for index in markerIndex + marker.count..<scanEnd where data[index] == UInt8(ascii: "{") {
|
|
jsonStart = index
|
|
break
|
|
}
|
|
}
|
|
if let jsonStart, let jsonEnd = findJsonEnd(data, start: jsonStart) {
|
|
return data.subdata(in: jsonStart..<(jsonEnd + 1))
|
|
}
|
|
searchStart = markerIndex + marker.count
|
|
}
|
|
return nil
|
|
}
|
|
|
|
private static func indexOf(
|
|
_ data: Data,
|
|
marker: [UInt8],
|
|
start: Int
|
|
) -> Int? {
|
|
guard !marker.isEmpty, data.count >= marker.count else {
|
|
return nil
|
|
}
|
|
var index = start
|
|
while index <= data.count - marker.count {
|
|
var matched = true
|
|
for offset in marker.indices where data[index + offset] != marker[offset] {
|
|
matched = false
|
|
break
|
|
}
|
|
if matched {
|
|
return index
|
|
}
|
|
index += 1
|
|
}
|
|
return nil
|
|
}
|
|
|
|
private static func findJsonEnd(_ data: Data, start: Int) -> Int? {
|
|
var depth = 0
|
|
var inString = false
|
|
var escaped = false
|
|
var index = start
|
|
while index < data.count {
|
|
let char = data[index]
|
|
if inString {
|
|
if escaped {
|
|
escaped = false
|
|
} else if char == UInt8(ascii: "\\") {
|
|
escaped = true
|
|
} else if char == UInt8(ascii: "\"") {
|
|
inString = false
|
|
}
|
|
} else {
|
|
if char == UInt8(ascii: "\"") {
|
|
inString = true
|
|
} else if char == UInt8(ascii: "{") {
|
|
depth += 1
|
|
} else if char == UInt8(ascii: "}") {
|
|
depth -= 1
|
|
if depth == 0 {
|
|
return index
|
|
}
|
|
}
|
|
}
|
|
index += 1
|
|
}
|
|
return nil
|
|
}
|
|
|
|
private static func normalizedRect(
|
|
values: [Any],
|
|
videoWidth: Double,
|
|
videoHeight: Double
|
|
) -> GiftMp4SourceRect {
|
|
GiftMp4SourceRect(
|
|
x: clamp(doubleValue(values[0]) ?? 0, min: 0, max: videoWidth) / videoWidth,
|
|
y: clamp(doubleValue(values[1]) ?? 0, min: 0, max: videoHeight) / videoHeight,
|
|
width: clamp(doubleValue(values[2]) ?? 0, min: 0, max: videoWidth) / videoWidth,
|
|
height: clamp(doubleValue(values[3]) ?? 0, min: 0, max: videoHeight) / videoHeight
|
|
)
|
|
}
|
|
|
|
private static func detectPackedAlphaTopRightSourceLayout(
|
|
asset: AVAsset
|
|
) -> GiftMp4SourceLayout? {
|
|
guard let candidate = detectPackedAlphaTopRight(asset: asset) else {
|
|
return nil
|
|
}
|
|
let frameWidth = max(candidate.frameWidth, 1)
|
|
let frameHeight = max(candidate.frameHeight, 1)
|
|
let alphaStartX = clamp(candidate.alphaStartX, min: 1, max: frameWidth - 1)
|
|
let alphaEndX = clamp(candidate.alphaEndX, min: alphaStartX, max: frameWidth - 1)
|
|
let colorWidth = alphaStartX
|
|
let alphaWidth = max(alphaEndX - alphaStartX + 1, 1)
|
|
let alphaHeight = clamp(
|
|
Int(Double(frameHeight) * Double(alphaWidth) / Double(colorWidth) + 0.5),
|
|
min: 1,
|
|
max: frameHeight
|
|
)
|
|
|
|
return GiftMp4SourceLayout(
|
|
kind: "packedAlphaTopRight",
|
|
hasAlphaMask: true,
|
|
contentAspectRatio: Double(colorWidth) / Double(frameHeight),
|
|
colorRect: GiftMp4SourceRect(
|
|
x: 0,
|
|
y: 0,
|
|
width: Double(colorWidth) / Double(frameWidth),
|
|
height: 1
|
|
),
|
|
alphaRect: GiftMp4SourceRect(
|
|
x: Double(alphaStartX) / Double(frameWidth),
|
|
y: 0,
|
|
width: Double(alphaWidth) / Double(frameWidth),
|
|
height: Double(alphaHeight) / Double(frameHeight)
|
|
)
|
|
)
|
|
}
|
|
|
|
private static func detectPackedAlphaTopRight(
|
|
asset: AVAsset
|
|
) -> GiftMp4PackedAlphaCandidate? {
|
|
let frames = samplePackedAlphaFrames(asset: asset)
|
|
let candidates = frames.compactMap(detectPackedAlphaTopRightInFrame)
|
|
guard !candidates.isEmpty else {
|
|
return nil
|
|
}
|
|
|
|
let frameWidth = median(candidates.map(\.frameWidth))
|
|
let frameHeight = median(candidates.map(\.frameHeight))
|
|
let alphaStartX = median(candidates.map(\.alphaStartX))
|
|
let alphaEndX = median(candidates.map(\.alphaEndX))
|
|
let alphaWidth = alphaEndX - alphaStartX + 1
|
|
let alphaStartRatio = Double(alphaStartX) / Double(frameWidth)
|
|
let alphaWidthRatio = Double(alphaWidth) / Double(frameWidth)
|
|
let colorToAlphaRatio = Double(alphaStartX) / Double(alphaWidth)
|
|
|
|
guard
|
|
(0.58...0.78).contains(alphaStartRatio),
|
|
(0.18...0.42).contains(alphaWidthRatio),
|
|
(1.45...2.65).contains(colorToAlphaRatio),
|
|
Double(alphaEndX) >= Double(frameWidth) * 0.82
|
|
else {
|
|
return nil
|
|
}
|
|
|
|
return GiftMp4PackedAlphaCandidate(
|
|
frameWidth: frameWidth,
|
|
frameHeight: frameHeight,
|
|
alphaStartX: alphaStartX,
|
|
alphaEndX: alphaEndX
|
|
)
|
|
}
|
|
|
|
private static func detectPackedAlphaTopRightInFrame(
|
|
_ image: CGImage
|
|
) -> GiftMp4PackedAlphaCandidate? {
|
|
guard let pixels = PixelSampler(image: image) else {
|
|
return nil
|
|
}
|
|
let width = pixels.width
|
|
let height = pixels.height
|
|
guard width >= 12, height >= 12 else {
|
|
return nil
|
|
}
|
|
|
|
let searchStart = clamp(Int(Double(width) * 0.54), min: 0, max: width - 1)
|
|
let sampleHeight = clamp(Int(Double(height) * 0.58), min: 1, max: height)
|
|
let stepY = max(1, sampleHeight / 220)
|
|
let rowSamples = max(1, (sampleHeight + stepY - 1) / stepY)
|
|
let threshold = max(8, Int(Double(rowSamples) * 0.32))
|
|
var grayColumns = Array(repeating: false, count: width)
|
|
|
|
for x in searchStart..<width {
|
|
var grayCount = 0
|
|
var y = 0
|
|
while y < sampleHeight {
|
|
if isLikelyAlphaMaskPixel(pixels.pixel(x: x, y: y)) {
|
|
grayCount += 1
|
|
}
|
|
y += stepY
|
|
}
|
|
grayColumns[x] = grayCount >= threshold
|
|
}
|
|
|
|
let minRangeWidth = max(24, Int(Double(width) * 0.12))
|
|
var bestStart = -1
|
|
var bestEnd = -1
|
|
var index = searchStart
|
|
while index < width {
|
|
while index < width, !grayColumns[index] {
|
|
index += 1
|
|
}
|
|
if index >= width {
|
|
break
|
|
}
|
|
let start = index
|
|
while index < width, grayColumns[index] {
|
|
index += 1
|
|
}
|
|
let end = index - 1
|
|
if end - start + 1 >= minRangeWidth, end > bestEnd {
|
|
bestStart = start
|
|
bestEnd = end
|
|
}
|
|
}
|
|
|
|
guard bestStart >= 0, bestEnd >= 0 else {
|
|
return nil
|
|
}
|
|
return GiftMp4PackedAlphaCandidate(
|
|
frameWidth: width,
|
|
frameHeight: height,
|
|
alphaStartX: bestStart,
|
|
alphaEndX: bestEnd
|
|
)
|
|
}
|
|
|
|
private static func detectSplitSourceLayout(
|
|
asset: AVAsset,
|
|
videoSize: CGSize
|
|
) -> GiftMp4SourceLayout? {
|
|
guard let candidate = detectSplitAlpha(asset: asset) else {
|
|
return nil
|
|
}
|
|
return splitSourceLayout(candidate: candidate, videoSize: videoSize)
|
|
}
|
|
|
|
private static func splitSourceLayout(
|
|
candidate: GiftMp4SplitAlphaCandidate,
|
|
videoSize: CGSize
|
|
) -> GiftMp4SourceLayout {
|
|
let left = GiftMp4SourceRect(x: 0, y: 0, width: 0.5, height: 1)
|
|
let right = GiftMp4SourceRect(x: 0.5, y: 0, width: 0.5, height: 1)
|
|
let top = GiftMp4SourceRect(x: 0, y: 0, width: 1, height: 0.5)
|
|
let bottom = GiftMp4SourceRect(x: 0, y: 0.5, width: 1, height: 0.5)
|
|
let isHorizontal = candidate.orientation == .horizontal
|
|
let first = isHorizontal ? left : top
|
|
let second = isHorizontal ? right : bottom
|
|
let aspect = isHorizontal
|
|
? Double(videoSize.width / 2) / Double(videoSize.height)
|
|
: Double(videoSize.width) / Double(videoSize.height / 2)
|
|
|
|
return GiftMp4SourceLayout(
|
|
kind: isHorizontal ? "splitAlphaHorizontal" : "splitAlphaVertical",
|
|
hasAlphaMask: true,
|
|
contentAspectRatio: aspect,
|
|
colorRect: candidate.alphaFirst ? second : first,
|
|
alphaRect: candidate.alphaFirst ? first : second
|
|
)
|
|
}
|
|
|
|
private static func detectSplitAlpha(asset: AVAsset) -> GiftMp4SplitAlphaCandidate? {
|
|
let frames = sampleFrames(asset: asset)
|
|
guard !frames.isEmpty else {
|
|
return nil
|
|
}
|
|
|
|
var horizontalScore = GiftMp4SplitScore()
|
|
var verticalScore = GiftMp4SplitScore()
|
|
for frame in frames {
|
|
guard let pixels = PixelSampler(image: frame) else {
|
|
continue
|
|
}
|
|
let horizontal = statsByHorizontalHalves(pixels)
|
|
let vertical = statsByVerticalHalves(pixels)
|
|
addSplitScore(&horizontalScore, first: horizontal.0, second: horizontal.1)
|
|
addSplitScore(&verticalScore, first: vertical.0, second: vertical.1)
|
|
}
|
|
|
|
let horizontalCandidate = splitCandidateFromScore(
|
|
orientation: .horizontal,
|
|
score: horizontalScore
|
|
)
|
|
let verticalCandidate = splitCandidateFromScore(
|
|
orientation: .vertical,
|
|
score: verticalScore
|
|
)
|
|
|
|
return [horizontalCandidate, verticalCandidate]
|
|
.compactMap { $0 }
|
|
.max { $0.1 < $1.1 }?
|
|
.0
|
|
}
|
|
|
|
private static func sampleFrames(asset: AVAsset) -> [CGImage] {
|
|
let durationSeconds = CMTimeGetSeconds(asset.duration)
|
|
let desiredSeconds = [0.3, 0.6, 0.9, 1.2, 1.5, 1.8, 2.1, 2.4, 2.7, 3.0]
|
|
.filter { durationSeconds.isNaN || durationSeconds <= 0 || $0 <= durationSeconds }
|
|
return sampleFrames(
|
|
asset: asset,
|
|
seconds: desiredSeconds.isEmpty ? [0.0] : desiredSeconds,
|
|
toleranceBefore: .zero,
|
|
toleranceAfter: .positiveInfinity
|
|
)
|
|
}
|
|
|
|
private static func samplePackedAlphaFrames(asset: AVAsset) -> [CGImage] {
|
|
let durationSeconds = CMTimeGetSeconds(asset.duration)
|
|
let desiredSeconds = [0.0, 0.5, 1.0, 2.0, 3.0]
|
|
.filter { durationSeconds.isNaN || durationSeconds <= 0 || $0 <= durationSeconds }
|
|
return sampleFrames(
|
|
asset: asset,
|
|
seconds: desiredSeconds.isEmpty ? [0.0] : desiredSeconds,
|
|
toleranceBefore: .positiveInfinity,
|
|
toleranceAfter: .positiveInfinity
|
|
)
|
|
}
|
|
|
|
private static func sampleFrames(
|
|
asset: AVAsset,
|
|
seconds: [Double],
|
|
toleranceBefore: CMTime,
|
|
toleranceAfter: CMTime
|
|
) -> [CGImage] {
|
|
let generator = AVAssetImageGenerator(asset: asset)
|
|
generator.appliesPreferredTrackTransform = true
|
|
generator.requestedTimeToleranceBefore = toleranceBefore
|
|
generator.requestedTimeToleranceAfter = toleranceAfter
|
|
|
|
let times = seconds.map { NSValue(time: CMTime(seconds: $0, preferredTimescale: 600)) }
|
|
|
|
var frames: [CGImage] = []
|
|
for time in times {
|
|
if let frame = try? generator.copyCGImage(
|
|
at: time.timeValue,
|
|
actualTime: nil
|
|
) {
|
|
frames.append(frame)
|
|
}
|
|
}
|
|
return frames
|
|
}
|
|
|
|
private static func statsByHorizontalHalves(
|
|
_ pixels: PixelSampler
|
|
) -> (GiftMp4HalfStats, GiftMp4HalfStats) {
|
|
let half = pixels.width / 2
|
|
return (
|
|
statsForRect(pixels, startX: 0, startY: 0, endX: half, endY: pixels.height),
|
|
statsForRect(
|
|
pixels,
|
|
startX: half,
|
|
startY: 0,
|
|
endX: pixels.width,
|
|
endY: pixels.height
|
|
)
|
|
)
|
|
}
|
|
|
|
private static func statsByVerticalHalves(
|
|
_ pixels: PixelSampler
|
|
) -> (GiftMp4HalfStats, GiftMp4HalfStats) {
|
|
let half = pixels.height / 2
|
|
return (
|
|
statsForRect(pixels, startX: 0, startY: 0, endX: pixels.width, endY: half),
|
|
statsForRect(
|
|
pixels,
|
|
startX: 0,
|
|
startY: half,
|
|
endX: pixels.width,
|
|
endY: pixels.height
|
|
)
|
|
)
|
|
}
|
|
|
|
private static func statsForRect(
|
|
_ pixels: PixelSampler,
|
|
startX: Int,
|
|
startY: Int,
|
|
endX: Int,
|
|
endY: Int
|
|
) -> GiftMp4HalfStats {
|
|
let width = max(1, endX - startX)
|
|
let height = max(1, endY - startY)
|
|
let stepX = max(1, width / 80)
|
|
let stepY = max(1, height / 120)
|
|
var saturationSum = 0.0
|
|
var grayErrorSum = 0.0
|
|
var brightnessSum = 0.0
|
|
var count = 0.0
|
|
|
|
var y = startY
|
|
while y < endY {
|
|
var x = startX
|
|
while x < endX {
|
|
let pixel = pixels.pixel(x: x, y: y)
|
|
let maxChannel = max(pixel.r, pixel.g, pixel.b)
|
|
let minChannel = min(pixel.r, pixel.g, pixel.b)
|
|
let saturation = maxChannel == 0
|
|
? 0
|
|
: Double(maxChannel - minChannel) / Double(maxChannel)
|
|
let grayError = (
|
|
abs(Double(pixel.r) - Double(pixel.g)) +
|
|
abs(Double(pixel.g) - Double(pixel.b)) +
|
|
abs(Double(pixel.b) - Double(pixel.r))
|
|
) / (3.0 * 255.0)
|
|
let brightness = (
|
|
Double(pixel.r) * 0.299 +
|
|
Double(pixel.g) * 0.587 +
|
|
Double(pixel.b) * 0.114
|
|
) / 255.0
|
|
saturationSum += saturation
|
|
grayErrorSum += grayError
|
|
brightnessSum += brightness
|
|
count += 1
|
|
x += stepX
|
|
}
|
|
y += stepY
|
|
}
|
|
|
|
if count == 0 {
|
|
return GiftMp4HalfStats(saturation: 0, grayError: 0, brightness: 0)
|
|
}
|
|
return GiftMp4HalfStats(
|
|
saturation: saturationSum / count,
|
|
grayError: grayErrorSum / count,
|
|
brightness: brightnessSum / count
|
|
)
|
|
}
|
|
|
|
private static func addSplitScore(
|
|
_ score: inout GiftMp4SplitScore,
|
|
first: GiftMp4HalfStats,
|
|
second: GiftMp4HalfStats
|
|
) {
|
|
if isLowBrightnessFrame(first: first, second: second) {
|
|
return
|
|
}
|
|
let alphaStats = first.saturation < second.saturation ? first : second
|
|
let saturationGap = abs(second.saturation - first.saturation)
|
|
guard
|
|
saturationGap >= minSplitSaturationGap,
|
|
alphaStats.saturation <= maxSplitMaskSaturation,
|
|
alphaStats.grayError <= maxSplitMaskGrayError
|
|
else {
|
|
return
|
|
}
|
|
score.direction += second.saturation - first.saturation
|
|
score.saturationGap += saturationGap
|
|
score.maskSaturation += alphaStats.saturation
|
|
score.maskGrayError += alphaStats.grayError
|
|
score.frames += 1
|
|
}
|
|
|
|
private static func isLowBrightnessFrame(
|
|
first: GiftMp4HalfStats,
|
|
second: GiftMp4HalfStats
|
|
) -> Bool {
|
|
max(first.brightness, second.brightness) < minUsefulFrameBrightness
|
|
}
|
|
|
|
private static func splitCandidateFromScore(
|
|
orientation: GiftMp4SplitOrientation,
|
|
score: GiftMp4SplitScore
|
|
) -> (GiftMp4SplitAlphaCandidate, Double)? {
|
|
guard score.frames >= minSplitMatchFrames else {
|
|
return nil
|
|
}
|
|
let frames = Double(score.frames)
|
|
let averageSaturationGap = score.saturationGap / frames
|
|
let averageMaskSaturation = score.maskSaturation / frames
|
|
let averageMaskGrayError = score.maskGrayError / frames
|
|
|
|
guard
|
|
averageSaturationGap >= minSplitSaturationGap,
|
|
averageMaskSaturation <= maxSplitMaskSaturation,
|
|
averageMaskGrayError <= maxSplitMaskGrayError
|
|
else {
|
|
return nil
|
|
}
|
|
|
|
return (
|
|
GiftMp4SplitAlphaCandidate(
|
|
orientation: orientation,
|
|
alphaFirst: score.direction > 0
|
|
),
|
|
averageSaturationGap
|
|
)
|
|
}
|
|
|
|
private static func isLikelyAlphaMaskPixel(_ pixel: Pixel) -> Bool {
|
|
let maxChannel = max(pixel.r, pixel.g, pixel.b)
|
|
if maxChannel < 22 {
|
|
return false
|
|
}
|
|
let minChannel = min(pixel.r, pixel.g, pixel.b)
|
|
let saturation = maxChannel == 0
|
|
? 0
|
|
: Double(maxChannel - minChannel) / Double(maxChannel)
|
|
let grayError = (
|
|
abs(Double(pixel.r) - Double(pixel.g)) +
|
|
abs(Double(pixel.g) - Double(pixel.b)) +
|
|
abs(Double(pixel.b) - Double(pixel.r))
|
|
) / (3.0 * 255.0)
|
|
return saturation < 0.16 && grayError < 0.055
|
|
}
|
|
|
|
private static func doubleValue(_ value: Any?) -> Double? {
|
|
if let value = value as? Double {
|
|
return value
|
|
}
|
|
if let value = value as? Int {
|
|
return Double(value)
|
|
}
|
|
if let value = value as? NSNumber {
|
|
return value.doubleValue
|
|
}
|
|
if let value = value as? String {
|
|
return Double(value)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
private static func median(_ values: [Int]) -> Int {
|
|
guard !values.isEmpty else {
|
|
return 0
|
|
}
|
|
return values.sorted()[values.count / 2]
|
|
}
|
|
|
|
private static func clamp<T: Comparable>(_ value: T, min: T, max: T) -> T {
|
|
Swift.max(min, Swift.min(max, value))
|
|
}
|
|
}
|
|
|
|
private struct Pixel {
|
|
let r: UInt8
|
|
let g: UInt8
|
|
let b: UInt8
|
|
let a: UInt8
|
|
}
|
|
|
|
private final class PixelSampler {
|
|
let width: Int
|
|
let height: Int
|
|
|
|
private let bytes: [UInt8]
|
|
private let bytesPerPixel = 4
|
|
|
|
init?(image: CGImage) {
|
|
width = image.width
|
|
height = image.height
|
|
guard width > 0, height > 0 else {
|
|
return nil
|
|
}
|
|
|
|
var buffer = [UInt8](repeating: 0, count: width * height * bytesPerPixel)
|
|
guard
|
|
let context = CGContext(
|
|
data: &buffer,
|
|
width: width,
|
|
height: height,
|
|
bitsPerComponent: 8,
|
|
bytesPerRow: width * bytesPerPixel,
|
|
space: CGColorSpaceCreateDeviceRGB(),
|
|
bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
|
|
)
|
|
else {
|
|
return nil
|
|
}
|
|
context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height))
|
|
bytes = buffer
|
|
}
|
|
|
|
func pixel(x: Int, y: Int) -> Pixel {
|
|
let safeX = Swift.max(0, Swift.min(width - 1, x))
|
|
let safeY = Swift.max(0, Swift.min(height - 1, y))
|
|
let index = (safeY * width + safeX) * bytesPerPixel
|
|
return Pixel(
|
|
r: bytes[index],
|
|
g: bytes[index + 1],
|
|
b: bytes[index + 2],
|
|
a: bytes[index + 3]
|
|
)
|
|
}
|
|
}
|