合并视频时 Swift -AVMutableVideoCompositionLayerInstruction 错位

Question

我关注了Ray Wenderlich来合并视频。完成的结果是 1 个合并视频，其中纵向视频位于屏幕顶部，横向视频位于屏幕底部。在下图中，先播放纵向视频，然后播放横向视频。风景视频来自照片库。

代码：

func mergVideos() {

    let mixComposition = AVMutableComposition()
            
    let videoCompositionTrack = mixComposition.addMutableTrack(withMediaType: .video, preferredTrackID: Int32(kCMPersistentTrackID_Invalid))
    let audioCompositionTrack = mixComposition.addMutableTrack(withMediaType: .audio, preferredTrackID: Int32(kCMPersistentTrackID_Invalid))
    
    var count = 0
    var insertTime = CMTime.zero
    var instructions = [AVMutableVideoCompositionInstruction]()
    
    for videoAsset in arrOfAssets {

        let audioTrack = videoAsset.tracks(withMediaType: .audio)[0]

        do {
    
            try videoCompositionTrack?.insertTimeRange(CMTimeRangeMake(start: .zero, duration: videoAsset.duration), of: videoAsset.tracks(withMediaType: .video)[0], at: insertTime)
            try audioCompositionTrack?.insertTimeRange(CMTimeRangeMake(start: .zero, duration: videoAsset.duration), of: audioTrack, at: insertTime)
    
            let layerInstruction = videoCompositionInstruction(videoCompositionTrack!, asset: videoAsset, count: count)
    
            let videoCompositionInstruction = AVMutableVideoCompositionInstruction()
            videoCompositionInstruction.timeRange = CMTimeRangeMake(start: insertTime, duration: videoAsset.duration)
            videoCompositionInstruction.layerInstructions = [layerInstruction]

            instructions.append(videoCompositionInstruction)
    
            insertTime = CMTimeAdd(insertTime, videoAsset.duration)

            count += 1

        } catch { }
    }
    
    let videoComposition = AVMutableVideoComposition()
    videoComposition.instructions = instructions
    videoComposition.frameDuration = CMTimeMake(value: 1, timescale: 30)
    videoComposition.renderSize = CGSize(width: UIScreen.main.bounds.width, height: UIScreen.main.bounds.height)

    // ...
    exporter.videoComposition = videoComposition
}

雷文德利希代码：

func videoCompositionInstruction(_ track: AVCompositionTrack, asset: AVAsset, count: Int) -> AVMutableVideoCompositionLayerInstruction {
    
    let instruction = AVMutableVideoCompositionLayerInstruction(assetTrack: track)
    
    let assetTrack = asset.tracks(withMediaType: .video)[0]
    
    let transform = assetTrack.preferredTransform
    let assetInfo = orientationFromTransform(transform)
    
    var scaleToFitRatio = UIScreen.main.bounds.width / assetTrack.naturalSize.width
    if assetInfo.isPortrait {
        
        scaleToFitRatio = UIScreen.main.bounds.width / assetTrack.naturalSize.height
        let scaleFactor = CGAffineTransform(scaleX: scaleToFitRatio, y: scaleToFitRatio)
        instruction.setTransform(assetTrack.preferredTransform.concatenating(scaleFactor), at: .zero)
        
    } else {
        
        let scaleFactor = CGAffineTransform(scaleX: scaleToFitRatio, y: scaleToFitRatio)
        var concat = assetTrack.preferredTransform.concatenating(scaleFactor)
            .concatenating(CGAffineTransform(translationX: 0,y: UIScreen.main.bounds.width / 2))
        if assetInfo.orientation == .down {
            let fixUpsideDown = CGAffineTransform(rotationAngle: CGFloat(Double.pi))
            let windowBounds = UIScreen.main.bounds
            let yFix = assetTrack.naturalSize.height + windowBounds.height
            let centerFix = CGAffineTransform(translationX: assetTrack.naturalSize.width, y: yFix)
            concat = fixUpsideDown.concatenating(centerFix).concatenating(scaleFactor)
        }
        instruction.setTransform(concat, at: .zero)
    }
    
    if count == 0 {
        instruction.setOpacity(0.0, at: asset.duration)
    }
    
    return instruction
}

func orientationFromTransform(_ transform: CGAffineTransform) -> (orientation: UIImage.Orientation, isPortrait: Bool) {
    var assetOrientation = UIImage.Orientation.up
      var isPortrait = false
      let tfA = transform.a
      let tfB = transform.b
      let tfC = transform.c
      let tfD = transform.d

      if tfA == 0 && tfB == 1.0 && tfC == -1.0 && tfD == 0 {
        assetOrientation = .right
        isPortrait = true
      } else if tfA == 0 && tfB == -1.0 && tfC == 1.0 && tfD == 0 {
        assetOrientation = .left
        isPortrait = true
      } else if tfA == 1.0 && tfB == 0 && tfC == 0 && tfD == 1.0 {
        assetOrientation = .up
      } else if tfA == -1.0 && tfB == 0 && tfC == 0 && tfD == -1.0 {
        assetOrientation = .down
      }
      return (assetOrientation, isPortrait)
}

我还关注了这个 Medium post 中的代码。它将渲染大小设置为默认值 let renderSize = CGSize(width: 1280.0, height: 720.0)，而不是使用整个屏幕的 Ray。

1280/720 结果是纵向视频正确居中，但横向视频可以播放声音，但视频不在屏幕上的任何位置。我没有添加风景图片，因为它只是一个黑屏。

Answer 1

我让它适用于肖像和风景。

我用纵向、左/右横向、倒置、前置摄像头和后置摄像头录制的视频测试了这个答案。我没有任何问题。我远不是CGAffineTransform专家，所以如果有人有更好的答案请发布。

Ray Wenderlich 的合并代码有效，但不适用于不同方向的视频。我使用这个 answer 来检查 preferredTransform 的属性以进行 orientation 检查。

我还为 portrait orientation 部分使用了 deleted 答案，为 landscape orientation 部分使用了 downvoted 答案。被否决的答案让我找到了他的GitHub，其中景观代码是不正确的但足够接近，我可以对其进行调整以使其正常工作。

需要指出的一件事是来自@DonMag 的 cmets 告诉我使用的好处720x1280.下面的代码会将所有视频与 renderSize 合并在一起720x1280这将使它们保持相同的大小。

代码：

// class property
let renderSize = CGSize(width: 720, height: 1280) // for higher quality use CGSize(width: 1080, height: 1920)

func mergVideos() {

    let mixComposition = AVMutableComposition()
            
    let videoCompositionTrack = mixComposition.addMutableTrack(withMediaType: .video, preferredTrackID: Int32(kCMPersistentTrackID_Invalid))
    let audioCompositionTrack = mixComposition.addMutableTrack(withMediaType: .audio, preferredTrackID: Int32(kCMPersistentTrackID_Invalid))
    
    var count = 0
    var insertTime = CMTime.zero
    var instructions = [AVMutableVideoCompositionInstruction]()
    
    for videoAsset in arrOfAssets {

        guard let firstTrack = videoAsset.tracks.first, let _ = videoAsset.tracks(withMediaType: .video).first else { continue }

        do {
    
            try videoCompositionTrack?.insertTimeRange(CMTimeRangeMake(start: .zero, duration: videoAsset.duration), of: videoAsset.tracks(withMediaType: .video)[0], at: insertTime)

            if let audioTrack = videoAsset.tracks(withMediaType: .audio).first {
                try audioCompositionTrack?.insertTimeRange(CMTimeRangeMake(start: .zero, duration: videoAsset.duration), of: audioTrack, at: insertTime)
            }

            let layerInstruction = videoCompositionInstruction(firstTrack, asset: videoAsset, count: count)
    
            let videoCompositionInstruction = AVMutableVideoCompositionInstruction()
            videoCompositionInstruction.timeRange = CMTimeRangeMake(start: insertTime, duration: videoAsset.duration)
            videoCompositionInstruction.layerInstructions = [layerInstruction]

            instructions.append(videoCompositionInstruction)
    
            insertTime = CMTimeAdd(insertTime, videoAsset.duration)

            count += 1

        } catch { }
    }
    
    let videoComposition = AVMutableVideoComposition()
    videoComposition.instructions = instructions
    videoComposition.frameDuration = CMTimeMake(value: 1, timescale: 30)
    videoComposition.renderSize = self.renderSize // <--- **** IMPORTANT ****

    // ...
    exporter.videoComposition = videoComposition
}

最多重要部分这个替换 RW 代码的答案：

func videoCompositionInstruction(_ firstTrack: AVAssetTrack, asset: AVAsset, count: Int) -> AVMutableVideoCompositionLayerInstruction {

    let instruction = AVMutableVideoCompositionLayerInstruction(assetTrack: firstTrack)

    let assetTrack = asset.tracks(withMediaType: .video)[0]            
    let t = assetTrack.fixedPreferredTransform // new transform fix 
    let assetInfo = orientationFromTransform(t)

    if assetInfo.isPortrait {

        let scaleToFitRatio = self.renderSize.width / assetTrack.naturalSize.height
        let scaleFactor = CGAffineTransform(scaleX: scaleToFitRatio, y: scaleToFitRatio)
        var finalTransform = assetTrack.fixedPreferredTransform.concatenating(scaleFactor)

        // This was needed in the case of the OP's answer that I used for the portrait part. I haven't tested this but this is what he said: "(if video not taking entire screen and leaving some parts black - don't know when actually needed so you'll have to try and see when it's needed)"
        if assetInfo.orientation == .rightMirrored || assetInfo.orientation == .leftMirrored {
            finalTransform = finalTransform.translatedBy(x: -transform.ty, y: 0)
        }
        instruction.setTransform(finalTransform, at: CMTime.zero)

    } else {

        let renderRect = CGRect(x: 0, y: 0, width: self.renderSize.width, height: self.renderSize.height)
        let videoRect = CGRect(origin: .zero, size: assetTrack.naturalSize).applying(assetTrack.fixedPreferredTransform)

        let scale = renderRect.width / videoRect.width
        let transform = CGAffineTransform(scaleX: renderRect.width / videoRect.width, y: (videoRect.height * scale) / assetTrack.naturalSize.height)
        let translate = CGAffineTransform(translationX: .zero, y: ((self.renderSize.height - (videoRect.height * scale))) / 2)

        instruction.setTransform(assetTrack.fixedPreferredTransform.concatenating(transform).concatenating(translate), at: .zero)
    }

    if count == 0 {
        instruction.setOpacity(0.0, at: asset.duration)
    }
    
    return instruction
}

新方向检查：

func orientationFromTransform(_ transform: CGAffineTransform) -> (orientation: UIImage.Orientation, isPortrait: Bool) {
    var assetOrientation = UIImage.Orientation.up
    var isPortrait = false
    
    if transform.a == 0 && transform.b == 1.0 && transform.c == -1.0 && transform.d == 0 {
        assetOrientation = .right
        isPortrait = true
    } else if transform.a == 0 && transform.b == 1.0 && transform.c == 1.0 && transform.d == 0 {
        assetOrientation = .rightMirrored
        isPortrait = true
    } else if transform.a == 0 && transform.b == -1.0 && transform.c == 1.0 && transform.d == 0 {
        assetOrientation = .left
        isPortrait = true
    } else if transform.a == 0 && transform.b == -1.0 && transform.c == -1.0 && transform.d == 0 {
        assetOrientation = .leftMirrored
        isPortrait = true
    } else if transform.a == 1.0 && transform.b == 0 && transform.c == 0 && transform.d == 1.0 {
        assetOrientation = .up
    } else if transform.a == -1.0 && transform.b == 0 && transform.c == 0 && transform.d == -1.0 {
        assetOrientation = .down
    }
}

首选转换fix：

extension AVAssetTrack {
    
    var fixedPreferredTransform: CGAffineTransform {
        var t = preferredTransform
        switch(t.a, t.b, t.c, t.d) {
        case (1, 0, 0, 1):
            t.tx = 0
            t.ty = 0
        case (1, 0, 0, -1):
            t.tx = 0
            t.ty = naturalSize.height
        case (-1, 0, 0, 1):
            t.tx = naturalSize.width
            t.ty = 0
        case (-1, 0, 0, -1):
            t.tx = naturalSize.width
            t.ty = naturalSize.height
        case (0, -1, 1, 0):
            t.tx = 0
            t.ty = naturalSize.width
        case (0, 1, -1, 0):
            t.tx = naturalSize.height
            t.ty = 0
        case (0, 1, 1, 0):
            t.tx = 0
            t.ty = 0
        case (0, -1, -1, 0):
            t.tx = naturalSize.height
            t.ty = naturalSize.width
        default:
            break
        }
        return t
    }
}