AVMutableComposition resizing issue

Question

I'm trying to render an image into a video captured with the front camera using AVMutableComposition. The size of the resulting video (including the image) is perfectly fine.

However, the initial video will be resized as shown in this picture:

I'm using the NextLevelSessionExporter and this is my code snippet:

// * MARK - Creating composition
        /// Create AVMutableComposition object. This object will hold the AVMutableCompositionTrack instances.
        let mainMutableComposition = AVMutableComposition()

        /// Creating an empty video track
        let videoTrack = mainMutableComposition.addMutableTrack(withMediaType: AVMediaType.video, preferredTrackID: kCMPersistentTrackID_Invalid)
        let videoAssetTrack = videoAsset.tracks(withMediaType: AVMediaType.video)[0]

        do {
            //Adding the video track
            try videoTrack?.insertTimeRange(CMTimeRange(start: kCMTimeZero, duration: videoAsset.duration), of: videoAsset.tracks(withMediaType: AVMediaType.video).first!, at: kCMTimeZero)

        } catch {
            completion(false,  nil)
        }

        /// Adding audio if user wants to.
        if withAudio {
            do {
                //Adding the video track
                let audio = videoAsset.tracks(withMediaType: AVMediaType.audio).first
                if audio != nil {
                    let audioTrack = mainMutableComposition.addMutableTrack(withMediaType: AVMediaType.audio, preferredTrackID: kCMPersistentTrackID_Invalid)
                    try audioTrack?.insertTimeRange(CMTimeRange(start: kCMTimeZero, duration: videoAsset.duration), of: audio!, at: kCMTimeZero)

                }

            } catch {
                completion(false,  nil)
            }

        }

        // * MARK - Composition is ready ----------

        // Create AVMutableVideoCompositionInstruction
        let compositionInstructions = AVMutableVideoCompositionInstruction()
        compositionInstructions.timeRange = CMTimeRange(start: kCMTimeZero, duration: videoAsset.duration)

        // Create an AvmutableVideoCompositionLayerInstruction
        let videoLayerInstruction = AVMutableVideoCompositionLayerInstruction.init(assetTrack: videoTrack!)
        videoLayerInstruction.setTransform(videoAssetTrack.preferredTransform, at: kCMTimeZero)
        compositionInstructions.layerInstructions = [videoLayerInstruction]

        //Add instructions
        let videoComposition = AVMutableVideoComposition()

        let naturalSize : CGSize = videoAssetTrack.naturalSize

        ///Rendering image into video
        let renderWidth = naturalSize.width
        let renderHeight = naturalSize.height

        //Assigning instructions and rendering size
        videoComposition.renderSize = CGSize(width: renderWidth, height: renderHeight)
        videoComposition.instructions = [compositionInstructions]
        videoComposition.frameDuration = CMTime(value: 1, timescale: Int32((videoTrack?.nominalFrameRate)!))

        //Applying image to instruction
        self.applyVideoImage(to: videoComposition, withSize: naturalSize, image: image)

        // Getting the output path
        let documentsURL = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first
        let outputPath = documentsURL?.appendingPathComponent("lastEditedVideo.mp4")
        if FileManager.default.fileExists(atPath: (outputPath?.path)!) {
            do {
                try FileManager.default.removeItem(atPath: (outputPath?.path)!)
            }
            catch {
                completion(false, nil)
            }
        }



        // Create exporter
        let exporter = NextLevelSessionExporter(withAsset: mainMutableComposition)
        exporter.outputURL = outputPath
        exporter.outputFileType = AVFileType.mp4
        exporter.videoComposition = videoComposition

        let compressionDict: [String: Any] = [
            AVVideoAverageBitRateKey: NSNumber(integerLiteral: 2300000),
            AVVideoProfileLevelKey: AVVideoProfileLevelH264BaselineAutoLevel as String
            ]

        exporter.videoOutputConfiguration = [
            AVVideoCodecKey: AVVideoCodecType.h264,
            AVVideoWidthKey: NSNumber(integerLiteral: Int(naturalSize.width)),
            AVVideoHeightKey: NSNumber(integerLiteral: Int(naturalSize.height)),
            AVVideoCompressionPropertiesKey: compressionDict
        ]

        exporter.audioOutputConfiguration = [
            AVFormatIDKey: kAudioFormatMPEG4AAC,
            AVEncoderBitRateKey: NSNumber(integerLiteral: 128000),
            AVNumberOfChannelsKey: NSNumber(integerLiteral: 2),
            AVSampleRateKey: NSNumber(value: Float(44100))
        ]

        completion(true, exporter)
    }

This is my applyVideoImage() function.

private func applyVideoImage(to composition: AVMutableVideoComposition, withSize size: CGSize, image: UIImage) { //Adds an image to a video composition

    //Creating image layer
    let overlayLayer = CALayer()
    let overlayImage: UIImage = image
    overlayLayer.contents = overlayImage.cgImage
    overlayLayer.frame = CGRect(x: 0, y: 0, width: size.width, height: size.height)
    overlayLayer.contentsGravity = kCAGravityResizeAspectFill
    overlayLayer.masksToBounds = true

    //Creating parent and video layer
    let parentLayer = CALayer()
    let videoLayer = CALayer()
    parentLayer.frame = CGRect(x: 0, y: 0, width: size.width, height: size.height)
    videoLayer.frame = CGRect(x: 0, y: 0, width: size.width, height: size.height)
    parentLayer.addSublayer(videoLayer)
    parentLayer.addSublayer(overlayLayer)

    //Adding those layers to video
    composition.animationTool = AVVideoCompositionCoreAnimationTool(postProcessingAsVideoLayer: videoLayer, in: parentLayer)
}

EDIT 1: This bug only occurs when I'm exporting a mirrored video that has been captured with the front camera.

Interdenominational

Answer 1

This is really tricky: You need to check the preferredTransform of the video track to determine wether it is a portrait video or not.

    var videoAssetOrientation = UIImageOrientation.up
    var isVideoAssetPortrait = false
    var videoTransform = videoAssetTrack.preferredTransform
    if needsMirroring == true  {
        isVideoAssetPortrait = true
    }else if videoTransform.a == 0 && videoTransform.b == 1.0 && videoTransform.c == -1.0 && videoTransform.d == 0 {
        videoAssetOrientation = .right
        isVideoAssetPortrait = true
    }else if videoTransform.a == 0 && videoTransform.b == -1.0 && videoTransform.c == 1.0 && videoTransform.d == 0 {
        videoAssetOrientation = .left
        isVideoAssetPortrait = true
    }else if videoTransform.a == 1.0 && videoTransform.b == 0 && videoTransform.c == 0 && videoTransform.d == 1.0 {
        videoAssetOrientation = .up
    }else if videoTransform.a == -1.0 && videoTransform.b == 0 && videoTransform.c == 0 && videoTransform.d == -1.0 {
        videoAssetOrientation = .down
    }

    //Add instructions
    mainInstruction.layerInstructions = [videoLayerInstruction]
    let mainCompositionInst = AVMutableVideoComposition()
    let naturalSize : CGSize!
    if isVideoAssetPortrait {
        naturalSize = CGSize(width: videoAssetTrack.naturalSize.height, height: videoAssetTrack.naturalSize.width)
    } else {
        naturalSize = videoAssetTrack.naturalSize
    }

Hope that helps.

Interdenominational

just a quick question but what is needsMirroring and how is it defined? – Delorenzo

Answer 2

Try applying a negative scale transform to flip the video when mirrored:

// Create an AvmutableVideoCompositionLayerInstruction
let videoLayerInstruction = AVMutableVideoCompositionLayerInstruction.init(assetTrack: videoTrack!)
let flipped = videoAssetTrack.preferredTransform.scaledBy(x: -1.0, y: 1.0)
videoLayerInstruction.setTransform(flipped, at: kCMTimeZero)
compositionInstructions.layerInstructions = [videoLayerInstruction]    

Telephonist

Unfortunately, this does not fix my issue. Thanks anyways! – Interdenominational