How to convert CVImageBuffer to UIImage?

Question

I have temporary variable tmpPixelBuffer with pixel buffer data, which is not nil, and when metadata objects are detected I want to create image from that buffer, so I could crop metadata images from that image.

Image is always nil, what do I do wrong?

func captureOutput(captureOutput: AVCaptureOutput!, didOutputSampleBuffer sampleBuffer: CMSampleBuffer!, fromConnection connection: AVCaptureConnection!) {

    tmpPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)
}


func captureOutput(captureOutput: AVCaptureOutput!, didOutputMetadataObjects metadataObjects: [AnyObject]!, fromConnection connection: AVCaptureConnection!) {

    let image = CIImage(CVPixelBuffer: tmpPixelBuffer)
    let context = CIContext()
    let cgiImage = context.createCGImage(image, fromRect: image.extent())
    let capturedImage = UIImage(CGImage: cgiImage)
    ...
}

I also tried to do it like that:

func captureOutput(captureOutput: AVCaptureOutput!, didOutputMetadataObjects metadataObjects: [AnyObject]!, fromConnection connection: AVCaptureConnection!) {

    let image = CIImage(CVPixelBuffer: tmpPixelBuffer)
    let context = CIContext(options: nil)

    let cgiImage = context.createCGImage(image, fromRect: CGRect(x: 0, y: 0, width: Int(CVPixelBufferGetWidth(tmpPixelBuffer)), height: Int(CVPixelBufferGetHeight(tmpPixelBuffer))))
    ...
}

But in this case UIImage is not readable.

Inept

My guess is that if the output format was not BGRA, then you got junk back. Just a guess. BTW, why didn't you award an answer to Andrea who gave you a very nice function (I know - its ObjC]. – Anthesis

Answer 1

I converted Andrea's answer into Swift 3.1:

static func DegreesToRadians(_ degrees: CGFloat) -> CGFloat { return CGFloat( (degrees * .pi) / 180 ) }

static func CreateCGImageFromCVPixelBuffer(pixelBuffer: CVPixelBuffer) -> CGImage? {
    let bitmapInfo: CGBitmapInfo
    let sourcePixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer)
    if kCVPixelFormatType_32ARGB == sourcePixelFormat {
        bitmapInfo = [.byteOrder32Big, CGBitmapInfo(rawValue: CGImageAlphaInfo.noneSkipFirst.rawValue)]
    } else
    if kCVPixelFormatType_32BGRA == sourcePixelFormat {
        bitmapInfo = [.byteOrder32Little, CGBitmapInfo(rawValue: CGImageAlphaInfo.noneSkipFirst.rawValue)]
    } else {
        return nil
    }

    // only uncompressed pixel formats
    let sourceRowBytes = CVPixelBufferGetBytesPerRow(pixelBuffer)
    let width = CVPixelBufferGetWidth(pixelBuffer)
    let height = CVPixelBufferGetHeight(pixelBuffer)
    print("Buffer image size \(width) height \(height)")

    let val: CVReturn = CVPixelBufferLockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
    if  val == kCVReturnSuccess,
        let sourceBaseAddr = CVPixelBufferGetBaseAddress(pixelBuffer),
        let provider = CGDataProvider(dataInfo: nil, data: sourceBaseAddr, size: sourceRowBytes * height, releaseData: {_,_,_ in })
    {
        let colorspace = CGColorSpaceCreateDeviceRGB()
        let image = CGImage(width: width, height: height, bitsPerComponent: 8, bitsPerPixel: 32, bytesPerRow: sourceRowBytes,
                        space: colorspace, bitmapInfo: bitmapInfo, provider: provider, decode: nil,
                        shouldInterpolate: true, intent: CGColorRenderingIntent.defaultIntent)
        CVPixelBufferUnlockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
        return image
    } else {
        return nil
    }
}
// utility used by newSquareOverlayedImageForFeatures for
static func CreateCGBitmapContextForSize(_ size: CGSize) -> CGContext? {
    let bitmapBytesPerRow = Int(size.width * 4)
    let colorSpace = CGColorSpaceCreateDeviceRGB()

    guard let context = CGContext(data: nil, width: Int(size.width), height: Int(size.height), bitsPerComponent: 8,
                    bytesPerRow: bitmapBytesPerRow, space: colorSpace, bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)
    else { return nil }
    context.setAllowsAntialiasing(false)
    return context
}

Anthesis

App is getting crashed Terminated due to memory issue, any idea how to fix it ? – Interjection

@sharadchauhan probably a leak - you are keeping a strong reference to one of the buffer types. – Anthesis

Answer 2

I don't know in SWIFT, but I think that you can easily convert, this C function that was taken from Apple and works perfectly.
The problem using CIImage is that create a context is quite an expensive task, so if you want to go that way is better to build the context before everything and keep a strong reference to it.
Furthermore I dont' remeber if the default context is build for GPU or CPU, there are other subtile differences between the 2. For instance if you want to make the image creation on a background thread on GPU it won't work.

static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;};

static void ReleaseCVPixelBuffer(void *pixel, const void *data, size_t size)
{
    CVPixelBufferRef pixelBuffer = (CVPixelBufferRef)pixel;
    CVPixelBufferUnlockBaseAddress( pixelBuffer, 0 );
    CVPixelBufferRelease( pixelBuffer );
}

// create a CGImage with provided pixel buffer, pixel buffer must be uncompressed kCVPixelFormatType_32ARGB or kCVPixelFormatType_32BGRA
static OSStatus CreateCGImageFromCVPixelBuffer(CVPixelBufferRef pixelBuffer, CGImageRef *imageOut)
{
    OSStatus err = noErr;
    OSType sourcePixelFormat;
    size_t width, height, sourceRowBytes;
    void *sourceBaseAddr = NULL;
    CGBitmapInfo bitmapInfo;
    CGColorSpaceRef colorspace = NULL;
    CGDataProviderRef provider = NULL;
    CGImageRef image = NULL;

    sourcePixelFormat = CVPixelBufferGetPixelFormatType( pixelBuffer );
    if ( kCVPixelFormatType_32ARGB == sourcePixelFormat )
        bitmapInfo = kCGBitmapByteOrder32Big | kCGImageAlphaNoneSkipFirst;
    else if ( kCVPixelFormatType_32BGRA == sourcePixelFormat )
        bitmapInfo = kCGBitmapByteOrder32Little | kCGImageAlphaNoneSkipFirst;
    else
        return -95014; // only uncompressed pixel formats

    sourceRowBytes = CVPixelBufferGetBytesPerRow( pixelBuffer );
    width = CVPixelBufferGetWidth( pixelBuffer );
    height = CVPixelBufferGetHeight( pixelBuffer );
    DLog(@"Buffer image size %zu e %zu",width,height );
    CVReturn val = CVPixelBufferLockBaseAddress( pixelBuffer, 0 );
    if (val == kCVReturnSuccess) {

        sourceBaseAddr = CVPixelBufferGetBaseAddress( pixelBuffer );

        colorspace = CGColorSpaceCreateDeviceRGB();

        CVPixelBufferRetain( pixelBuffer );
        provider = CGDataProviderCreateWithData( (void *)pixelBuffer, sourceBaseAddr, sourceRowBytes * height, ReleaseCVPixelBuffer);
        image = CGImageCreate(width, height, 8, 32, sourceRowBytes, colorspace, bitmapInfo, provider, NULL, true, kCGRenderingIntentDefault);
    }
bail:
    if ( err && image ) {
        CGImageRelease( image );
        image = NULL;
    }
    if ( provider ) CGDataProviderRelease( provider );
    if ( colorspace ) CGColorSpaceRelease( colorspace );
    *imageOut = image;
    return err;
}

// utility used by newSquareOverlayedImageForFeatures for
static CGContextRef CreateCGBitmapContextForSize(CGSize size)
{
    CGContextRef    context = NULL;
    CGColorSpaceRef colorSpace;
    int             bitmapBytesPerRow;

    bitmapBytesPerRow = (size.width * 4);

    colorSpace = CGColorSpaceCreateDeviceRGB();
    context = CGBitmapContextCreate (NULL,
                                     size.width,
                                     size.height,
                                     8,      // bits per component
                                     bitmapBytesPerRow,
                                     colorSpace,
                                     kCGImageAlphaPremultipliedLast);
    CGContextSetAllowsAntialiasing(context, NO);
    CGColorSpaceRelease( colorSpace );
    return context;
}

Clammy

Answer 3

    let cvImageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)

    guard cvImageBuffer != nil else { return  }


    let attachments = CMCopyDictionaryOfAttachments(kCFAllocatorDefault, sampleBuffer, kCMAttachmentMode_ShouldPropagate)
    let ciImage = CIImage(cvImageBuffer: cvImageBuffer!, options: attachments as! [String : Any]?)

    let image = UIImage(ciImage: ciImage)

Perigee