14

我在 iPhone 上使用 OpenCV 2.2 来检测人脸。我正在使用 IOS 4 的 AVCaptureSession 来访问摄像头流，如下面的代码所示。

我的挑战是视频帧以 CVBufferRef（指向 CVImageBuffer 的指针）对象的形式出现，它们以风景的形式出现，480px 宽 x 300px 高。如果您将手机横向握持，这很好，但是当手机处于直立位置时，我想将这些框架顺时针旋转 90 度，以便 OpenCV 可以正确找到面部。

我可以将 CVBufferRef 转换为 CGImage，然后转换为 UIImage，然后旋转，就像这个人正在做的那样：Rotate CGImage take from video frame

但是，这会浪费大量 CPU。我正在寻找一种更快的方法来旋转传入的图像，如果可能的话，最好使用 GPU 进行此处理。

有任何想法吗？

伊恩

代码示例：

 -(void) startCameraCapture {
  // Start up the face detector

  faceDetector = [[FaceDetector alloc] initWithCascade:@"haarcascade_frontalface_alt2" withFileExtension:@"xml"];

  // Create the AVCapture Session
  session = [[AVCaptureSession alloc] init];

  // create a preview layer to show the output from the camera
  AVCaptureVideoPreviewLayer *previewLayer = [AVCaptureVideoPreviewLayer layerWithSession:session];
  previewLayer.frame = previewView.frame;
  previewLayer.videoGravity = AVLayerVideoGravityResizeAspectFill;

  [previewView.layer addSublayer:previewLayer];

  // Get the default camera device
  AVCaptureDevice* camera = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];

  // Create a AVCaptureInput with the camera device
  NSError *error=nil;
  AVCaptureInput* cameraInput = [[AVCaptureDeviceInput alloc] initWithDevice:camera error:&error];
  if (cameraInput == nil) {
   NSLog(@"Error to create camera capture:%@",error);
  }

  // Set the output
  AVCaptureVideoDataOutput* videoOutput = [[AVCaptureVideoDataOutput alloc] init];
  videoOutput.alwaysDiscardsLateVideoFrames = YES;

  // create a queue besides the main thread queue to run the capture on
  dispatch_queue_t captureQueue = dispatch_queue_create("catpureQueue", NULL);

  // setup our delegate
  [videoOutput setSampleBufferDelegate:self queue:captureQueue];

  // release the queue.  I still don't entirely understand why we're releasing it here,
  // but the code examples I've found indicate this is the right thing.  Hmm...
  dispatch_release(captureQueue);

  // configure the pixel format
  videoOutput.videoSettings = [NSDictionary dictionaryWithObjectsAndKeys:
          [NSNumber numberWithUnsignedInt:kCVPixelFormatType_32BGRA], 
          (id)kCVPixelBufferPixelFormatTypeKey,
          nil];

  // and the size of the frames we want
  // try AVCaptureSessionPresetLow if this is too slow...
  [session setSessionPreset:AVCaptureSessionPresetMedium];

  // If you wish to cap the frame rate to a known value, such as 10 fps, set 
  // minFrameDuration.
  videoOutput.minFrameDuration = CMTimeMake(1, 10);

  // Add the input and output
  [session addInput:cameraInput];
  [session addOutput:videoOutput];

  // Start the session
  [session startRunning];  
 }

 - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection {
  // only run if we're not already processing an image
  if (!faceDetector.imageNeedsProcessing) {

   // Get CVImage from sample buffer
   CVImageBufferRef cvImage = CMSampleBufferGetImageBuffer(sampleBuffer);

   // Send the CVImage to the FaceDetector for later processing
   [faceDetector setImageFromCVPixelBufferRef:cvImage];

   // Trigger the image processing on the main thread
   [self performSelectorOnMainThread:@selector(processImage) withObject:nil waitUntilDone:NO];
  }
 }
4

4 回答 4

17

vImage 是一种非常快速的方法。不过需要ios5。该调用显示 ARGB，但它适用于您从缓冲区获得的 BGRA。

这也有一个优点，您可以切出缓冲区的一部分并旋转它。在这里查看我的答案

- (unsigned char*) rotateBuffer: (CMSampleBufferRef) sampleBuffer
{
 CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
 CVPixelBufferLockBaseAddress(imageBuffer,0);

 size_t bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer);
 size_t width = CVPixelBufferGetWidth(imageBuffer);
 size_t height = CVPixelBufferGetHeight(imageBuffer);
 size_t currSize = bytesPerRow*height*sizeof(unsigned char); 
 size_t bytesPerRowOut = 4*height*sizeof(unsigned char); 

 void *srcBuff = CVPixelBufferGetBaseAddress(imageBuffer); 
 unsigned char *outBuff = (unsigned char*)malloc(currSize);  

 vImage_Buffer ibuff = { srcBuff, height, width, bytesPerRow};
 vImage_Buffer ubuff = { outBuff, width, height, bytesPerRowOut};

 uint8_t rotConst = 1;   // 0, 1, 2, 3 is equal to 0, 90, 180, 270 degrees rotation

 vImage_Error err= vImageRotate90_ARGB8888 (&ibuff, &ubuff, NULL, rotConst, NULL,0);
 if (err != kvImageNoError) NSLog(@"%ld", err);

 return outBuff;
}
于 2012-09-07T09:43:31.037 回答
3

以您想要的方式设置视频方向可能更容易：

connection.videoOrientation = AVCaptureVideoOrientationPortrait

这样你根本不需要做那种旋转噱头

于 2015-03-25T17:41:33.477 回答
3

如果您在 90 度停止处旋转，那么您可以在内存中进行。这是示例代码，它只是简单地将数据复制到新的像素缓冲区。进行蛮力旋转应该是直截了当的。

- (CVPixelBufferRef) rotateBuffer: (CMSampleBufferRef) sampleBuffer
{
    CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
    CVPixelBufferLockBaseAddress(imageBuffer,0);

    size_t bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer);
    size_t width = CVPixelBufferGetWidth(imageBuffer);
    size_t height = CVPixelBufferGetHeight(imageBuffer);

    void *src_buff = CVPixelBufferGetBaseAddress(imageBuffer);

    NSDictionary *options = [NSDictionary dictionaryWithObjectsAndKeys:
                             [NSNumber numberWithBool:YES], kCVPixelBufferCGImageCompatibilityKey,
                             [NSNumber numberWithBool:YES], kCVPixelBufferCGBitmapContextCompatibilityKey,
                             nil];

    CVPixelBufferRef pxbuffer = NULL;
    //CVReturn status = CVPixelBufferPoolCreatePixelBuffer (NULL, _pixelWriter.pixelBufferPool, &pxbuffer);
    CVReturn status = CVPixelBufferCreate(kCFAllocatorDefault, width,
                                          height, kCVPixelFormatType_32BGRA, (CFDictionaryRef) options, 
                                          &pxbuffer);

    NSParameterAssert(status == kCVReturnSuccess && pxbuffer != NULL);

    CVPixelBufferLockBaseAddress(pxbuffer, 0);
    void *dest_buff = CVPixelBufferGetBaseAddress(pxbuffer);
    NSParameterAssert(dest_buff != NULL);

    int *src = (int*) src_buff ;
    int *dest= (int*) dest_buff ;
    size_t count = (bytesPerRow * height) / 4 ;
    while (count--) {
        *dest++ = *src++;
    }

    //Test straight copy.
    //memcpy(pxdata, baseAddress, width * height * 4) ;
    CVPixelBufferUnlockBaseAddress(pxbuffer, 0);
    CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
    return pxbuffer;
}

然后，如果要将其写回 AVAssetWriterInput，则可以使用 AVAssetWriterInputPixelBufferAdaptor。

上面没有优化。您可能想要寻找更有效的复制算法。一个很好的起点是In-place Matrix Transpose。您还想使用像素缓冲池，而不是每次都创建一个新缓冲池。

编辑。您可以使用 GPU 来执行此操作。这听起来像是大量数据被推送。在 CVPixelBufferRef 中有键 kCVPixelBufferOpenGLCompatibilityKey。我假设您可以从 CVImageBufferRef（它只是一个像素缓冲区 ref）创建一个 OpenGL 兼容的图像，然后通过着色器将其推送。再次，过度杀伤 IMO。您可能会看到 BLAS 或 LAPACK 是否有“不合适”的转置方法。如果他们这样做，那么您可以放心，他们是高度优化的。

90 CW where new_width = width ...这将为您提供纵向图像。

for (int i = 1; i <= new_height; i++) {
    for (int j = new_width - 1; j > -1; j--) {
        *dest++ = *(src + (j * width) + i) ;
    }
}
于 2011-01-02T06:12:52.883 回答
3

我知道这是一个很老的问题，但我最近一直在解决类似的问题，也许有人会发现我的解决方案很有用。

我需要从 iPhone 摄像头提供的 YCbCr 格式的图像缓冲区中提取原始图像数据（从 [AVCaptureVideoDataOutput.availableVideoCVPixelFormatTypes firstObject] 获取），删除诸如标头、元信息等信息以将其传递给进一步处理。

此外，我只需要提取捕获的视频帧中心的小区域，因此需要进行一些裁剪。

我的条件只允许在任一横向拍摄视频，但是当设备位于横向左侧时，图像被颠倒传送，所以我需要在两个轴上翻转它。如果图像被翻转，我的想法是以相反的顺序从源图像缓冲区复制数据，并在每行读取数据中反转字节以在两个轴上翻转图像。这个想法确实有效，因为无论如何我都需要从源缓冲区复制数据，所以如果从头或尾读取似乎没有太大的性能损失（当然，更大的图像=更长的处理时间，但我处理的数字非常小） .

我想知道其他人对此解决方案的看法，当然还有一些提示如何改进代码：

/// Lock pixel buffer
CVPixelBufferLockBaseAddress(imageBuffer, 0);

/// Address where image buffer starts
uint8_t *baseAddress = (uint8_t *)CVPixelBufferGetBaseAddress(imageBuffer);

/// Read image parameters
size_t width = CVPixelBufferGetWidth(imageBuffer);
size_t height = CVPixelBufferGetHeight(imageBuffer);

/// See whether image is flipped upside down
BOOL isFlipped = (_previewLayer.connection.videoOrientation == AVCaptureVideoOrientationLandscapeLeft);

/// Calculate cropping frame. Crop to scanAreaSize (defined as CGSize constant elsewhere) from the center of an image
CGRect cropFrame = CGRectZero;
cropFrame.size = scanAreaSize;
cropFrame.origin.x = (width / 2.0f) - (scanAreaSize.width / 2.0f);
cropFrame.origin.y = (height / 2.0f) - (scanAreaSize.height / 2.0f);

/// Update proportions to cropped size
width = (size_t)cropFrame.size.width;
height = (size_t)cropFrame.size.height;

/// Allocate memory for output image data. W*H for Y component, W*H/2 for CbCr component
size_t bytes = width * height + (width * height / 2);

uint8_t *outputDataBaseAddress = (uint8_t *)malloc(bytes);

if(outputDataBaseAddress == NULL) {

    /// Memory allocation failed, unlock buffer and give up
    CVPixelBufferUnlockBaseAddress(imageBuffer, 0);

    return NULL;
}

/// Get parameters of YCbCr pixel format
CVPlanarPixelBufferInfo_YCbCrBiPlanar *bufferInfo = (CVPlanarPixelBufferInfo_YCbCrBiPlanar *)baseAddress;

NSUInteger bytesPerRowY = EndianU32_BtoN(bufferInfo->componentInfoY.rowBytes);
NSUInteger offsetY = EndianU32_BtoN(bufferInfo->componentInfoY.offset);

NSUInteger bytesPerRowCbCr = EndianU32_BtoN(bufferInfo->componentInfoCbCr.rowBytes);
NSUInteger offsetCbCr = EndianU32_BtoN(bufferInfo->componentInfoCbCr.offset);

/// Copy image data only, skipping headers and metadata. Create single buffer which will contain Y component data
/// followed by CbCr component data.

/// Process Y component
/// Pointer to the source buffer
uint8_t *src;

/// Pointer to the destination buffer
uint8_t *destAddress;

/// Calculate crop rect offset. Crop offset is number of rows (y * bytesPerRow) + x offset.
/// If image is flipped, then read buffer from the end to flip image vertically. End address is height-1!
int flipOffset = (isFlipped) ? (int)((height - 1) * bytesPerRowY) : 0;

int cropOffset = (int)((cropFrame.origin.y * bytesPerRowY) + flipOffset + cropFrame.origin.x);

/// Set source pointer to Y component buffer start address plus crop rect offset
src = baseAddress + offsetY + cropOffset;

for(int y = 0; y < height; y++) {

    /// Copy one row of pixel data from source into the output buffer.
    destAddress = (outputDataBaseAddress + y * width);

    memcpy(destAddress, src, width);

    if(isFlipped) {

        /// Reverse bytes in row to flip image horizontally
        [self reverseBytes:destAddress bytesSize:(int)width];

        /// Move one row up
        src -= bytesPerRowY;
    }
    else {

        /// Move to the next row
        src += bytesPerRowY;
    }
}

/// Calculate crop offset for CbCr component
flipOffset = (isFlipped) ? (int)(((height - 1) / 2) * bytesPerRowCbCr) : 0;
cropOffset = (int)((cropFrame.origin.y * bytesPerRowCbCr) + flipOffset + cropFrame.origin.x);

/// Set source pointer to the CbCr component offset + crop offset
src = (baseAddress + offsetCbCr + cropOffset);

for(int y = 0; y < (height / 2); y++) {

    /// Copy one row of pixel data from source into the output buffer.
    destAddress = (outputDataBaseAddress + (width * height) + y * width);

    memcpy(destAddress, src, width);

    if(isFlipped) {

        /// Reverse bytes in row to flip image horizontally
        [self reverseBytes:destAddress bytesSize:(int)width];

        /// Move one row up
        src -= bytesPerRowCbCr;
    }
    else {

        src += bytesPerRowCbCr;
    }
}

/// Unlock pixel buffer
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);

/// Continue with image data in outputDataBaseAddress;
于 2015-06-03T22:02:41.140 回答