ios - 在 iOS 中使用 glsl 将 yv12 转换为 rgb，附上结果图像

Question

我使用 glsl 着色器将 yv12 帧数据转换为 rgb 数据，原始图像如下：在此处输入图像描述

但结果图像与前者不一样，附在下面：在此处输入图像描述

以下是我将三个平面数据上传到纹理的代码：

- (GLuint) textureY: (Byte*)imageData        
      widthType: (int) width       
     heightType: (int) height       
{          
    GLuint texName;    
    glGenTextures( 1, &texName );     
    glBindTexture(GL_TEXTURE_2D, texName);

    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );    
    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );    
    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);    
    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);    

    glTexImage2D( GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, imageData );  
    //free(imageData);

    return texName;    
}    

- (GLuint) textureU: (Byte*)imageData        
          widthType: (int) width       
         heightType: (int) height       
{          
    GLuint texName;    

    glGenTextures( 1, &texName );     
    glBindTexture(GL_TEXTURE_2D, texName);


    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );    
    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );    
    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);    
    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);    

    glTexImage2D( GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, imageData );    

    //free(imageData);
    return texName;    
}    

- (GLuint) textureV: (Byte*)imageData        
          widthType: (int) width       
         heightType: (int) height       
{          
    GLuint texName;    
    glGenTextures( 1, &texName );     
    glBindTexture(GL_TEXTURE_2D, texName);


    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );    
    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );    
    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);    
    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);    

    glTexImage2D( GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, imageData );    

    //free(imageData);
    return texName;    
}    


- (void) readYUVFile     
{    
    NSString *file = [[NSBundle mainBundle] pathForResource:@"video" ofType:@"yv12"];
    NSLog(@"%@",file);
    NSData* fileData = [NSData dataWithContentsOfFile:file]; 
    //NSLog(@"%@",[fileData description]);
    NSInteger width  = 352;    
    NSInteger height = 288;
    NSInteger uv_width  = width  / 2;    
    NSInteger uv_height = height / 2;
    NSInteger dataSize = [fileData length];
    NSLog(@"%i\n",dataSize);

    GLint nYsize  = width * height;     
    GLint nUVsize = uv_width * uv_height;      
    GLint nCbOffSet = nYsize;    
    GLint nCrOffSet = nCbOffSet + nUVsize;    

    Byte *spriteData = (Byte *)malloc(dataSize);
    [fileData getBytes:spriteData length:dataSize];


    Byte* uData = spriteData + nCbOffSet;
    //NSLog(@"%@\n",[[NSData dataWithBytes:uData length:nUVsize] description]);
    Byte* vData = spriteData + nCrOffSet;  
    //NSLog(@"%@\n",[[NSData dataWithBytes:vData length:nUVsize] description]);
    /**
    Byte *YPlanarData = (Byte *)malloc(nYsize);
    for (int i=0; i<nYsize; i++) {
        YPlanarData[i]= spriteData[i];
    }    

    Byte *UPlanarData = (Byte *)malloc(nYsize);
    for (int i=0; i<height; i++) {
        for (int j=0; j<width; j++) {
            int numInUVsize = (i/2)*uv_width+j/2;
            UPlanarData[i*width+j]=uData[numInUVsize];
        }
    }

    Byte *VPlanarData = (Byte *)malloc(nYsize);
    for (int i=0; i<height; i++) {
        for (int j=0; j<width; j++) {
            int numInUVsize = (i/2)*uv_width+j/2;
            VPlanarData[i*width+j]=vData[numInUVsize];
        }
    }
    **/

    _YPlanarTexture = [self textureY:spriteData widthType:width heightType:height];    
    _UPlanarTexture = [self textureU:uData widthType:uv_width  heightType:uv_height];    
    _VPlanarTexture = [self textureV:vData widthType:uv_width heightType:uv_height];       

    free(spriteData);

}

和我的片段着色器代码：

   precision highp float;
uniform sampler2D SamplerY;
uniform sampler2D SamplerU;
uniform sampler2D SamplerV;

varying highp vec2 coordinate;

void main()
{
    highp vec3 yuv,yuv1;
    highp vec3 rgb;

    yuv.x = texture2D(SamplerY, coordinate).r;

    yuv.y = texture2D(SamplerU, coordinate).r-0.5;

    yuv.z = texture2D(SamplerV, coordinate).r-0.5 ;

   rgb = mat3(      1,       1,      1,
                     0, -.34414, 1.772,
               1.402, -.71414,      0) * yuv;

    gl_FragColor = vec4(rgb, 1);
}

我的困惑是转换公式，而我使用这个公式直接将 yv12 数据转换为 rgb24，并用

CGImageCreate(iwidth, 
                                       iheight, 
                                       8, 
                                       24, 
                                       iwidth*3, 
                                       colorSpace, 
                                       bitmapInfo, 
                                       provider, 
                                       NULL, 
                                       NO, 
                                       kCGRenderingIntentDefault);

结果图像是正确的。但是使用着色器（在iOS设备上运行的直接变换方法是转储）转向这个问题，我尝试了一些技巧（将UV刨床扩展到（2 * uv_width）* 2（uv_height）矩形，然后上传纹理)，但在同一张更红的图像中失败了。

如何解决这个问题？

附上我的整个 glView.m 代码：

#import "OpenGLView.h"

typedef struct {
    float Position[3];
    float TexCoord[2];
} Vertex;

const Vertex Vertices[] = {
    {{1, -1, 0},{1,1}},
    {{1, 1, 0},{1,0}},
    {{-1, 1, 0},{0,0}},
    {{-1, -1, 0},{0,1}}
};

const GLubyte Indices[] = {
    0, 1, 2,
    2, 3, 0
};



@interface OpenGLView ()
- (void)setupLayer;
- (void)setupContext;
- (void)setupRenderBuffer;
- (void)setupFrameBuffer;
- (void)render;

- (GLuint)compileShader:(NSString*)shaderName withType:(GLenum)shaderType;
- (void)setupVBOs;
- (void)compileShaders;

- (void) readYUVFile;
@end

@implementation OpenGLView

- (void)setupVBOs {

    GLuint vertexBuffer;
    glGenBuffers(1, &vertexBuffer);
    glBindBuffer(GL_ARRAY_BUFFER, vertexBuffer);
    glBufferData(GL_ARRAY_BUFFER, sizeof(Vertices), Vertices, GL_STATIC_DRAW);

    GLuint indexBuffer;
    glGenBuffers(1, &indexBuffer);
    glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, indexBuffer);
    glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(Indices), Indices, GL_STATIC_DRAW);

}



- (id)initWithFrame:(CGRect)frame
{
    self = [super initWithFrame:frame];
    if (self) {
        // Initialization code[]
        self.backgroundColor = [UIColor redColor];
        [self setupLayer];
        [self setupContext];
        [self setupRenderBuffer];
        [self setupFrameBuffer];

        [self setupVBOs];
        [self compileShaders];
        [self readYUVFile];

        [self render];

    }
    return self;
}

+ (Class)layerClass{
    return [CAEAGLLayer class];
}

-(void)setupLayer{
    _eaglLayer = (CAEAGLLayer *)self.layer;
    _eaglLayer.opaque = YES;
}

- (void)setupContext{
    EAGLRenderingAPI api = kEAGLRenderingAPIOpenGLES2;
    _context = [[[EAGLContext alloc] initWithAPI:api] autorelease];

    if (!_context) {
        NSLog(@"Failed to initialize OpenGLES 2.0 context");
        exit(1);
    }

    if (![EAGLContext setCurrentContext:_context]) {
        NSLog(@"Failed to set current OpenGL context");
        exit(1);
    }
}

- (void)setupRenderBuffer {
    glGenRenderbuffers(1, &_colorRenderBuffer);
    glBindRenderbuffer(GL_RENDERBUFFER, _colorRenderBuffer);        
    [_context renderbufferStorage:GL_RENDERBUFFER fromDrawable:_eaglLayer];    
}

- (void)setupFrameBuffer {    
    GLuint framebuffer;
    glGenFramebuffers(1, &framebuffer);
    glBindFramebuffer(GL_FRAMEBUFFER, framebuffer);
    glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, 
                              GL_RENDERBUFFER, _colorRenderBuffer);
}

- (GLuint) textureY: (Byte*)imageData        
          widthType: (int) width       
         heightType: (int) height       
{          
    GLuint texName;    
    glGenTextures( 1, &texName );     
    glBindTexture(GL_TEXTURE_2D, texName);

    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );    
    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );    
    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);    
    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);    

    glTexImage2D( GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, imageData );  
    //free(imageData);

    return texName;    
}    

- (GLuint) textureU: (Byte*)imageData        
          widthType: (int) width       
         heightType: (int) height       
{          
    GLuint texName;    

    glGenTextures( 1, &texName );     
    glBindTexture(GL_TEXTURE_2D, texName);


    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );    
    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );    
    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);    
    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);    

    glTexImage2D( GL_TEXTURE_2D, 0, GL_RED_EXT, width, height, 0, GL_RED_EXT, GL_UNSIGNED_BYTE, imageData );    

    //free(imageData);
    return texName;    
}    

- (GLuint) textureV: (Byte*)imageData        
          widthType: (int) width       
         heightType: (int) height       
{          
    GLuint texName;    
    glGenTextures( 1, &texName );     
    glBindTexture(GL_TEXTURE_2D, texName);


    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );    
    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );    
    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);    
    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);    

    glTexImage2D( GL_TEXTURE_2D, 0, GL_RED_EXT, width, height, 0, GL_RED_EXT, GL_UNSIGNED_BYTE, imageData );    

    //free(imageData);
    return texName;    
}    


- (void) readYUVFile     
{    
    NSString *file = [[NSBundle mainBundle] pathForResource:@"video" ofType:@"yv12"];
    NSLog(@"%@",file);
    NSData* fileData = [NSData dataWithContentsOfFile:file]; 
    //NSLog(@"%@",[fileData description]);
    NSInteger width  = 352;    
    NSInteger height = 288;
    NSInteger uv_width  = width  / 2;    
    NSInteger uv_height = height / 2;
    NSInteger dataSize = [fileData length];
    NSLog(@"%i\n",dataSize);

    GLint nYsize  = width * height;     
    GLint nUVsize = uv_width * uv_height;      
    GLint nCbOffSet = nYsize;    
    GLint nCrOffSet = nCbOffSet + nUVsize;    

    Byte *spriteData = (Byte *)malloc(dataSize);
    [fileData getBytes:spriteData length:dataSize];


    Byte* uData = spriteData + nCbOffSet;
    //NSLog(@"%@\n",[[NSData dataWithBytes:uData length:nUVsize] description]);
    Byte* vData = spriteData + nCrOffSet;  
    //NSLog(@"%@\n",[[NSData dataWithBytes:vData length:nUVsize] description]);

    Byte *YPlanarData = (Byte *)malloc(nYsize);
    for (int i=0; i<nYsize; i++) {
        YPlanarData[i]= spriteData[i];
    }    

    Byte *UPlanarData = (Byte *)malloc(nYsize);
    for (int i=0; i<height; i++) {
        for (int j=0; j<width; j++) {
            int numInUVsize = (i/2)*uv_width+j/2;
            UPlanarData[i*width+j]=uData[numInUVsize];
        }
    }

    Byte *VPlanarData = (Byte *)malloc(nYsize);
    for (int i=0; i<height; i++) {
        for (int j=0; j<width; j++) {
            int numInUVsize = (i/2)*uv_width+j/2;
            VPlanarData[i*width+j]=vData[numInUVsize];
        }
    }


    _YPlanarTexture = [self textureY:YPlanarData widthType:width heightType:height];    
    _UPlanarTexture = [self textureU:UPlanarData widthType:width  heightType:height];    
    _VPlanarTexture = [self textureV:VPlanarData widthType:width heightType:height];       

    free(spriteData);

}



- (void)render {
    glClearColor(0,0,0 , 1.0);
    glClear(GL_COLOR_BUFFER_BIT);

    // 1
    glViewport(0, 200, self.frame.size.width, 558);

    // 2
    glVertexAttribPointer(_positionSlot, 3, GL_FLOAT, GL_FALSE, 
                          sizeof(Vertex), 0);


    glVertexAttribPointer(_texCoordSlot, 2, GL_FLOAT, GL_FALSE, 
                          sizeof(Vertex), (GLvoid*) (sizeof(float) *3));    

    glActiveTexture(GL_TEXTURE0); 
    glBindTexture(GL_TEXTURE_2D, _YPlanarTexture);
    glUniform1i(_textureUniformY, 0);

    glActiveTexture(GL_TEXTURE1); 
    glBindTexture(GL_TEXTURE_2D, _UPlanarTexture);
    glUniform1i(_textureUniformU, 1);

    glActiveTexture(GL_TEXTURE2); 
    glBindTexture(GL_TEXTURE_2D, _VPlanarTexture);
    glUniform1i(_textureUniformV, 2);


    // 3
    glDrawElements(GL_TRIANGLES, sizeof(Indices)/sizeof(Indices[0]), 
                   GL_UNSIGNED_BYTE, 0);

    [_context presentRenderbuffer:GL_RENDERBUFFER];
}


/*
// Only override drawRect: if you perform custom drawing.
// An empty implementation adversely affects performance during animation.
- (void)drawRect:(CGRect)rect
{
    // Drawing code
}
*/

- (GLuint)compileShader:(NSString*)shaderName withType:(GLenum)shaderType {

    // 1
    NSString* shaderPath = [[NSBundle mainBundle] pathForResource:shaderName 
                                                           ofType:@"glsl"];
    NSError* error;
    NSString* shaderString = [NSString stringWithContentsOfFile:shaderPath 
                                                       encoding:NSUTF8StringEncoding error:&error];
    if (!shaderString) {
        NSLog(@"Error loading shader: %@", error.localizedDescription);
        exit(1);
    }

    // 2
    GLuint shaderHandle = glCreateShader(shaderType);    

    // 3
    const char* shaderStringUTF8 = [shaderString UTF8String];    
    int shaderStringLength = [shaderString length];
    glShaderSource(shaderHandle, 1, &shaderStringUTF8, &shaderStringLength);

    // 4
    glCompileShader(shaderHandle);

    // 5
    GLint compileSuccess;
    glGetShaderiv(shaderHandle, GL_COMPILE_STATUS, &compileSuccess);
    if (compileSuccess == GL_FALSE) {
        GLchar messages[256];
        glGetShaderInfoLog(shaderHandle, sizeof(messages), 0, &messages[0]);
        NSString *messageString = [NSString stringWithUTF8String:messages];
        NSLog(@"%@", messageString);
        exit(1);
    }

    return shaderHandle;

}

- (void)compileShaders {

    // 1
    GLuint vertexShader = [self compileShader:@"SimpleVertex" 
                                     withType:GL_VERTEX_SHADER];
    GLuint fragmentShader = [self compileShader:@"SimpleFragment" 
                                       withType:GL_FRAGMENT_SHADER];

    // 2
    GLuint programHandle = glCreateProgram();
    glAttachShader(programHandle, vertexShader);
    glAttachShader(programHandle, fragmentShader);
    glLinkProgram(programHandle);

    // 3
    GLint linkSuccess;
    glGetProgramiv(programHandle, GL_LINK_STATUS, &linkSuccess);
    if (linkSuccess == GL_FALSE) {
        GLchar messages[256];
        glGetProgramInfoLog(programHandle, sizeof(messages), 0, &messages[0]);
        NSString *messageString = [NSString stringWithUTF8String:messages];
        NSLog(@"%@", messageString);
        exit(1);
    }

    // 4
    glUseProgram(programHandle);

    // 5
    _positionSlot = glGetAttribLocation(programHandle, "position");
    glEnableVertexAttribArray(_positionSlot);


    _texCoordSlot = glGetAttribLocation(programHandle, "textureCoordinate");
    glEnableVertexAttribArray(_texCoordSlot);

    _YPlanarTexture = glGetUniformLocation(programHandle, "SamplerY");
    _UPlanarTexture = glGetUniformLocation(programHandle, "SamplerU");
    _VPlanarTexture = glGetUniformLocation(programHandle, "SamplerV");



}




@end

score 2 · Accepted Answer

我的愚蠢错误，三个上传纹理过程和片段着色器是正确的，但以下代码不兼容：

glActiveTexture(GL_TEXTURE0); 
glBindTexture(GL_TEXTURE_2D, _YPlanarTexture);
glUniform1i(_textureUniformY, 0);

glActiveTexture(GL_TEXTURE1); 
glBindTexture(GL_TEXTURE_2D, _UPlanarTexture);
glUniform1i(_textureUniformU, 1);

glActiveTexture(GL_TEXTURE2); 
glBindTexture(GL_TEXTURE_2D, _VPlanarTexture);
glUniform1i(_textureUniformV, 2);

及以下：

_YPlanarTexture = glGetUniformLocation(programHandle, "SamplerY");
_UPlanarTexture = glGetUniformLocation(programHandle, "SamplerU");
_VPlanarTexture = glGetUniformLocation(programHandle, "SamplerV");

所以用这个替换：

_textureUniformY = glGetUniformLocation(programHandle, "SamplerY");
_textureUniformU = glGetUniformLocation(programHandle, "SamplerU");
_textureUniformV = glGetUniformLocation(programHandle, "SamplerV");

那么它会做正确的事情。

ios - 在 iOS 中使用 glsl 将 yv12 转换为 rgb，附上结果图像

1 回答 1

Related

Reference