0

我正在用 android 开发相机预览应用程序。
我的应用流程如下所示。

1) JAVA：获取相机预览缓冲区并将其传递给 JNI
2) CPP：使用 OpenGL ES 3.0 从相机预览缓冲区制作纹理
3) CPP：使用 OpenGL ES 3.0 渲染纹理

所以，我的应用程序在正常的纹理更新/渲染解决方法下运行良好。没关系。

问题是性能。我必须将每一帧更新到 GPU 内存，glTexSubImage2D(...)而且速度有点慢。
所以我正在尝试使用 PBO，但渲染的屏幕总是空白。

我不知道我错过了什么，有人可以帮忙吗？

这是我的代码：

#include "native-lib.h"
#include "textureloader.h"
#include "vecmath.h"
#include "glutil.h"
#include "logger.h"

GLuint program;
GLuint p_mvp, p_vertices, p_uvs, p_tex;
GLuint tex;
GLuint pboIds[2];
GLfloat vertices[] = {0,0,0,0,0,0,0,0};
GLfloat uvs[] = {0,1,1,1,1,0,0,0};
GLushort indices[] = {0,1,2,0,2,3};
int dataSize;
float zoom;
mat4 mvp;

bool usePBO = true;
bool doublePBO = false;

cv::Mat mat;

std::string vs =
        "#version 300 es\n"
                "in vec4 a_vertices;\n"
                "in vec2 a_uvs;\n"
                "uniform mat4 u_mvp;\n"
                "out vec2 v_texCoord;\n"
                "void main(){\n"
                "  gl_Position = u_mvp * a_vertices;\n"
                "  v_texCoord = a_uvs;\n"
                "}\n";
std::string fs =
        "#version 300 es\n"
                "uniform sampler2D u_tex;\n"
                "in vec2 v_texCoord;\n"
                "out vec4 fragColor;\n"
                "void main(){\n"
                "  fragColor = texture(u_tex, v_texCoord);\n"
                "  fragColor.a = 1.0f;\n"
                "}\n";


JNIEXPORT jint JNICALL
Java_com_quram_vmtest3_Native_glInit(JNIEnv *env, jclass)
{
    // create program
    program = loadProgram(vs, fs, false);

    // create texture basic data
    p_vertices = glGetAttribLocation(program, "a_vertices");
    p_uvs = glGetAttribLocation(program, "a_uvs");
    p_mvp = glGetUniformLocation(program, "u_mvp");
    p_tex = glGetUniformLocation(program, "u_tex");

    // create PBO
    if(usePBO) {
        glGenBuffers(2, pboIds);
    }

    return 1;
}

JNIEXPORT jint JNICALL
Java_com_quram_vmtest3_Native_surfaceChanged(JNIEnv *env, jclass,
                                                   jint w, jint h)
{
    glViewport(0, 0, w, h);

    zoom = (w < h ? w : h) / 2;
    vec4 target = vec4(w / 2, h / 2, 0, 0);
    vec4 eye = target + vec4(0, 0, zoom, 1);
    mat4 view(mat4::lookAt(eye, target, vec4(0, 1, 0, 0)));
    mat4 proj(mat4::perspective(90, (float)w / (float)h, 1, 1000));
    mvp = proj * view;

    vertices[5] = vertices[7] = h;
    vertices[0] = vertices[6] = w;

    glGenTextures(1, texId);
    glBindTexture(GL_TEXTURE_2D, *texId);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, w, h, 0, GL_RGB, GL_UNSIGNED_BYTE, NULL);

    if(usePBO) {
        dataSize = w * h * 3;

        glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[0]);
        glBufferData(GL_PIXEL_UNPACK_BUFFER, dataSize, NULL, GL_STREAM_DRAW);
        glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[1]);
        glBufferData(GL_PIXEL_UNPACK_BUFFER, dataSize, NULL, GL_STREAM_DRAW);
        glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
    }

    return 1;
}


JNIEXPORT jint JNICALL
Java_com_quram_vmtest3_Native_process(JNIEnv *env, jclass, jbyteArray bytearray,
                                      jint w, jint h)
{
    jbyte *buf = env->GetByteArrayElements(bytearray, 0);

    if(mat.empty())
        mat = cv::Mat(h+h/2, w, CV_8UC1, buf);
    else
        mat.data = reinterpret_cast<uchar*>(buf);
    cv::cvtColor(mat, mat, CV_YUV2RGB_NV21);
    rotateMat(mat, mat, -90);

    if(usePBO) {
        static int index = 0;
        if(doublePBO) {
            int nextIndex = 0;

            index = (index + 1) % 2;
            nextIndex = (index + 1) % 2;

            // bind the texture and PBO
            glBindTexture(GL_TEXTURE_2D, tex);
            glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[index]);
            glPixelStorei(GL_UNPACK_ALIGNMENT, 1);

            // copy pixels from PBO to texture object
            // Use offset instead of ponter.
            glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, w, h, GL_RGB, GL_UNSIGNED_BYTE, 0);
            glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[nextIndex]);
            glBufferData(GL_PIXEL_UNPACK_BUFFER, dataSize, 0, GL_STREAM_DRAW);
            GLubyte *ptr = (GLubyte *) glMapBufferRange(GL_PIXEL_UNPACK_BUFFER, 0, dataSize, GL_MAP_WRITE_BIT);
            if (ptr) {
                memcpy(ptr, mat.ptr(), dataSize);
                glUnmapBuffer(GL_PIXEL_UNPACK_BUFFER); // release pointer to mapping buffer
            }
            glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
            glBindTexture(GL_TEXTURE_2D, 0);
        }
        else {
            glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[index]);
            glBindTexture(GL_TEXTURE_2D, tex);
            GLubyte *ptr = (GLubyte *) glMapBufferRange(GL_PIXEL_UNPACK_BUFFER, 0, dataSize, GL_MAP_WRITE_BIT);
            if (ptr) {
                //  memcpy(ptr, mat.ptr(), dataSize);
                memset(ptr, 255, dataSize);
                glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, w, h, GL_RGB, GL_UNSIGNED_BYTE, 0);
                glUnmapBuffer(GL_PIXEL_UNPACK_BUFFER); // release pointer to mapping buffer
            }
            glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
            glBindTexture(GL_TEXTURE_2D, 0);
        }
    }
    else {
        loadTextureWithMat(mat, &tex, GL_RGB, GL_RGB, GL_UNSIGNED_BYTE);
    }

    // enable depth test
    glEnable(GL_DEPTH_TEST);
    glDepthFunc(GL_LEQUAL);

    // enable alpha blending option
    glEnable(GL_BLEND);
    glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);

    glClearColor(1.f, 0.f, 0.f, 0.f);
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

    glUseProgram(program);

    glUniformMatrix4fv(p_mvp, 1, GL_FALSE, &mvp.x.x);
    glEnableVertexAttribArray(p_vertices);
    glVertexAttribPointer(p_vertices, 2, GL_FLOAT, GL_FALSE, 0, vertices);
    glEnableVertexAttribArray(p_uvs);
    glVertexAttribPointer(p_uvs, 2, GL_FLOAT, GL_FALSE, 0, uvs);
    glUniform1i(p_tex, 0);

    glActiveTexture(GL_TEXTURE0);
    glBindTexture(GL_TEXTURE_2D, tex);

    glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices);

    glBindTexture(GL_TEXTURE_2D, 0);
    glUseProgram(0);

    // disable alpha blending option
    glDisable(GL_BLEND);

    // disable depth test
    glDisable(GL_DEPTH_TEST);

    env->ReleaseByteArrayElements(bytearray, buf, JNI_ABORT);
    mat.release();

    return 1;
}

void rotateMat(cv::Mat const &src, cv::Mat &dst, int angle)
{
    if(angle == 270 || angle == -90){
        // Rotate clockwise 270 degrees
        cv::transpose(src, dst);
        cv::flip(dst, dst, 0);
    }else if(angle == 180 || angle == -180){
        // Rotate clockwise 180 degrees
        cv::flip(src, dst, -1);
    }else if(angle == 90 || angle == -270){
        // Rotate clockwise 90 degrees
        cv::transpose(src, dst);
        cv::flip(dst, dst, 1);
    }else if(angle == 360 || angle == 0 || angle == -360){
        if(src.data != dst.data){
            src.copyTo(dst);
        }
    }
}


如果我不使用 PBO，一切正常（fps 除外），所以我认为检查内部代码usePBO和doublePBO标志就足够了。

4

1 回答 1

0

经过一番挣扎，我终于发现了我的微小（但关键）错误。

我旋转了我的 cv::Mat ，因此交换了宽度和高度。

Java_com_quram_vmtest3_Native_process(JNIEnv *env, jclass, jbyteArray bytearray,
                                      jint w, jint h)
{
    mat = cv::Mat(h+h/2, w, CV_8UC1, buf);  // created w*(h*1.5) size mat
    cv::cvtColor(mat, mat, CV_YUV2RGB_NV21); // This makes mat height from h*1.5 to h
    rotateMat(mat, mat, -90);  // Now our mat is h*w
}

我使用了宽度、高度和函数参数值（没有交换）。

glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, w, h, GL_RGB, GL_UNSIGNED_BYTE, 0);

我用这个改变了上面的代码：

glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, mat.cols, mat.rows, GL_RGB, GL_UNSIGNED_BYTE, 0);

是的，一切正常：）

注意：
顺便说一句，性能看起来与以前几乎相同。
也许使用 PBO 并不总能保证性能提升......

于 2017-04-26T07:46:11.583 回答