Using video stream as open GL ES 2.0 texture

Question

I'm trying to capture video and display it to the screen by setting an Open GL ES texture to an android surfaceTexture. I can't use a TextureView and implement SurfaceTextureListener as per this tutorial since I am using Google Cardboard.

I have followed the Android documentation on how to initialise Open GL ES 2.0 and use it, and also this tutorial on texturing.

Putting the 2 together I get a blank screen and occasionally get <core_glBindTexture:572>: GL_INVALID_OPERATION in the console window.

Overwhelmed by so many new concepts that I don't know, I'm not able to debug or just understand if the two approach can be used like this. Here is my drawing code, it is initialised in the onSurfaceCreated() of the MainActivity class, and drawn from onEyeDraw() which is Cardboard's draw function.

package com.example.rich.test3;

import android.hardware.Camera;
import android.opengl.GLES20;
import android.view.TextureView;

import java.nio.ShortBuffer;
import java.nio.FloatBuffer;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;

/**
 * Created by rich on 03/05/2015.
 */
public class Square {

private java.nio.FloatBuffer vertexBuffer;
private java.nio.ShortBuffer drawListBuffer;
private final java.nio.FloatBuffer mCubeTextureCoordinates;

float color[] = { 1.f, 1.f, 1.f, 1.0f };

private final String vertexShaderCode =
        "attribute vec4 vPosition;" +
        "attribute vec2 a_TexCoordinate;" +
        "varying vec2 v_TexCoordinate;" +
                "void main() {" +
                " gl_Position = vPosition;" +
                " v_TexCoordinate = a_TexCoordinate;" +
                "}";

private final String fragmentShaderCode =
        "precision mediump float;" +
                "uniform vec4 vColor;" +
                "uniform sampler2D u_Texture;" +
                "varying vec2 v_TexCoordinate;" +
                "void main() {" +
                "gl_FragColor = (texture2D(u_Texture, v_TexCoordinate));" +
                "}";

// number of coordinates per vertex in this array
static final int COORDS_PER_VERTEX = 3;
static float squareCoords[] = {
        -0.5f, -0.5f, 0.0f,   // bottom left
        0.5f, -0.5f, 0.0f,   // bottom right
        -0.5f,  0.5f, 0.0f,   // top left
        0.5f,  0.5f, 0.0f}; // top right

private short drawOrder[] = { 0, 1, 2, 0, 2, 3 }; // order to draw vertices

private int mProgram;

private int mPositionHandle;
private int mColorHandle;
private int mTextureUniformHandle;
private int mTextureCoordinateHandle;
private final int mTextureCoordinateDataSize = 2;

private final int vertexCount = squareCoords.length / COORDS_PER_VERTEX;
private final int vertexStride = COORDS_PER_VERTEX * 4; // 4 bytes per vertex

private int mTextureDataHandle;

float textureCoordinates[] =
        {0.0f, 1.0f,
        1.0f, 1.0f,
        0.0f, 0.0f,
        1.0f, 0.0f };

Camera _camera;
TextureView _textureView;
int[] textures;
android.graphics.SurfaceTexture _surface;

public Square()
{
    ByteBuffer bb = ByteBuffer.allocateDirect(
            // (# of coordinate values * 4 bytes per float)
            squareCoords.length * 4);
    bb.order(ByteOrder.nativeOrder());
    vertexBuffer = bb.asFloatBuffer();
    vertexBuffer.put(squareCoords);
    vertexBuffer.position(0);

    // initialize byte buffer for the draw list
    ByteBuffer dlb = ByteBuffer.allocateDirect(
            // (# of coordinate values * 2 bytes per short)
            drawOrder.length * 2);
    dlb.order(ByteOrder.nativeOrder());
    drawListBuffer = dlb.asShortBuffer();
    drawListBuffer.put(drawOrder);
    drawListBuffer.position(0);

    mCubeTextureCoordinates = ByteBuffer.allocateDirect(textureCoordinates.length * 4)
            .order(ByteOrder.nativeOrder()).asFloatBuffer();
    mCubeTextureCoordinates.put(textureCoordinates).position(0);

    // create empty OpenGL ES Program
    mProgram = GLES20.glCreateProgram();

    textures = new int[1];
    GLES20.glGenTextures(1, textures, 0);

    _surface = new android.graphics.SurfaceTexture(textures[0]);
    _camera = Camera.open();
    Camera.Size previewSize = _camera.getParameters().getPreviewSize();

    try
    {
        _camera.setPreviewTexture(_surface);
    }
    catch (java.io.IOException ex)
    {
        // Console.writeLine (ex.Message);
    }

    final int vertexShaderHandle = GLES20.glCreateShader(GLES20.GL_VERTEX_SHADER);
    GLES20.glShaderSource(vertexShaderHandle, vertexShaderCode);
    GLES20.glCompileShader(vertexShaderHandle);
    final int[] compileStatus = new int[1];
    GLES20.glGetShaderiv(vertexShaderHandle, GLES20.GL_COMPILE_STATUS, compileStatus, 0);
    if (compileStatus[0] == 0)
    {
        //do check here
    }

    final int fragmentShaderHandle = GLES20.glCreateShader(GLES20.GL_FRAGMENT_SHADER);
    GLES20.glShaderSource(fragmentShaderHandle, fragmentShaderCode);
    GLES20.glCompileShader(fragmentShaderHandle);
    GLES20.glGetShaderiv(fragmentShaderHandle, GLES20.GL_COMPILE_STATUS, compileStatus, 0);
    if (compileStatus[0] == 0)
    {
        //do check here
    }

    GLES20.glAttachShader(mProgram, vertexShaderHandle);
    GLES20.glAttachShader(mProgram, fragmentShaderHandle);
    GLES20.glBindAttribLocation(mProgram, 0, "a_Position");
    GLES20.glBindAttribLocation(mProgram, 0, "a_TexCoordinate");

    GLES20.glLinkProgram(mProgram);
    final int[] linkStatus = new int[1];
    GLES20.glGetProgramiv(mProgram, GLES20.GL_LINK_STATUS, linkStatus, 0);
    if (linkStatus[0] == 0)
    {
        //do check here
    }

    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textures[0]);
    mTextureDataHandle = textures[0];

    // Set filtering
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_NEAREST);
}

public void draw()
{
    _surface.updateTexImage();
    GLES20.glUseProgram(mProgram);

    mTextureUniformHandle = GLES20.glGetUniformLocation(mProgram, "u_Texture");
    mPositionHandle = GLES20.glGetAttribLocation(mProgram, "a_Position");
    mColorHandle = GLES20.glGetAttribLocation(mProgram, "a_Color");
    mTextureCoordinateHandle = GLES20.glGetAttribLocation(mProgram, "a_TexCoordinate");

    GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureDataHandle);

    GLES20.glVertexAttribPointer(mPositionHandle, COORDS_PER_VERTEX,
            GLES20.GL_FLOAT, false,
            vertexStride, vertexBuffer);
    GLES20.glVertexAttribPointer(mTextureCoordinateHandle, mTextureCoordinateDataSize, GLES20.GL_FLOAT, false,
            0, mCubeTextureCoordinates);

    GLES20.glEnableVertexAttribArray(mTextureCoordinateHandle);
    GLES20.glEnableVertexAttribArray(mPositionHandle);
    GLES20.glUniform1i(mTextureUniformHandle, 0);

    GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, vertexCount);
    GLES20.glDisableVertexAttribArray(mPositionHandle);
}

}

Lowerclassman

Have you found github.com/google/grafika ? The "texture from camera" activity probably does most of what you want. – Klement

Answer 1

When rendering a SurfaceTexture texture object, you need to use the GL_TEXTURE_EXTERNAL_OES texture target:

The texture object uses the GL_TEXTURE_EXTERNAL_OES texture target, which is defined by the GL_OES_EGL_image_external OpenGL ES extension. This limits how the texture may be used. Each time the texture is bound it must be bound to the GL_TEXTURE_EXTERNAL_OES target rather than the GL_TEXTURE_2D target. Additionally, any OpenGL ES 2.0 shader that samples from the texture must declare its use of this extension using, for example, an "#extension GL_OES_EGL_image_external : require" directive. Such shaders must also access the texture using the samplerExternalOES GLSL sampler type.

So you need to change your fragment shader like this, adding the #extension declaration and declaring your texture uniform as samplerExternalOES:

private final String fragmentShaderCode =
    "#extension GL_OES_EGL_image_external : require\n" +
    "precision mediump float;" +
    "uniform vec4 vColor;" +
    "uniform samplerExternalOES u_Texture;" +
    "varying vec2 v_TexCoordinate;" +
    "void main() {" +
            "gl_FragColor = (texture2D(u_Texture, v_TexCoordinate));" +
    "}";

Also in your draw() function, bind the texture like this:

GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureDataHandle);

Garrulity

just had a chance to try it and get W/Adreno-ES20﹕ <core_glBindTexture:572>: GL_INVALID_OPERATION – Lowerclassman

You can get that error if you bind to different targets: khronos.org/opengles/sdk/docs/man/xhtml/glBindTexture.xml you are calling glBindTexture in the constructor as well as in draw(), change that to use GLES11Ext.GL_TEXTURE_EXTERNAL_OES as well – Garrulity

well spotted. I now get this error W/Adreno-ES20﹕ <core_glVertexAttribPointer:533>: GL_INVALID_VALUE. google isn't coming up with anything. can you see anything that could be causing this? – Lowerclassman

I think you're passing an invalid value in mPositionHandle. Your call to glGetAttribLocation asks for "a_Position" but it's called "vPosition" in the shader. Same for "a_Color"/"vColor" (although you don't seem to be using it). You should be calling GetUniformLocation for the color because it's a uniform not an attribute. – Garrulity

thanks, i still get nothing but at least i know there are no errors now. – Lowerclassman

Answer 2

You can't use normal texture to render camera or video preview, you have to use GL_TEXTURE_EXTERNAL_OES extension. I had same problem and i found a complete working solution on github. The name of the project is android_instacam .

Here you will find source code to study. If you want to see it in action directly on your device just go on play store here.

Photobathic