September 24, 2012

Render camera preview using OpenGL ES 2.0 on Android

Please refer -> updated version for Lollipop

// in Manifest
< uses-feature android:glEsVersion="0x00020000" android:required="true"/>
< uses-feature android:name="android.hardware.camera"/>
< uses-permission android:name="android.permission.CAMERA"/>
< uses-permission android:name="android.permission.WAKE_LOCK"/>
... android:screenOrientation="landscape" ... // activity property


// Activity
public class MainActivity extends Activity {
  private MainView mView;
  private WakeLock mWL;

  @Override
  public void onCreate(Bundle savedInstanceState) {
    super.onCreate(savedInstanceState);
    // full screen & full brightness
    requestWindowFeature ( Window.FEATURE_NO_TITLE);
    getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN, WindowManager.LayoutParams.FLAG_FULLSCREEN);
    getWindow().setFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON, WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
    mWL = ((PowerManager)getSystemService ( Context.POWER_SERVICE )).newWakeLock(PowerManager.FULL_WAKE_LOCK, "WakeLock");
    mWL.acquire();
    mView = new MainView(this);
    setContentView ( mView );
  }
  
  @Override
  protected void onPause() {
    if ( mWL.isHeld() )
      mWL.release();
    mView.onPause();
    super.onPause();
  }
    
  @Override
  protected void onResume() {
    super.onResume();
    mView.onResume();
    if(!mWL.isHeld()) mWL.acquire();
  }
}


// View
class MainView extends GLSurfaceView {
  MainRenderer mRenderer;
 
  MainView ( Context context ) {
    super ( context );
    mRenderer = new MainRenderer(this);
    setEGLContextClientVersion ( 2 );
    setRenderer ( mRenderer );
    setRenderMode ( GLSurfaceView.RENDERMODE_WHEN_DIRTY );
  }
 
  public void surfaceCreated ( SurfaceHolder holder ) {
    super.surfaceCreated ( holder );
  }
 
  public void surfaceDestroyed ( SurfaceHolder holder ) {
    mRenderer.close();
    super.surfaceDestroyed ( holder );
  }
 
  public void surfaceChanged ( SurfaceHolder holder, int format, int w, int h ) {
    super.surfaceChanged ( holder, format, w, h );
  }
}


// Renderer
public class MainRenderer implements GLSurfaceView.Renderer, SurfaceTexture.OnFrameAvailableListener {
  private final String vss =
      "attribute vec2 vPosition;\n" +
      "attribute vec2 vTexCoord;\n" +
      "varying vec2 texCoord;\n" +
      "void main() {\n" +
      "  texCoord = vTexCoord;\n" +
      "  gl_Position = vec4 ( vPosition.x, vPosition.y, 0.0, 1.0 );\n" +
      "}";
 
  private final String fss =
      "#extension GL_OES_EGL_image_external : require\n" +
      "precision mediump float;\n" +
      "uniform samplerExternalOES sTexture;\n" +
      "varying vec2 texCoord;\n" +
      "void main() {\n" +
      "  gl_FragColor = texture2D(sTexture,texCoord);\n" +
      "}";

  private int[] hTex;
  private FloatBuffer pVertex;
  private FloatBuffer pTexCoord;
  private int hProgram;
 
  private Camera mCamera;
  private SurfaceTexture mSTexture;
 
  private boolean mUpdateST = false;
 
  private MainView mView;
 
  MainRenderer ( MainView view ) {
    mView = view;
    float[] vtmp = { 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f };
    float[] ttmp = { 1.0f, 1.0f, 0.0f, 1.0f, 1.0f, 0.0f, 0.0f, 0.0f };
    pVertex = ByteBuffer.allocateDirect(8*4).order(ByteOrder.nativeOrder()).asFloatBuffer();
    pVertex.put ( vtmp );
    pVertex.position(0);
    pTexCoord = ByteBuffer.allocateDirect(8*4).order(ByteOrder.nativeOrder()).asFloatBuffer();
    pTexCoord.put ( ttmp );
    pTexCoord.position(0);
  }
 
  public void close()
  {
    mUpdateST = false;
    mSTexture.release();
    mCamera.stopPreview();
    mCamera.release();
    mCamera = null;
    deleteTex();
  }
 
  public void onSurfaceCreated ( GL10 unused, EGLConfig config ) {
    //String extensions = GLES20.glGetString(GLES20.GL_EXTENSIONS);
    //Log.i("mr", "Gl extensions: " + extensions);
    //Assert.assertTrue(extensions.contains("OES_EGL_image_external"));
        
    initTex();
    mSTexture = new SurfaceTexture ( hTex[0] );
    mSTexture.setOnFrameAvailableListener(this);

    mCamera = Camera.open();
    try {
      mCamera.setPreviewTexture(mSTexture);
    } catch ( IOException ioe ) {
    }

    GLES20.glClearColor ( 1.0f, 1.0f, 0.0f, 1.0f );
  
    hProgram = loadShader ( vss, fss );
  }
 
  public void onDrawFrame ( GL10 unused ) {
    GLES20.glClear( GLES20.GL_COLOR_BUFFER_BIT );
  
    synchronized(this) {
      if ( mUpdateST ) {
        mSTexture.updateTexImage();
        mUpdateST = false;
      }
    }

    GLES20.glUseProgram(hProgram);

    int ph = GLES20.glGetAttribLocation(hProgram, "vPosition");
    int tch = GLES20.glGetAttribLocation ( hProgram, "vTexCoord" );
    int th = GLES20.glGetUniformLocation ( hProgram, "sTexture" );
  
    GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
    GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, hTex[0]);
    GLES20.glUniform1i(th, 0);
  
    GLES20.glVertexAttribPointer(ph, 2, GLES20.GL_FLOAT, false, 4*2, pVertex);
    GLES20.glVertexAttribPointer(tch, 2, GLES20.GL_FLOAT, false, 4*2, pTexCoord );
    GLES20.glEnableVertexAttribArray(ph);
    GLES20.glEnableVertexAttribArray(tch);
  
    GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
    GLES20.glFlush();
  }
 
  public void onSurfaceChanged ( GL10 unused, int width, int height ) {
    GLES20.glViewport( 0, 0, width, height );
    Camera.Parameters param = mCamera.getParameters();
    List psize = param.getSupportedPreviewSizes();
    if ( psize.size() > 0 ) {
      int i;
      for ( i = 0; i < psize.size(); i++ ) {
        if ( psize.get(i).width < width || psize.get(i).height < height )
          break;
      }
      if ( i > 0 )
        i--;
      param.setPreviewSize(psize.get(i).width, psize.get(i).height);
      //Log.i("mr","ssize: "+psize.get(i).width+", "+psize.get(i).height);
    }
    param.set("orientation", "landscape");
    mCamera.setParameters ( param );
    mCamera.startPreview();
  }
 
  private void initTex() {
    hTex = new int[1];
    GLES20.glGenTextures ( 1, hTex, 0 );
    GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, hTex[0]);
    GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
    GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
    GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
    GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_NEAREST);
  }
 
  private void deleteTex() {
    GLES20.glDeleteTextures ( 1, hTex, 0 );
  }
 
  public synchronized void onFrameAvailable ( SurfaceTexture st ) {
    mUpdateST = true;
    mView.requestRender();
  }
 
  private static int loadShader ( String vss, String fss ) {
    int vshader = GLES20.glCreateShader(GLES20.GL_VERTEX_SHADER);
    GLES20.glShaderSource(vshader, vss);
    GLES20.glCompileShader(vshader);
    int[] compiled = new int[1];
    GLES20.glGetShaderiv(vshader, GLES20.GL_COMPILE_STATUS, compiled, 0);
    if (compiled[0] == 0) {
      Log.e("Shader", "Could not compile vshader");
      Log.v("Shader", "Could not compile vshader:"+GLES20.glGetShaderInfoLog(vshader));
      GLES20.glDeleteShader(vshader);
      vshader = 0;
    }
  
    int fshader = GLES20.glCreateShader(GLES20.GL_FRAGMENT_SHADER);
    GLES20.glShaderSource(fshader, fss);
    GLES20.glCompileShader(fshader);
    GLES20.glGetShaderiv(fshader, GLES20.GL_COMPILE_STATUS, compiled, 0);
    if (compiled[0] == 0) {
      Log.e("Shader", "Could not compile fshader");
      Log.v("Shader", "Could not compile fshader:"+GLES20.glGetShaderInfoLog(fshader));
      GLES20.glDeleteShader(fshader);
      fshader = 0;
    }

    int program = GLES20.glCreateProgram();
    GLES20.glAttachShader(program, vshader);
    GLES20.glAttachShader(program, fshader);
    GLES20.glLinkProgram(program);
        
    return program;
  }
}

12 comments:

  1. thanks for sharing !!
    I noticed some small typo - spelling of "uses-feature".

    Linda: after few minor changes it builds.
    Notice that you can't just paste it into a single file and expect it to compile...
    You need to have some minimal experience with building android projects.

    ReplyDelete
  2. This right here is what I wanted.
    Thank you. Thanks a lot!!!!!

    ReplyDelete
  3. I tried to run your code on portrait mode but camera preview went wrong orientation. Can you help me?

    I'm not good at English, sorry.

    ReplyDelete
    Replies
    1. I have also the same issue please help me...!!!

      Delete
    2. set your activity screenOrientation="landscape"

      Delete
  4. Thanks for sharing this it saved me a lot of time!

    ReplyDelete
  5. Thanks for sharing this example, it is the most complete on the internet (including Stack Overflow!). How did you know to synchronize onFrameAvailible and OnFrameDraw? This seems to be a crucial piece of the puzzle but I can't find any documentation or other examples that have this. Without that synchronize, this method doesn't seem to work.

    Thanks again, great example!

    ReplyDelete
  6. There is a problem in your code. You are not releasing the camera instance. When the APP resumes, you will have crash http://stackoverflow.com/questions/14191061/android-camera-api-app-crashes-at-onresume

    I suggest you add 'mCamera.release();' in 'public void close()' function before 'mCamera = null;'

    ReplyDelete
  7. I would also change onResume as follows

    @Override
    protected void onResume() {
    super.onResume();
    mView.onResume();
    if(!mWL.isHeld()) {
    mWL.acquire();
    }
    }

    apart from close() which should look like

    public void close()
    {
    mUpdateST = false;
    mSTexture.release();
    mCamera.stopPreview();
    mCamera.release();
    mCamera = null;
    deleteTex();
    }

    ReplyDelete
  8. Please also add the following to
    onSurfaceCreated as a safety check....otherwise most likely the application will crash.

    if (mCamera != null) {
    mCamera.release();
    mCamera = null;
    }

    initTex();
    ...
    ...
    mCamera = Camera.open();
    ...
    ...

    ReplyDelete