package org.webrtc;/*
* libjingle
* Copyright 2014, Google Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
* EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.util.ArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.opengles.GL10;
import android.opengl.GLES20;
import android.opengl.GLSurfaceView;
import android.util.Log;
import org.webrtc.VideoRenderer.I420Frame;
/**
* Efficiently renders YUV frames using the GPU for CSC.
* Clients will want first to call setView() to pass GLSurfaceView
* and then for each video stream either create instance of VideoRenderer using
* createGui() call or VideoRenderer.Callbacks interface using create() call.
* Only one instance of the class can be created.
*/
public class VideoRendererGui implements GLSurfaceView.Renderer {
private static VideoRendererGui instance = null;
private static final String TAG = "VideoRendererGui";
private GLSurfaceView surface;
// Indicates if SurfaceView.Renderer.onSurfaceCreated was called.
// If true then for every newly created yuv image renderer createTexture()
// should be called. The variable is accessed on multiple threads and
// all accesses are synchronized on yuvImageRenderers' object lock.
private boolean onSurfaceCreatedCalled;
// List of yuv renderers.
private ArrayList<YuvImageRenderer> yuvImageRenderers;
private int program;
private final String VERTEX_SHADER_STRING =
"varying vec2 interp_tc;\n" +
"attribute vec4 in_pos;\n" +
"attribute vec2 in_tc;\n" +
"\n" +
"void main() {\n" +
" gl_Position = in_pos;\n" +
" interp_tc = in_tc;\n" +
"}\n";
private final String FRAGMENT_SHADER_STRING =
"precision mediump float;\n" +
"varying vec2 interp_tc;\n" +
"\n" +
"uniform sampler2D y_tex;\n" +
"uniform sampler2D u_tex;\n" +
"uniform sampler2D v_tex;\n" +
"\n" +
"void main() {\n" +
// CSC according to http://www.fourcc.org/fccyvrgb.php
" float y = texture2D(y_tex, interp_tc).r;\n" +
" float u = texture2D(u_tex, interp_tc).r - 0.5;\n" +
" float v = texture2D(v_tex, interp_tc).r - 0.5;\n" +
" gl_FragColor = vec4(y + 1.403 * v, " +
" y - 0.344 * u - 0.714 * v, " +
" y + 1.77 * u, 1);\n" +
"}\n";
private VideoRendererGui(GLSurfaceView surface) {
this.surface = surface;
// Create an OpenGL ES 2.0 context.
surface.setPreserveEGLContextOnPause(true);
surface.setEGLContextClientVersion(2);
surface.setRenderer(this);
surface.setRenderMode(GLSurfaceView.RENDERMODE_WHEN_DIRTY);
yuvImageRenderers = new ArrayList<YuvImageRenderer>();
}
// Poor-man's assert(): die with |msg| unless |condition| is true.
private static void abortUnless(boolean condition, String msg) {
if (!condition) {
throw new RuntimeException(msg);
}
}
// Assert that no OpenGL ES 2.0 error has been raised.
private static void checkNoGLES2Error() {
int error = GLES20.glGetError();
abortUnless(error == GLES20.GL_NO_ERROR, "GLES20 error: " + error);
}
// Wrap a float[] in a direct FloatBuffer using native byte order.
private static FloatBuffer directNativeFloatBuffer(float[] array) {
FloatBuffer buffer = ByteBuffer.allocateDirect(array.length * 4).order(
ByteOrder.nativeOrder()).asFloatBuffer();
buffer.put(array);
buffer.flip();
return buffer;
}
// Compile & attach a |type| shader specified by |source| to |program|.
private static void addShaderTo(
int type, String source, int program) {
int[] result = new int[] {
GLES20.GL_FALSE
};
int shader = GLES20.glCreateShader(type);
GLES20.glShaderSource(shader, source);
GLES20.glCompileShader(shader);
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, result, 0);
abortUnless(result[0] == GLES20.GL_TRUE,
GLES20.glGetShaderInfoLog(shader) + ", source: " + source);
GLES20.glAttachShader(program, shader);
GLES20.glDeleteShader(shader);
checkNoGLES2Error();
}
/**
* Class used to display stream of YUV420 frames at particular location
* on a screen. New video frames are sent to display using renderFrame()
* call.
*/
private static class YuvImageRenderer implements VideoRenderer.Callbacks {
private GLSurfaceView surface;
private int program;
private FloatBuffer textureVertices;
private int[] yuvTextures = { -1, -1, -1 };
// Render frame queue - accessed by two threads. renderFrame() call does
// an offer (writing I420Frame to render) and early-returns (recording
// a dropped frame) if that queue is full. draw() call does a peek(),
// copies frame to texture and then removes it from a queue using poll().
LinkedBlockingQueue<I420Frame> frameToRenderQueue;
// Local copy of incoming video frame.
private I420Frame frameToRender;
// Flag if renderFrame() was ever called
boolean seenFrame;
// Total number of video frames received in renderFrame() call.
private int framesReceived;
// Number of video frames dropped by renderFrame() because previous
// frame has not been rendered yet.
private int framesDropped;
// Number of rendered video frames.
private int framesRendered;
// Time in ns when the first video frame was rendered.
private long startTimeNs = -1;
// Time in ns spent in draw() function.
private long drawTimeNs;
// Time in ns spent in renderFrame() function - including copying frame
// data to rendering planes
private long copyTimeNs;
// Texture Coordinates mapping the entire texture.
private final FloatBuffer textureCoords = directNativeFloatBuffer(
new float[] {
0, 0, 0, 1, 1, 0, 1, 1
});
private YuvImageRenderer(
GLSurfaceView surface,
int x, int y, int width, int height) {
Log.v(TAG, "YuvImageRenderer.Create");
this.surface = surface;
frameToRenderQueue = new LinkedBlockingQueue<I420Frame>(1);
// Create texture vertices.
float xLeft = (x - 50) / 50.0f;
float yTop = (50 - y) / 50.0f;
float xRight = Math.min(1.0f, (x + width - 50) / 50.0f);
float yBottom = Math.max(-1.0f, (50 - y - height) / 50.0f);
float textureVeticesFloat[] = new float[] {
xLeft, yTop,
xLeft, yBottom,
xRight, yTop,
xRight, yBottom
};
textureVertices = directNativeFloatBuffer(textureVeticesFloat);
}
private void createTextures(int program) {
Log.v(TAG, " YuvImageRenderer.createTextures");
this.program = program;
// Generate 3 texture ids for Y/U/V and place them into |textures|.
GLES20.glGenTextures(3, yuvTextures, 0);
for (int i = 0; i < 3; i++) {
GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]);
GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE,
128, 128, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, null);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
}
checkNoGLES2Error();
}
private void draw() {
long now = System.nanoTime();
if (!seenFrame) {
// No frame received yet - nothing to render.
return;
}
I420Frame frameFromQueue;
synchronized (frameToRenderQueue) {
frameFromQueue = frameToRenderQueue.peek();
if (frameFromQueue != null && startTimeNs == -1) {
startTimeNs = now;
}
for (int i = 0; i < 3; ++i) {
int w = (i == 0) ? frameToRender.width : frameToRender.width / 2;
int h = (i == 0) ? frameToRender.height : frameToRender.height / 2;
GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]);
if (frameFromQueue != null) {
GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE,
w, h, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE,
frameFromQueue.yuvPlanes[i]);
}
}
if (frameFromQueue != null) {
frameToRenderQueue.poll();
}
}
int posLocation = GLES20.glGetAttribLocation(program, "in_pos");
GLES20.glEnableVertexAttribArray(posLocation);
GLES20.glVertexAttribPointer(
posLocation, 2, GLES20.GL_FLOAT, false, 0, textureVertices);
int texLocation = GLES20.glGetAttribLocation(program, "in_tc");
GLES20.glEnableVertexAttribArray(texLocation);
GLES20.glVertexAttribPointer(
texLocation, 2, GLES20.GL_FLOAT, false, 0, textureCoords);
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
GLES20.glDisableVertexAttribArray(posLocation);
GLES20.glDisableVertexAttribArray(texLocation);
checkNoGLES2Error();
if (frameFromQueue != null) {
framesRendered++;
drawTimeNs += (System.nanoTime() - now);
if ((framesRendered % 150) == 0) {
logStatistics();
}
}
}
private void logStatistics() {
long timeSinceFirstFrameNs = System.nanoTime() - startTimeNs;
Log.v(TAG, "Frames received: " + framesReceived + ". Dropped: " +
framesDropped + ". Rendered: " + framesRendered);
if (framesReceived > 0 && framesRendered > 0) {
Log.v(TAG, "Duration: " + (int)(timeSinceFirstFrameNs / 1e6) +
" ms. FPS: " + (float)framesRendered * 1e9 / timeSinceFirstFrameNs);
Log.v(TAG, "Draw time: " +
(int) (drawTimeNs / (1000 * framesRendered)) + " us. Copy time: " +
(int) (copyTimeNs / (1000 * framesReceived)) + " us");
}
}
@Override
public void setSize(final int width, final int height) {
Log.v(TAG, "YuvImageRenderer.setSize: " + width + " x " + height);
int[] strides = { width, width / 2, width / 2 };
// Frame re-allocation need to be synchronized with copying
// frame to textures in draw() function to avoid re-allocating
// the frame while it is being copied.
synchronized (frameToRenderQueue) {
// Clear rendering queue
frameToRenderQueue.poll();
// Re-allocate / allocate the frame
frameToRender = new I420Frame(width, height, strides, null);
}
}
@Override
public synchronized void renderFrame(I420Frame frame) {
long now = System.nanoTime();
framesReceived++;
// Check input frame parameters.
if (!(frame.yuvStrides[0] == frame.width &&
frame.yuvStrides[1] == frame.width / 2 &&
frame.yuvStrides[2] == frame.width / 2)) {
Log.e(TAG, "Incorrect strides " + frame.yuvStrides[0] + ", " +
frame.yuvStrides[1] + ", " + frame.yuvStrides[2]);
return;
}
// Skip rendering of this frame if setSize() was not called.
if (frameToRender == null) {
framesDropped++;
return;
}
// Check incoming frame dimensions
if (frame.width != frameToRender.width ||
frame.height != frameToRender.height) {
throw new RuntimeException("Wrong frame size " +
frame.width + " x " + frame.height);
}
if (frameToRenderQueue.size() > 0) {
// Skip rendering of this frame if previous frame was not rendered yet.
framesDropped++;
return;
}
frameToRender.copyFrom(frame);
copyTimeNs += (System.nanoTime() - now);
frameToRenderQueue.offer(frameToRender);
seenFrame = true;
surface.requestRender();
}
}
/** Passes GLSurfaceView to video renderer. */
public static void setView(GLSurfaceView surface) {
Log.v(TAG, "VideoRendererGui.setView");
instance = new VideoRendererGui(surface);
}
/**
* Creates VideoRenderer with top left corner at (x, y) and resolution
* (width, height). All parameters are in percentage of screen resolution.
*/
public static VideoRenderer createGui(
int x, int y, int width, int height) throws Exception {
YuvImageRenderer javaGuiRenderer = create(x, y, width, height);
return new VideoRenderer(javaGuiRenderer);
}
/**
* Creates VideoRenderer.Callbacks with top left corner at (x, y) and
* resolution (width, height). All parameters are in percentage of
* screen resolution.
*/
public static YuvImageRenderer create(
int x, int y, int width, int height) {
// Check display region parameters.
if (x < 0 || x > 100 || y < 0 || y > 100 ||
width < 0 || width > 100 || height < 0 || height > 100 ||
x + width > 100 || y + height > 100) {
throw new RuntimeException("Incorrect window parameters.");
}
if (instance == null) {
throw new RuntimeException(
"Attempt to create yuv renderer before setting GLSurfaceView");
}
final YuvImageRenderer yuvImageRenderer = new YuvImageRenderer(
instance.surface, x, y, width, height);
synchronized (instance.yuvImageRenderers) {
if (instance.onSurfaceCreatedCalled) {
// onSurfaceCreated has already been called for VideoRendererGui -
// need to create texture for new image and add image to the
// rendering list.
final CountDownLatch countDownLatch = new CountDownLatch(1);
instance.surface.queueEvent(new Runnable() {
public void run() {
yuvImageRenderer.createTextures(instance.program);
countDownLatch.countDown();
}
});
// Wait for task completion.
try {
countDownLatch.await();
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
// Add yuv renderer to rendering list.
instance.yuvImageRenderers.add(yuvImageRenderer);
}
return yuvImageRenderer;
}
@Override
public void onSurfaceCreated(GL10 unused, EGLConfig config) {
Log.v(TAG, "VideoRendererGui.onSurfaceCreated");
// Create program.
program = GLES20.glCreateProgram();
addShaderTo(GLES20.GL_VERTEX_SHADER, VERTEX_SHADER_STRING, program);
addShaderTo(GLES20.GL_FRAGMENT_SHADER, FRAGMENT_SHADER_STRING, program);
GLES20.glLinkProgram(program);
int[] result = new int[] {
GLES20.GL_FALSE
};
result[0] = GLES20.GL_FALSE;
GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, result, 0);
abortUnless(result[0] == GLES20.GL_TRUE,
GLES20.glGetProgramInfoLog(program));
GLES20.glUseProgram(program);
GLES20.glUniform1i(GLES20.glGetUniformLocation(program, "y_tex"), 0);
GLES20.glUniform1i(GLES20.glGetUniformLocation(program, "u_tex"), 1);
GLES20.glUniform1i(GLES20.glGetUniformLocation(program, "v_tex"), 2);
synchronized (yuvImageRenderers) {
// Create textures for all images.
for (YuvImageRenderer yuvImageRenderer : yuvImageRenderers) {
yuvImageRenderer.createTextures(program);
}
onSurfaceCreatedCalled = true;
}
checkNoGLES2Error();
GLES20.glClearColor(0.0f, 0.0f, 0.3f, 1.0f);
}
@Override
public void onSurfaceChanged(GL10 unused, int width, int height) {
Log.v(TAG, "VideoRendererGui.onSurfaceChanged: " +
width + " x " + height + " ");
GLES20.glViewport(0, 0, width, height);
}
@Override
public void onDrawFrame(GL10 unused) {
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
synchronized (yuvImageRenderers) {
for (YuvImageRenderer yuvImageRenderer : yuvImageRenderers) {
yuvImageRenderer.draw();
}
}
}
}
