+ * The object wraps an encoder running on a dedicated thread. The various control messages + * may be sent from arbitrary threads (typically the app UI thread). The encoder thread + * manages both sides of the encoder (feeding and draining); the only external input is + * the GL texture. + *
+ * The design is complicated slightly by the need to create an EGL context that shares state + * with a view that gets restarted if (say) the device orientation changes. When the view + * in question is a GLSurfaceView, we don't have full control over the EGL context creation + * on that side, so we have to bend a bit backwards here. + *
+ * To use: + *
+ * Object is immutable, which means we can safely pass it between threads without + * explicit synchronization (and don't need to worry about it getting tweaked out from + * under us). + *
+ * with reasonable defaults for those and bit rate. + */ + public static class EncoderConfig { + final File mOutputFile; + final int mWidth; + final int mHeight; + final int mBitRate; + final EGLContext mEglContext; + + public EncoderConfig(File outputFile, int width, int height, int bitRate, + EGLContext sharedEglContext) { + mOutputFile = outputFile; + mWidth = width; + mHeight = height; + mBitRate = bitRate; + mEglContext = sharedEglContext; + } + + @Override + public String toString() { + return "EncoderConfig: " + mWidth + "x" + mHeight + " @" + mBitRate + + " to '" + mOutputFile.toString() + "' ctxt=" + mEglContext; + } + } + + public TextureMovieEncoder(GPUImageFilter filter) { + mFilter = filter; + } + + public void setFilter(GPUImageFilter filter) { + mFilter = filter; + } + + public void setGLCubeBuffer(FloatBuffer buffer) { + mGLCubeBuffer = buffer; + } + + public void setGLTextureBuffer(FloatBuffer buffer) { + mGLTextureBuffer = buffer; + } + + /** + * Tells the video recorder to start recording. (Call from non-encoder thread.) + *
+ * Creates a new thread, which will create an encoder using the provided configuration. + *
+ * Returns after the recorder thread has started and is ready to accept Messages. The + * encoder may not yet be fully configured. + */ + public void startRecording(EncoderConfig config) { + synchronized (mReadyFence) { + if (mRunning) { + return; + } + mRunning = true; + new Thread(this, "TextureMovieEncoder").start(); + while (!mReady) { + try { + mReadyFence.wait(); + } catch (InterruptedException ie) { + // ignore + } + } + } + + mHandler.sendMessage(mHandler.obtainMessage(MSG_START_RECORDING, config)); + } + + /** + * Tells the video recorder to stop recording. (Call from non-encoder thread.) + *
+ * Returns immediately; the encoder/muxer may not yet be finished creating the movie. + *
+ * so we can provide reasonable status UI (and let the caller know that movie encoding + * has completed). + */ + public void stopRecording() { + mHandler.sendMessage(mHandler.obtainMessage(MSG_STOP_RECORDING)); + mHandler.sendMessage(mHandler.obtainMessage(MSG_QUIT)); + // We don't know when these will actually finish (or even start). We don't want to + // delay the UI thread though, so we return immediately. + } + + /** + * Returns true if recording has been started. + */ + public boolean isRecording() { + synchronized (mReadyFence) { + return mRunning; + } + } + + /** + * Tells the video recorder to refresh its EGL surface. (Call from non-encoder thread.) + */ + public void updateSharedContext(EGLContext sharedContext) { + mHandler.sendMessage(mHandler.obtainMessage(MSG_UPDATE_SHARED_CONTEXT, sharedContext)); + } + + /** + * Tells the video recorder that a new frame is available. (Call from non-encoder thread.) + *
+ * This function sends a message and returns immediately. This isn't sufficient -- we + * don't want the caller to latch a new frame until we're done with this one -- but we + * can get away with it so long as the input frame rate is reasonable and the encoder + * thread doesn't stall. + *
+ * or have a separate "block if still busy" method that the caller can execute immediately + * before it calls updateTexImage(). The latter is preferred because we don't want to + * stall the caller while this thread does work. + */ + public void frameAvailable(SurfaceTexture st) { + synchronized (mReadyFence) { + if (!mReady) { + return; + } + } + + float[] transform = new float[16]; + st.getTransformMatrix(transform); + long timestamp = st.getTimestamp(); + if (timestamp == 0) { + // Seeing this after device is toggled off/on with power button. The + // first frame back has a zero timestamp. + // + // MPEG4Writer thinks this is cause to abort() in native code, so it's very + // important that we just ignore the frame. + return; + } + + mHandler.sendMessage(mHandler.obtainMessage(MSG_FRAME_AVAILABLE, + (int) (timestamp >> 32), (int) timestamp, transform)); + } + + /** + * Tells the video recorder what texture name to use. This is the external texture that + * we're receiving camera previews in. (Call from non-encoder thread.) + *
+ */ + public void setTextureId(int id) { + synchronized (mReadyFence) { + if (!mReady) { + return; + } + } + mHandler.sendMessage(mHandler.obtainMessage(MSG_SET_TEXTURE_ID, id, 0, null)); + } + + /** + * Encoder thread entry point. Establishes Looper/Handler and waits for messages. + *
+ * @see java.lang.Thread#run()
+ */
+ @Override
+ public void run() {
+ // Establish a Looper for this thread, and define a Handler for it.
+ Looper.prepare();
+ synchronized (mReadyFence) {
+ mHandler = new EncoderHandler(this);
+ mReady = true;
+ mReadyFence.notify();
+ }
+ Looper.loop();
+
+ synchronized (mReadyFence) {
+ mReady = mRunning = false;
+ mHandler = null;
+ }
+ }
+
+
+ /**
+ * Handles encoder state change requests. The handler is created on the encoder thread.
+ */
+ private static class EncoderHandler extends Handler {
+ private WeakReference
+ * The texture is rendered onto the encoder's input surface, along with a moving
+ * box (just because we can).
+ *
+ * @param transform The texture transform, from SurfaceTexture.
+ * @param timestampNanos The frame's timestamp, from SurfaceTexture.
+ */
+ private void handleFrameAvailable(float[] transform, long timestampNanos) {
+ mVideoEncoder.drainEncoder(false);
+ if (!mFilter.isInitialized()) {
+ mFilter.init();
+ }
+ mFilter.onDraw(mTextureId, mGLCubeBuffer, mGLTextureBuffer);
+
+ mInputWindowSurface.setPresentationTime(timestampNanos);
+ mInputWindowSurface.swapBuffers();
+ }
+
+ /**
+ * Handles a request to stop encoding.
+ */
+ private void handleStopRecording() {
+ mVideoEncoder.drainEncoder(true);
+ releaseEncoder();
+ }
+
+ /**
+ * Sets the texture name that SurfaceTexture will use when frames are received.
+ */
+ private void handleSetTexture(int id) {
+ mTextureId = id;
+ }
+
+ /**
+ * Tears down the EGL surface and context we've been using to feed the MediaCodec input
+ * surface, and replaces it with a new one that shares with the new context.
+ *
+ * This is useful if the old context we were sharing with went away (maybe a GLSurfaceView
+ * that got torn down) and we need to hook up with the new one.
+ */
+ private void handleUpdateSharedContext(EGLContext newSharedContext) {
+
+ // Release the EGLSurface and EGLContext.
+ mInputWindowSurface.releaseEglSurface();
+ mEglCore.release();
+
+ // Create a new EGLContext and recreate the window surface.
+ mEglCore = new EglCore(newSharedContext, EglCore.FLAG_RECORDABLE);
+ mInputWindowSurface.recreate(mEglCore);
+ mInputWindowSurface.makeCurrent();
+ }
+
+ private void prepareEncoder(EGLContext sharedContext, int width, int height, int bitRate,
+ File outputFile) {
+ try {
+ mVideoEncoder = new VideoEncoderCore(width, height, bitRate, outputFile);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ mEglCore = new EglCore(sharedContext, EglCore.FLAG_RECORDABLE);
+ mInputWindowSurface = new WindowSurface(mEglCore, mVideoEncoder.getInputSurface(), true);
+ mInputWindowSurface.makeCurrent();
+ }
+
+ private void releaseEncoder() {
+ mVideoEncoder.release();
+ if (mInputWindowSurface != null) {
+ mInputWindowSurface.release();
+ mInputWindowSurface = null;
+ }
+ if (mEglCore != null) {
+ mEglCore.release();
+ mEglCore = null;
+ }
+ }
+}
diff --git a/library/src/jp/co/cyberagent/android/gpuimage/VideoEncoderCore.java b/library/src/jp/co/cyberagent/android/gpuimage/VideoEncoderCore.java
new file mode 100644
index 000000000..8559afa10
--- /dev/null
+++ b/library/src/jp/co/cyberagent/android/gpuimage/VideoEncoderCore.java
@@ -0,0 +1,199 @@
+/*
+ * Original work Copyright 2014 Google Inc.
+ * Modified work Copyright 2016 Peter Lu
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package jp.co.cyberagent.android.gpuimage;
+
+import android.media.MediaCodec;
+import android.media.MediaCodecInfo;
+import android.media.MediaFormat;
+import android.media.MediaMuxer;
+import android.util.Log;
+import android.view.Surface;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * This class wraps up the core components used for surface-input video encoding.
+ *
+ * Once created, frames are fed to the input surface. Remember to provide the presentation
+ * time stamp, and always call drainEncoder() before swapBuffers() to ensure that the
+ * producer side doesn't get backed up.
+ *
+ * This class is not thread-safe, with one exception: it is valid to use the input surface
+ * on one thread, and drain the output on a different thread.
+ */
+public class VideoEncoderCore {
+ private static final String TAG = "VideoEncoderCore";
+
+ private static final String MIME_TYPE = "video/avc"; // H.264 Advanced Video Coding
+ private static final int FRAME_RATE = 30; // 30fps
+ private static final int IFRAME_INTERVAL = 5; // 5 seconds between I-frames
+
+ private Surface mInputSurface;
+ private MediaMuxer mMuxer;
+ private MediaCodec mEncoder;
+ private MediaCodec.BufferInfo mBufferInfo;
+ private int mTrackIndex;
+ private boolean mMuxerStarted;
+
+
+ /**
+ * Configures encoder and muxer state, and prepares the input Surface.
+ */
+ public VideoEncoderCore(int width, int height, int bitRate, File outputFile)
+ throws IOException {
+ mBufferInfo = new MediaCodec.BufferInfo();
+
+ try {
+ MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, width, height);
+
+ // Set some properties. Failing to specify some of these can cause the MediaCodec
+ // configure() call to throw an unhelpful exception.
+ format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
+ MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
+ format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);
+ format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE);
+ format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
+
+ // Create a MediaCodec encoder, and configure it with our format. Get a Surface
+ // we can use for input and wrap it with a class that handles the EGL work.
+ mEncoder = MediaCodec.createEncoderByType(MIME_TYPE);
+ mEncoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
+ mInputSurface = mEncoder.createInputSurface();
+ mEncoder.start();
+ } catch (MediaCodec.CodecException e) {
+ Log.e(TAG, "Failed to start encoder. Make sure the width and height are supported.");
+ }
+
+ // Create a MediaMuxer. We can't add the video track and start() the muxer here,
+ // because our MediaFormat doesn't have the Magic Goodies. These can only be
+ // obtained from the encoder after it has started processing data.
+ //
+ // We're not actually interested in multiplexing audio. We just want to convert
+ // the raw H.264 elementary stream we get from MediaCodec into a .mp4 file.
+ mMuxer = new MediaMuxer(outputFile.toString(),
+ MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
+
+ mTrackIndex = -1;
+ mMuxerStarted = false;
+ }
+
+ /**
+ * Returns the encoder's input surface.
+ */
+ public Surface getInputSurface() {
+ return mInputSurface;
+ }
+
+ /**
+ * Releases encoder resources.
+ */
+ public void release() {
+ if (mEncoder != null) {
+ mEncoder.stop();
+ mEncoder.release();
+ mEncoder = null;
+ }
+ if (mMuxer != null) {
+ mMuxer.stop();
+ mMuxer.release();
+ mMuxer = null;
+ }
+ }
+
+ /**
+ * Extracts all pending data from the encoder and forwards it to the muxer.
+ *
+ * If endOfStream is not set, this returns when there is no more data to drain. If it
+ * is set, we send EOS to the encoder, and then iterate until we see EOS on the output.
+ * Calling this with endOfStream set should be done once, right before stopping the muxer.
+ *
+ * We're just using the muxer to get a .mp4 file (instead of a raw H.264 stream). We're
+ * not recording audio.
+ */
+ public void drainEncoder(boolean endOfStream) {
+ final int TIMEOUT_USEC = 10000;
+
+ if (endOfStream) {
+ mEncoder.signalEndOfInputStream();
+ }
+
+ ByteBuffer[] encoderOutputBuffers = mEncoder.getOutputBuffers();
+ while (true) {
+ int encoderStatus = mEncoder.dequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC);
+ if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
+ // no output available yet
+ if (!endOfStream) {
+ break; // out of while
+ }
+ } else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
+ // not expected for an encoder
+ encoderOutputBuffers = mEncoder.getOutputBuffers();
+ } else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
+ // should happen before receiving buffers, and should only happen once
+ if (mMuxerStarted) {
+ throw new RuntimeException("format changed twice");
+ }
+ MediaFormat newFormat = mEncoder.getOutputFormat();
+
+ // now that we have the Magic Goodies, start the muxer
+ mTrackIndex = mMuxer.addTrack(newFormat);
+ mMuxer.start();
+ mMuxerStarted = true;
+ } else if (encoderStatus < 0) {
+ Log.w(TAG, "unexpected result from encoder.dequeueOutputBuffer: " +
+ encoderStatus);
+ // let's ignore it
+ } else {
+ ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
+ if (encodedData == null) {
+ throw new RuntimeException("encoderOutputBuffer " + encoderStatus +
+ " was null");
+ }
+
+ if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
+ // The codec config data was pulled out and fed to the muxer when we got
+ // the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it.
+ mBufferInfo.size = 0;
+ }
+
+ if (mBufferInfo.size != 0) {
+ if (!mMuxerStarted) {
+ throw new RuntimeException("muxer hasn't started");
+ }
+
+ // adjust the ByteBuffer values to match BufferInfo (not needed?)
+ encodedData.position(mBufferInfo.offset);
+ encodedData.limit(mBufferInfo.offset + mBufferInfo.size);
+
+ mMuxer.writeSampleData(mTrackIndex, encodedData, mBufferInfo);
+ }
+
+ mEncoder.releaseOutputBuffer(encoderStatus, false);
+
+ if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
+ if (!endOfStream) {
+ Log.w(TAG, "reached end of stream unexpectedly");
+ }
+ break; // out of while
+ }
+ }
+ }
+ }
+}
diff --git a/library/src/jp/co/cyberagent/android/gpuimage/gles/EglCore.java b/library/src/jp/co/cyberagent/android/gpuimage/gles/EglCore.java
new file mode 100644
index 000000000..e5353c402
--- /dev/null
+++ b/library/src/jp/co/cyberagent/android/gpuimage/gles/EglCore.java
@@ -0,0 +1,361 @@
+/*
+ * Original work Copyright 2013 Google Inc.
+ * Modified work Copyright 2016 Peter Lu
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package jp.co.cyberagent.android.gpuimage.gles;
+
+import android.graphics.SurfaceTexture;
+import android.opengl.EGL14;
+import android.opengl.EGLConfig;
+import android.opengl.EGLContext;
+import android.opengl.EGLDisplay;
+import android.opengl.EGLExt;
+import android.opengl.EGLSurface;
+import android.util.Log;
+import android.view.Surface;
+
+/**
+ * Core EGL state (display, context, config).
+ *
+ * The EGLContext must only be attached to one thread at a time. This class is not thread-safe.
+ */
+public final class EglCore {
+ private static final String TAG = GlUtil.TAG;
+
+ /**
+ * Constructor flag: surface must be recordable. This discourages EGL from using a
+ * pixel format that cannot be converted efficiently to something usable by the video
+ * encoder.
+ */
+ public static final int FLAG_RECORDABLE = 0x01;
+
+ /**
+ * Constructor flag: ask for GLES3, fall back to GLES2 if not available. Without this
+ * flag, GLES2 is used.
+ */
+ public static final int FLAG_TRY_GLES3 = 0x02;
+
+ // Android-specific extension.
+ private static final int EGL_RECORDABLE_ANDROID = 0x3142;
+
+ private EGLDisplay mEGLDisplay = EGL14.EGL_NO_DISPLAY;
+ private EGLContext mEGLContext = EGL14.EGL_NO_CONTEXT;
+ private EGLConfig mEGLConfig = null;
+ private int mGlVersion = -1;
+
+
+ /**
+ * Prepares EGL display and context.
+ *
+ * Equivalent to EglCore(null, 0).
+ */
+ public EglCore() {
+ this(null, 0);
+ }
+
+ /**
+ * Prepares EGL display and context.
+ *
+ * @param sharedContext The context to share, or null if sharing is not desired.
+ * @param flags Configuration bit flags, e.g. FLAG_RECORDABLE.
+ */
+ public EglCore(EGLContext sharedContext, int flags) {
+ if (mEGLDisplay != EGL14.EGL_NO_DISPLAY) {
+ throw new RuntimeException("EGL already set up");
+ }
+
+ if (sharedContext == null) {
+ sharedContext = EGL14.EGL_NO_CONTEXT;
+ }
+
+ mEGLDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY);
+ if (mEGLDisplay == EGL14.EGL_NO_DISPLAY) {
+ throw new RuntimeException("unable to get EGL14 display");
+ }
+ int[] version = new int[2];
+ if (!EGL14.eglInitialize(mEGLDisplay, version, 0, version, 1)) {
+ mEGLDisplay = null;
+ throw new RuntimeException("unable to initialize EGL14");
+ }
+
+ // Try to get a GLES3 context, if requested.
+ if ((flags & FLAG_TRY_GLES3) != 0) {
+ EGLConfig config = getConfig(flags, 3);
+ if (config != null) {
+ int[] attrib3_list = {
+ EGL14.EGL_CONTEXT_CLIENT_VERSION, 3,
+ EGL14.EGL_NONE
+ };
+ EGLContext context = EGL14.eglCreateContext(mEGLDisplay, config, sharedContext,
+ attrib3_list, 0);
+
+ if (EGL14.eglGetError() == EGL14.EGL_SUCCESS) {
+ mEGLConfig = config;
+ mEGLContext = context;
+ mGlVersion = 3;
+ }
+ }
+ }
+ if (mEGLContext == EGL14.EGL_NO_CONTEXT) { // GLES 2 only, or GLES 3 attempt failed
+ EGLConfig config = getConfig(flags, 2);
+ if (config == null) {
+ throw new RuntimeException("Unable to find a suitable EGLConfig");
+ }
+ int[] attrib2_list = {
+ EGL14.EGL_CONTEXT_CLIENT_VERSION, 2,
+ EGL14.EGL_NONE
+ };
+ EGLContext context = EGL14.eglCreateContext(mEGLDisplay, config, sharedContext,
+ attrib2_list, 0);
+ checkEglError("eglCreateContext");
+ mEGLConfig = config;
+ mEGLContext = context;
+ mGlVersion = 2;
+ }
+
+ // Confirm with query.
+ int[] values = new int[1];
+ EGL14.eglQueryContext(mEGLDisplay, mEGLContext, EGL14.EGL_CONTEXT_CLIENT_VERSION,
+ values, 0);
+ }
+
+ /**
+ * Finds a suitable EGLConfig.
+ *
+ * @param flags Bit flags from constructor.
+ * @param version Must be 2 or 3.
+ */
+ private EGLConfig getConfig(int flags, int version) {
+ int renderableType = EGL14.EGL_OPENGL_ES2_BIT;
+ if (version >= 3) {
+ renderableType |= EGLExt.EGL_OPENGL_ES3_BIT_KHR;
+ }
+
+ // The actual surface is generally RGBA or RGBX, so situationally omitting alpha
+ // doesn't really help. It can also lead to a huge performance hit on glReadPixels()
+ // when reading into a GL_RGBA buffer.
+ int[] attribList = {
+ EGL14.EGL_RED_SIZE, 8,
+ EGL14.EGL_GREEN_SIZE, 8,
+ EGL14.EGL_BLUE_SIZE, 8,
+ EGL14.EGL_ALPHA_SIZE, 8,
+ //EGL14.EGL_DEPTH_SIZE, 16,
+ //EGL14.EGL_STENCIL_SIZE, 8,
+ EGL14.EGL_RENDERABLE_TYPE, renderableType,
+ EGL14.EGL_NONE, 0, // placeholder for recordable [@-3]
+ EGL14.EGL_NONE
+ };
+ if ((flags & FLAG_RECORDABLE) != 0) {
+ attribList[attribList.length - 3] = EGL_RECORDABLE_ANDROID;
+ attribList[attribList.length - 2] = 1;
+ }
+ EGLConfig[] configs = new EGLConfig[1];
+ int[] numConfigs = new int[1];
+ if (!EGL14.eglChooseConfig(mEGLDisplay, attribList, 0, configs, 0, configs.length,
+ numConfigs, 0)) {
+ Log.w(TAG, "unable to find RGB8888 / " + version + " EGLConfig");
+ return null;
+ }
+ return configs[0];
+ }
+
+ /**
+ * Discards all resources held by this class, notably the EGL context. This must be
+ * called from the thread where the context was created.
+ *
+ * On completion, no context will be current.
+ */
+ public void release() {
+ if (mEGLDisplay != EGL14.EGL_NO_DISPLAY) {
+ // Android is unusual in that it uses a reference-counted EGLDisplay. So for
+ // every eglInitialize() we need an eglTerminate().
+ EGL14.eglMakeCurrent(mEGLDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE,
+ EGL14.EGL_NO_CONTEXT);
+ EGL14.eglDestroyContext(mEGLDisplay, mEGLContext);
+ EGL14.eglReleaseThread();
+ EGL14.eglTerminate(mEGLDisplay);
+ }
+
+ mEGLDisplay = EGL14.EGL_NO_DISPLAY;
+ mEGLContext = EGL14.EGL_NO_CONTEXT;
+ mEGLConfig = null;
+ }
+
+ @Override
+ protected void finalize() throws Throwable {
+ try {
+ if (mEGLDisplay != EGL14.EGL_NO_DISPLAY) {
+ // We're limited here -- finalizers don't run on the thread that holds
+ // the EGL state, so if a surface or context is still current on another
+ // thread we can't fully release it here. Exceptions thrown from here
+ // are quietly discarded. Complain in the log file.
+ Log.w(TAG, "WARNING: EglCore was not explicitly released -- state may be leaked");
+ release();
+ }
+ } finally {
+ super.finalize();
+ }
+ }
+
+ /**
+ * Destroys the specified surface. Note the EGLSurface won't actually be destroyed if it's
+ * still current in a context.
+ */
+ public void releaseSurface(EGLSurface eglSurface) {
+ EGL14.eglDestroySurface(mEGLDisplay, eglSurface);
+ }
+
+ /**
+ * Creates an EGL surface associated with a Surface.
+ *
+ * If this is destined for MediaCodec, the EGLConfig should have the "recordable" attribute.
+ */
+ public EGLSurface createWindowSurface(Object surface) {
+ if (!(surface instanceof Surface) && !(surface instanceof SurfaceTexture)) {
+ throw new RuntimeException("invalid surface: " + surface);
+ }
+
+ // Create a window surface, and attach it to the Surface we received.
+ int[] surfaceAttribs = {
+ EGL14.EGL_NONE
+ };
+ EGLSurface eglSurface = EGL14.eglCreateWindowSurface(mEGLDisplay, mEGLConfig, surface,
+ surfaceAttribs, 0);
+ checkEglError("eglCreateWindowSurface");
+ if (eglSurface == null) {
+ throw new RuntimeException("surface was null");
+ }
+ return eglSurface;
+ }
+
+ /**
+ * Creates an EGL surface associated with an offscreen buffer.
+ */
+ public EGLSurface createOffscreenSurface(int width, int height) {
+ int[] surfaceAttribs = {
+ EGL14.EGL_WIDTH, width,
+ EGL14.EGL_HEIGHT, height,
+ EGL14.EGL_NONE
+ };
+ EGLSurface eglSurface = EGL14.eglCreatePbufferSurface(mEGLDisplay, mEGLConfig,
+ surfaceAttribs, 0);
+ checkEglError("eglCreatePbufferSurface");
+ if (eglSurface == null) {
+ throw new RuntimeException("surface was null");
+ }
+ return eglSurface;
+ }
+
+ /**
+ * Makes our EGL context current, using the supplied surface for both "draw" and "read".
+ */
+ public void makeCurrent(EGLSurface eglSurface) {
+ if (!EGL14.eglMakeCurrent(mEGLDisplay, eglSurface, eglSurface, mEGLContext)) {
+ throw new RuntimeException("eglMakeCurrent failed");
+ }
+ }
+
+ /**
+ * Makes our EGL context current, using the supplied "draw" and "read" surfaces.
+ */
+ public void makeCurrent(EGLSurface drawSurface, EGLSurface readSurface) {
+ if (!EGL14.eglMakeCurrent(mEGLDisplay, drawSurface, readSurface, mEGLContext)) {
+ throw new RuntimeException("eglMakeCurrent(draw,read) failed");
+ }
+ }
+
+ /**
+ * Makes no context current.
+ */
+ public void makeNothingCurrent() {
+ if (!EGL14.eglMakeCurrent(mEGLDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE,
+ EGL14.EGL_NO_CONTEXT)) {
+ throw new RuntimeException("eglMakeCurrent failed");
+ }
+ }
+
+ /**
+ * Calls eglSwapBuffers. Use this to "publish" the current frame.
+ *
+ * @return false on failure
+ */
+ public boolean swapBuffers(EGLSurface eglSurface) {
+ return EGL14.eglSwapBuffers(mEGLDisplay, eglSurface);
+ }
+
+ /**
+ * Sends the presentation time stamp to EGL. Time is expressed in nanoseconds.
+ */
+ public void setPresentationTime(EGLSurface eglSurface, long nsecs) {
+ EGLExt.eglPresentationTimeANDROID(mEGLDisplay, eglSurface, nsecs);
+ }
+
+ /**
+ * Returns true if our context and the specified surface are current.
+ */
+ public boolean isCurrent(EGLSurface eglSurface) {
+ return mEGLContext.equals(EGL14.eglGetCurrentContext()) &&
+ eglSurface.equals(EGL14.eglGetCurrentSurface(EGL14.EGL_DRAW));
+ }
+
+ /**
+ * Performs a simple surface query.
+ */
+ public int querySurface(EGLSurface eglSurface, int what) {
+ int[] value = new int[1];
+ EGL14.eglQuerySurface(mEGLDisplay, eglSurface, what, value, 0);
+ return value[0];
+ }
+
+ /**
+ * Queries a string value.
+ */
+ public String queryString(int what) {
+ return EGL14.eglQueryString(mEGLDisplay, what);
+ }
+
+ /**
+ * Returns the GLES version this context is configured for (currently 2 or 3).
+ */
+ public int getGlVersion() {
+ return mGlVersion;
+ }
+
+ /**
+ * Writes the current display, context, and surface to the log.
+ */
+ public static void logCurrent(String msg) {
+ EGLDisplay display;
+ EGLContext context;
+ EGLSurface surface;
+
+ display = EGL14.eglGetCurrentDisplay();
+ context = EGL14.eglGetCurrentContext();
+ surface = EGL14.eglGetCurrentSurface(EGL14.EGL_DRAW);
+ Log.i(TAG, "Current EGL (" + msg + "): display=" + display + ", context=" + context +
+ ", surface=" + surface);
+ }
+
+ /**
+ * Checks for EGL errors. Throws an exception if an error has been raised.
+ */
+ private void checkEglError(String msg) {
+ int error;
+ if ((error = EGL14.eglGetError()) != EGL14.EGL_SUCCESS) {
+ throw new RuntimeException(msg + ": EGL error: 0x" + Integer.toHexString(error));
+ }
+ }
+}
diff --git a/library/src/jp/co/cyberagent/android/gpuimage/gles/EglSurfaceBase.java b/library/src/jp/co/cyberagent/android/gpuimage/gles/EglSurfaceBase.java
new file mode 100644
index 000000000..04da2a473
--- /dev/null
+++ b/library/src/jp/co/cyberagent/android/gpuimage/gles/EglSurfaceBase.java
@@ -0,0 +1,192 @@
+/*
+ * Original work Copyright 2013 Google Inc.
+ * Modified work Copyright 2016 Peter Lu
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package jp.co.cyberagent.android.gpuimage.gles;
+
+import android.graphics.Bitmap;
+import android.opengl.EGL14;
+import android.opengl.EGLSurface;
+import android.opengl.GLES20;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+/**
+ * Common base class for EGL surfaces.
+ *
+ * There can be multiple surfaces associated with a single context.
+ */
+public class EglSurfaceBase {
+
+ // EglCore object we're associated with. It may be associated with multiple surfaces.
+ protected EglCore mEglCore;
+
+ private EGLSurface mEGLSurface = EGL14.EGL_NO_SURFACE;
+ private int mWidth = -1;
+ private int mHeight = -1;
+
+ protected EglSurfaceBase(EglCore eglCore) {
+ mEglCore = eglCore;
+ }
+
+ /**
+ * Creates a window surface.
+ *
+ * @param surface May be a Surface or SurfaceTexture.
+ */
+ public void createWindowSurface(Object surface) {
+ if (mEGLSurface != EGL14.EGL_NO_SURFACE) {
+ throw new IllegalStateException("surface already created");
+ }
+ mEGLSurface = mEglCore.createWindowSurface(surface);
+
+ // Don't cache width/height here, because the size of the underlying surface can change
+ // out from under us (see e.g. HardwareScalerActivity).
+ //mWidth = mEglCore.querySurface(mEGLSurface, EGL14.EGL_WIDTH);
+ //mHeight = mEglCore.querySurface(mEGLSurface, EGL14.EGL_HEIGHT);
+ }
+
+ /**
+ * Creates an off-screen surface.
+ */
+ public void createOffscreenSurface(int width, int height) {
+ if (mEGLSurface != EGL14.EGL_NO_SURFACE) {
+ throw new IllegalStateException("surface already created");
+ }
+ mEGLSurface = mEglCore.createOffscreenSurface(width, height);
+ mWidth = width;
+ mHeight = height;
+ }
+
+ /**
+ * Returns the surface's width, in pixels.
+ *
+ * If this is called on a window surface, and the underlying surface is in the process
+ * of changing size, we may not see the new size right away (e.g. in the "surfaceChanged"
+ * callback). The size should match after the next buffer swap.
+ */
+ public int getWidth() {
+ if (mWidth < 0) {
+ return mEglCore.querySurface(mEGLSurface, EGL14.EGL_WIDTH);
+ } else {
+ return mWidth;
+ }
+ }
+
+ /**
+ * Returns the surface's height, in pixels.
+ */
+ public int getHeight() {
+ if (mHeight < 0) {
+ return mEglCore.querySurface(mEGLSurface, EGL14.EGL_HEIGHT);
+ } else {
+ return mHeight;
+ }
+ }
+
+ /**
+ * Release the EGL surface.
+ */
+ public void releaseEglSurface() {
+ mEglCore.releaseSurface(mEGLSurface);
+ mEGLSurface = EGL14.EGL_NO_SURFACE;
+ mWidth = mHeight = -1;
+ }
+
+ /**
+ * Makes our EGL context and surface current.
+ */
+ public void makeCurrent() {
+ mEglCore.makeCurrent(mEGLSurface);
+ }
+
+ /**
+ * Makes our EGL context and surface current for drawing, using the supplied surface
+ * for reading.
+ */
+ public void makeCurrentReadFrom(EglSurfaceBase readSurface) {
+ mEglCore.makeCurrent(mEGLSurface, readSurface.mEGLSurface);
+ }
+
+ /**
+ * Calls eglSwapBuffers. Use this to "publish" the current frame.
+ *
+ * @return false on failure
+ */
+ public boolean swapBuffers() {
+ boolean result = mEglCore.swapBuffers(mEGLSurface);
+ return result;
+ }
+
+ /**
+ * Sends the presentation time stamp to EGL.
+ *
+ * @param nsecs Timestamp, in nanoseconds.
+ */
+ public void setPresentationTime(long nsecs) {
+ mEglCore.setPresentationTime(mEGLSurface, nsecs);
+ }
+
+ /**
+ * Saves the EGL surface to a file.
+ *
+ * Expects that this object's EGL surface is current.
+ */
+ public void saveFrame(File file) throws IOException {
+ if (!mEglCore.isCurrent(mEGLSurface)) {
+ throw new RuntimeException("Expected EGL context/surface is not current");
+ }
+
+ // glReadPixels fills in a "direct" ByteBuffer with what is essentially big-endian RGBA
+ // data (i.e. a byte of red, followed by a byte of green...). While the Bitmap
+ // constructor that takes an int[] wants little-endian ARGB (blue/red swapped), the
+ // Bitmap "copy pixels" method wants the same format GL provides.
+ //
+ // Ideally we'd have some way to re-use the ByteBuffer, especially if we're calling
+ // here often.
+ //
+ // Making this even more interesting is the upside-down nature of GL, which means
+ // our output will look upside down relative to what appears on screen if the
+ // typical GL conventions are used.
+
+ String filename = file.toString();
+
+ int width = getWidth();
+ int height = getHeight();
+ ByteBuffer buf = ByteBuffer.allocateDirect(width * height * 4);
+ buf.order(ByteOrder.LITTLE_ENDIAN);
+ GLES20.glReadPixels(0, 0, width, height,
+ GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, buf);
+ GlUtil.checkGlError("glReadPixels");
+ buf.rewind();
+
+ BufferedOutputStream bos = null;
+ try {
+ bos = new BufferedOutputStream(new FileOutputStream(filename));
+ Bitmap bmp = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888);
+ bmp.copyPixelsFromBuffer(buf);
+ bmp.compress(Bitmap.CompressFormat.PNG, 90, bos);
+ bmp.recycle();
+ } finally {
+ if (bos != null) bos.close();
+ }
+ }
+}
diff --git a/library/src/jp/co/cyberagent/android/gpuimage/gles/GlUtil.java b/library/src/jp/co/cyberagent/android/gpuimage/gles/GlUtil.java
new file mode 100644
index 000000000..76b63b875
--- /dev/null
+++ b/library/src/jp/co/cyberagent/android/gpuimage/gles/GlUtil.java
@@ -0,0 +1,196 @@
+/*
+ * Original work Copyright 2014 Google Inc.
+ * Modified work Copyright 2016 Peter Lu
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package jp.co.cyberagent.android.gpuimage.gles;
+
+import android.opengl.GLES20;
+import android.opengl.GLES30;
+import android.opengl.Matrix;
+import android.util.Log;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.FloatBuffer;
+
+/**
+ * Some OpenGL utility functions.
+ */
+public class GlUtil {
+ public static final String TAG = "GlUtil";
+
+ /** Identity matrix for general use. Don't modify or life will get weird. */
+ public static final float[] IDENTITY_MATRIX;
+ static {
+ IDENTITY_MATRIX = new float[16];
+ Matrix.setIdentityM(IDENTITY_MATRIX, 0);
+ }
+
+ private static final int SIZEOF_FLOAT = 4;
+
+
+ private GlUtil() {} // do not instantiate
+
+ /**
+ * Creates a new program from the supplied vertex and fragment shaders.
+ *
+ * @return A handle to the program, or 0 on failure.
+ */
+ public static int createProgram(String vertexSource, String fragmentSource) {
+ int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
+ if (vertexShader == 0) {
+ return 0;
+ }
+ int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
+ if (pixelShader == 0) {
+ return 0;
+ }
+
+ int program = GLES20.glCreateProgram();
+ checkGlError("glCreateProgram");
+ if (program == 0) {
+ Log.e(TAG, "Could not create program");
+ }
+ GLES20.glAttachShader(program, vertexShader);
+ checkGlError("glAttachShader");
+ GLES20.glAttachShader(program, pixelShader);
+ checkGlError("glAttachShader");
+ GLES20.glLinkProgram(program);
+ int[] linkStatus = new int[1];
+ GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
+ if (linkStatus[0] != GLES20.GL_TRUE) {
+ Log.e(TAG, "Could not link program: ");
+ Log.e(TAG, GLES20.glGetProgramInfoLog(program));
+ GLES20.glDeleteProgram(program);
+ program = 0;
+ }
+ return program;
+ }
+
+ /**
+ * Compiles the provided shader source.
+ *
+ * @return A handle to the shader, or 0 on failure.
+ */
+ public static int loadShader(int shaderType, String source) {
+ int shader = GLES20.glCreateShader(shaderType);
+ checkGlError("glCreateShader type=" + shaderType);
+ GLES20.glShaderSource(shader, source);
+ GLES20.glCompileShader(shader);
+ int[] compiled = new int[1];
+ GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
+ if (compiled[0] == 0) {
+ Log.e(TAG, "Could not compile shader " + shaderType + ":");
+ Log.e(TAG, " " + GLES20.glGetShaderInfoLog(shader));
+ GLES20.glDeleteShader(shader);
+ shader = 0;
+ }
+ return shader;
+ }
+
+ /**
+ * Checks to see if a GLES error has been raised.
+ */
+ public static void checkGlError(String op) {
+ int error = GLES20.glGetError();
+ if (error != GLES20.GL_NO_ERROR) {
+ String msg = op + ": glError 0x" + Integer.toHexString(error);
+ Log.e(TAG, msg);
+ throw new RuntimeException(msg);
+ }
+ }
+
+ /**
+ * Checks to see if the location we obtained is valid. GLES returns -1 if a label
+ * could not be found, but does not set the GL error.
+ *
+ * Throws a RuntimeException if the location is invalid.
+ */
+ public static void checkLocation(int location, String label) {
+ if (location < 0) {
+ throw new RuntimeException("Unable to locate '" + label + "' in program");
+ }
+ }
+
+ /**
+ * Creates a texture from raw data.
+ *
+ * @param data Image data, in a "direct" ByteBuffer.
+ * @param width Texture width, in pixels (not bytes).
+ * @param height Texture height, in pixels.
+ * @param format Image data format (use constant appropriate for glTexImage2D(), e.g. GL_RGBA).
+ * @return Handle to texture.
+ */
+ public static int createImageTexture(ByteBuffer data, int width, int height, int format) {
+ int[] textureHandles = new int[1];
+ int textureHandle;
+
+ GLES20.glGenTextures(1, textureHandles, 0);
+ textureHandle = textureHandles[0];
+ GlUtil.checkGlError("glGenTextures");
+
+ // Bind the texture handle to the 2D texture target.
+ GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureHandle);
+
+ // Configure min/mag filtering, i.e. what scaling method do we use if what we're rendering
+ // is smaller or larger than the source image.
+ GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER,
+ GLES20.GL_LINEAR);
+ GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER,
+ GLES20.GL_LINEAR);
+ GlUtil.checkGlError("loadImageTexture");
+
+ // Load the data from the buffer into the texture handle.
+ GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, /*level*/ 0, format,
+ width, height, /*border*/ 0, format, GLES20.GL_UNSIGNED_BYTE, data);
+ GlUtil.checkGlError("loadImageTexture");
+
+ return textureHandle;
+ }
+
+ /**
+ * Allocates a direct float buffer, and populates it with the float array data.
+ */
+ public static FloatBuffer createFloatBuffer(float[] coords) {
+ // Allocate a direct ByteBuffer, using 4 bytes per float, and copy coords into it.
+ ByteBuffer bb = ByteBuffer.allocateDirect(coords.length * SIZEOF_FLOAT);
+ bb.order(ByteOrder.nativeOrder());
+ FloatBuffer fb = bb.asFloatBuffer();
+ fb.put(coords);
+ fb.position(0);
+ return fb;
+ }
+
+ /**
+ * Writes GL version info to the log.
+ */
+ public static void logVersionInfo() {
+ Log.i(TAG, "vendor : " + GLES20.glGetString(GLES20.GL_VENDOR));
+ Log.i(TAG, "renderer: " + GLES20.glGetString(GLES20.GL_RENDERER));
+ Log.i(TAG, "version : " + GLES20.glGetString(GLES20.GL_VERSION));
+
+ if (false) {
+ int[] values = new int[1];
+ GLES30.glGetIntegerv(GLES30.GL_MAJOR_VERSION, values, 0);
+ int majorVersion = values[0];
+ GLES30.glGetIntegerv(GLES30.GL_MINOR_VERSION, values, 0);
+ int minorVersion = values[0];
+ if (GLES30.glGetError() == GLES30.GL_NO_ERROR) {
+ Log.i(TAG, "iversion: " + majorVersion + "." + minorVersion);
+ }
+ }
+ }
+}
diff --git a/library/src/jp/co/cyberagent/android/gpuimage/gles/WindowSurface.java b/library/src/jp/co/cyberagent/android/gpuimage/gles/WindowSurface.java
new file mode 100644
index 000000000..92ea49b61
--- /dev/null
+++ b/library/src/jp/co/cyberagent/android/gpuimage/gles/WindowSurface.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2013 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package jp.co.cyberagent.android.gpuimage.gles;
+
+import android.graphics.SurfaceTexture;
+import android.view.Surface;
+
+/**
+ * Recordable EGL window surface.
+ *
+ * It's good practice to explicitly release() the surface, preferably from a "finally" block.
+ */
+public class WindowSurface extends EglSurfaceBase {
+ private Surface mSurface;
+ private boolean mReleaseSurface;
+
+ /**
+ * Associates an EGL surface with the native window surface.
+ *
+ * Set releaseSurface to true if you want the Surface to be released when release() is
+ * called. This is convenient, but can interfere with framework classes that expect to
+ * manage the Surface themselves (e.g. if you release a SurfaceView's Surface, the
+ * surfaceDestroyed() callback won't fire).
+ */
+ public WindowSurface(EglCore eglCore, Surface surface, boolean releaseSurface) {
+ super(eglCore);
+ createWindowSurface(surface);
+ mSurface = surface;
+ mReleaseSurface = releaseSurface;
+ }
+
+ /**
+ * Associates an EGL surface with the SurfaceTexture.
+ */
+ public WindowSurface(EglCore eglCore, SurfaceTexture surfaceTexture) {
+ super(eglCore);
+ createWindowSurface(surfaceTexture);
+ }
+
+ /**
+ * Releases any resources associated with the EGL surface (and, if configured to do so,
+ * with the Surface as well).
+ *
+ * Does not require that the surface's EGL context be current.
+ */
+ public void release() {
+ releaseEglSurface();
+ if (mSurface != null) {
+ if (mReleaseSurface) {
+ mSurface.release();
+ }
+ mSurface = null;
+ }
+ }
+
+ /**
+ * Recreate the EGLSurface, using the new EglBase. The caller should have already
+ * freed the old EGLSurface with releaseEglSurface().
+ *
+ * This is useful when we want to update the EGLSurface associated with a Surface.
+ * For example, if we want to share with a different EGLContext, which can only
+ * be done by tearing down and recreating the context. (That's handled by the caller;
+ * this just creates a new EGLSurface for the Surface we were handed earlier.)
+ *
+ * If the previous EGLSurface isn't fully destroyed, e.g. it's still current on a
+ * context somewhere, the create call will fail with complaints from the Surface
+ * about already being connected.
+ */
+ public void recreate(EglCore newEglCore) {
+ if (mSurface == null) {
+ throw new RuntimeException("not yet implemented for SurfaceTexture");
+ }
+ mEglCore = newEglCore; // switch to new context
+ createWindowSurface(mSurface); // create new surface
+ }
+}