diff --git a/encoder/src/main/java/com/pedro/encoder/input/gl/render/filters/AndroidViewFilterRender.java b/encoder/src/main/java/com/pedro/encoder/input/gl/render/filters/AndroidViewFilterRender.java index 4bfaa0792..4fb7ab4dc 100644 --- a/encoder/src/main/java/com/pedro/encoder/input/gl/render/filters/AndroidViewFilterRender.java +++ b/encoder/src/main/java/com/pedro/encoder/input/gl/render/filters/AndroidViewFilterRender.java @@ -28,293 +28,556 @@ import android.os.Build; import android.os.Handler; import android.os.Looper; +import android.util.Log; import android.view.Surface; import android.view.View; +import android.view.ViewGroup; +import android.view.ViewTreeObserver; import androidx.annotation.RequiresApi; import com.pedro.encoder.R; import com.pedro.encoder.input.gl.AndroidViewSprite; +import com.pedro.encoder.input.gl.render.filters.BaseFilterRender; import com.pedro.encoder.utils.gl.GlUtil; import com.pedro.encoder.utils.gl.TranslateTo; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; /** * Created by pedro on 4/02/18. + * + * Renders any Android View onto an OpenGL texture that can be composited with + * a camera feed. Supports two dirty-detection strategies: + * + * - Normal Views: ViewTreeObserver.OnDrawListener fires on the main thread + * before each draw pass, setting the dirty flag. + * + * - ObservableWebView: invalidate() is overridden to set the dirty flag + * directly from the compositor thread, which is more reliable for off-screen + * WebView rendering where ViewTreeObserver does not fire. + * + * When dirty is false the render thread sleeps and draws nothing — effectively + * 0 FPS when the view is static. When content changes, it draws up to + * TARGET_FPS. */ - @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) public class AndroidViewFilterRender extends BaseFilterRender { - //rotation matrix - private final float[] squareVertexDataFilter = { + private static final String TAG = "AndroidViewFilterRender"; + private static final int TARGET_FPS = 30; + private static final long FRAME_INTERVAL_MS = 1000L / TARGET_FPS; + + // Vertex data: X, Y, Z, U, V + private final float[] squareVertexDataFilter = { // X, Y, Z, U, V -1f, -1f, 0f, 0f, 0f, //bottom left 1f, -1f, 0f, 1f, 0f, //bottom right -1f, 1f, 0f, 0f, 1f, //top left 1f, 1f, 0f, 1f, 1f, //top right - }; - - private int program = -1; - private int aPositionHandle = -1; - private int aTextureHandle = -1; - private int uMVPMatrixHandle = -1; - private int uSTMatrixHandle = -1; - private int uSamplerHandle = -1; - private int uSamplerViewHandle = -1; - - private int[] viewId = new int[] { -1, -1 }; - private View view; - //Use 2 surfaces to avoid block render thread - private SurfaceTexture surfaceTexture, surfaceTexture2; - private Surface surface, surface2; - private final Handler mainHandler; - private boolean running = false; - private ExecutorService thread = null; - private boolean hardwareMode = true; - private final AndroidViewSprite sprite; - private volatile Status renderingStatus = Status.DONE1; - - private enum Status { - RENDER1, RENDER2, DONE1, DONE2 - } - - public AndroidViewFilterRender() { - squareVertex = ByteBuffer.allocateDirect(squareVertexDataFilter.length * FLOAT_SIZE_BYTES) - .order(ByteOrder.nativeOrder()) - .asFloatBuffer(); - squareVertex.put(squareVertexDataFilter).position(0); - Matrix.setIdentityM(MVPMatrix, 0); - Matrix.setIdentityM(STMatrix, 0); - sprite = new AndroidViewSprite(); - mainHandler = new Handler(Looper.getMainLooper()); - } - - @Override - protected void initGlFilter(Context context) { - String vertexShader = GlUtil.getStringFromRaw(context, R.raw.simple_vertex); - String fragmentShader = GlUtil.getStringFromRaw(context, R.raw.android_view_fragment); - - program = GlUtil.createProgram(vertexShader, fragmentShader); - aPositionHandle = GLES20.glGetAttribLocation(program, "aPosition"); - aTextureHandle = GLES20.glGetAttribLocation(program, "aTextureCoord"); - uMVPMatrixHandle = GLES20.glGetUniformLocation(program, "uMVPMatrix"); - uSTMatrixHandle = GLES20.glGetUniformLocation(program, "uSTMatrix"); - uSamplerHandle = GLES20.glGetUniformLocation(program, "uSampler"); - uSamplerViewHandle = GLES20.glGetUniformLocation(program, "uSamplerView"); - - GlUtil.createExternalTextures(viewId.length, viewId, 0); - surfaceTexture = new SurfaceTexture(viewId[0]); - surfaceTexture2 = new SurfaceTexture(viewId[1]); - surface = new Surface(surfaceTexture); - surface2 = new Surface(surfaceTexture2); - } - - @Override - protected void drawFilter() { - final Status status = renderingStatus; - switch (status) { - case DONE1: + }; + + private int program = -1; + private int aPositionHandle = -1; + private int aTextureHandle = -1; + private int uMVPMatrixHandle = -1; + private int uSTMatrixHandle = -1; + private int uSamplerHandle = -1; + private int uSamplerViewHandle = -1; + private int uDrawViewHandle = -1; + + private int[] viewId = new int[] { -1, -1 }; + private View view; + + // Two surfaces so the render thread never blocks the GL thread + private SurfaceTexture surfaceTexture, surfaceTexture2; + private Surface surface, surface2; + + private final Handler mainHandler; + private volatile boolean running = false; + private ExecutorService thread = null; + private boolean hardwareMode = true; + private final AndroidViewSprite sprite; + + private volatile Status renderingStatus = Status.DONE1; + + // Each startRender() increments this. Threads capture their own value and + // self-terminate when the global value no longer matches. + private volatile int renderGeneration = 0; + + // Tracked so stopRender() can cancel only our own callback, not unrelated ones + private volatile Runnable pendingMainThreadDraw = null; + + // Track current preview dimensions to avoid redundant GL updates + private int currentWidth = -1; + private int currentHeight = -1; + + // When false, the render thread stays alive but skips all drawing + private boolean render = true; + + // Dirty flag: true means the view has new content and needs a frame drawn. + // Initialised to true so the very first frame always renders. + private volatile boolean dirty = true; + + // Fired by ViewTreeObserver for non-WebView views + private final ViewTreeObserver.OnDrawListener onDrawListener = () -> dirty = true; + + private enum Status { + RENDER1, RENDER2, DONE1, DONE2 + } + + // ------------------------------------------------------------------------- + // Constructors + // ------------------------------------------------------------------------- + + public AndroidViewFilterRender() { + this(true); + } + + /** + * @param render if false, drawing is suppressed until setRender(true) is + * called. + * The render thread is still started so switching back is + * instant. + */ + public AndroidViewFilterRender(boolean render) { + this.render = render; + squareVertex = ByteBuffer.allocateDirect(squareVertexDataFilter.length * FLOAT_SIZE_BYTES) + .order(ByteOrder.nativeOrder()) + .asFloatBuffer(); + squareVertex.put(squareVertexDataFilter).position(0); + Matrix.setIdentityM(MVPMatrix, 0); + Matrix.setIdentityM(STMatrix, 0); + sprite = new AndroidViewSprite(); + mainHandler = new Handler(Looper.getMainLooper()); + } + + @Override + protected void initGlFilter(Context context) { + String vertexShader = GlUtil.getStringFromRaw(context, R.raw.simple_vertex); + String fragmentShader = GlUtil.getStringFromRaw(context, R.raw.android_view_fragment); + + program = GlUtil.createProgram(vertexShader, fragmentShader); + aPositionHandle = GLES20.glGetAttribLocation(program, "aPosition"); + aTextureHandle = GLES20.glGetAttribLocation(program, "aTextureCoord"); + uMVPMatrixHandle = GLES20.glGetUniformLocation(program, "uMVPMatrix"); + uSTMatrixHandle = GLES20.glGetUniformLocation(program, "uSTMatrix"); + uSamplerHandle = GLES20.glGetUniformLocation(program, "uSampler"); + uSamplerViewHandle = GLES20.glGetUniformLocation(program, "uSamplerView"); + uDrawViewHandle = GLES20.glGetUniformLocation(program, "uDrawView"); + + GlUtil.createExternalTextures(viewId.length, viewId, 0); + surfaceTexture = new SurfaceTexture(viewId[0]); + surfaceTexture2 = new SurfaceTexture(viewId[1]); + surface = new Surface(surfaceTexture); + surface2 = new Surface(surfaceTexture2); + } + + @Override + protected void drawFilter() { + final Status status = renderingStatus; + switch (status) { + case DONE1: surfaceTexture.setDefaultBufferSize(getPreviewWidth(), getPreviewHeight()); - surfaceTexture.updateTexImage(); - renderingStatus = Status.RENDER2; - break; - case DONE2: - surfaceTexture2.setDefaultBufferSize(getPreviewWidth(), getPreviewHeight()); - surfaceTexture2.updateTexImage(); - renderingStatus = Status.RENDER1; - break; - case RENDER1: + surfaceTexture.updateTexImage(); + renderingStatus = Status.RENDER2; + break; + case DONE2: surfaceTexture2.setDefaultBufferSize(getPreviewWidth(), getPreviewHeight()); - surfaceTexture2.updateTexImage(); - break; - case RENDER2: - default: - surfaceTexture.setDefaultBufferSize(getPreviewWidth(), getPreviewHeight()); - surfaceTexture.updateTexImage(); - break; - } + surfaceTexture2.updateTexImage(); + renderingStatus = Status.RENDER1; + break; + case RENDER1: + // No new frame. Keep using surfaceTexture2. + break; + case RENDER2: + default: + // No new frame. Keep using surfaceTexture. + break; + } - GLES20.glUseProgram(program); + GLES20.glUseProgram(program); - squareVertex.position(SQUARE_VERTEX_DATA_POS_OFFSET); - GLES20.glVertexAttribPointer(aPositionHandle, 3, GLES20.GL_FLOAT, false, - SQUARE_VERTEX_DATA_STRIDE_BYTES, squareVertex); - GLES20.glEnableVertexAttribArray(aPositionHandle); + squareVertex.position(SQUARE_VERTEX_DATA_POS_OFFSET); + GLES20.glVertexAttribPointer(aPositionHandle, 3, GLES20.GL_FLOAT, false, + SQUARE_VERTEX_DATA_STRIDE_BYTES, squareVertex); + GLES20.glEnableVertexAttribArray(aPositionHandle); - squareVertex.position(SQUARE_VERTEX_DATA_UV_OFFSET); - GLES20.glVertexAttribPointer(aTextureHandle, 2, GLES20.GL_FLOAT, false, - SQUARE_VERTEX_DATA_STRIDE_BYTES, squareVertex); - GLES20.glEnableVertexAttribArray(aTextureHandle); + squareVertex.position(SQUARE_VERTEX_DATA_UV_OFFSET); + GLES20.glVertexAttribPointer(aTextureHandle, 2, GLES20.GL_FLOAT, false, + SQUARE_VERTEX_DATA_STRIDE_BYTES, squareVertex); + GLES20.glEnableVertexAttribArray(aTextureHandle); - GLES20.glUniformMatrix4fv(uMVPMatrixHandle, 1, false, MVPMatrix, 0); - GLES20.glUniformMatrix4fv(uSTMatrixHandle, 1, false, STMatrix, 0); + GLES20.glUniformMatrix4fv(uMVPMatrixHandle, 1, false, MVPMatrix, 0); + GLES20.glUniformMatrix4fv(uSTMatrixHandle, 1, false, STMatrix, 0); + GLES20.glUniform1f(uDrawViewHandle, render ? 1.0f : 0.0f); - GLES20.glUniform1i(uSamplerHandle, 0); - GLES20.glActiveTexture(GLES20.GL_TEXTURE0); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, previousTexId); + GLES20.glUniform1i(uSamplerHandle, 0); + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, previousTexId); //android view - GLES20.glUniform1i(uSamplerViewHandle, 1); - GLES20.glActiveTexture(GLES20.GL_TEXTURE1); - - switch (status) { - case DONE2: - case RENDER1: - GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, viewId[1]); - break; - case RENDER2: - case DONE1: - default: - GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, viewId[0]); - break; + GLES20.glUniform1i(uSamplerViewHandle, 1); + GLES20.glActiveTexture(GLES20.GL_TEXTURE1); + + switch (status) { + case DONE2: + case RENDER1: + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, viewId[1]); + break; + case RENDER2: + case DONE1: + default: + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, viewId[0]); + break; + } } - } - - @Override - protected void disableResources() { - GlUtil.disableResources(aTextureHandle, aPositionHandle); - } - - @Override - public void release() { - stopRender(); - GLES20.glDeleteProgram(program); - viewId = new int[] { -1, -1 }; + + @Override + protected void disableResources() { + GLES20.glActiveTexture(GLES20.GL_TEXTURE1); + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0); + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0); + + GlUtil.disableResources(aTextureHandle, aPositionHandle); + } + + @Override + public void release() { + stopRender(); + GLES20.glDeleteProgram(program); + viewId = new int[] { -1, -1 }; if (surfaceTexture != null) surfaceTexture.release(); if (surfaceTexture2 != null) surfaceTexture2.release(); - } - - public View getView() { - return view; - } - - public void setView(final View view) { - stopRender(); - this.view = view; - if (view != null) { - view.measure(View.MeasureSpec.UNSPECIFIED, View.MeasureSpec.UNSPECIFIED); - sprite.setView(view); - startRender(); } - } - - /** - * - * @param x Position in percent - * @param y Position in percent - */ - public void setPosition(float x, float y) { - sprite.translate(x, y); - } - - public void setPosition(TranslateTo positionTo) { - sprite.translate(positionTo); - } - - public void setRotation(int rotation) { - sprite.setRotation(rotation); - } - - public void setScale(float scaleX, float scaleY) { - sprite.scale(scaleX, scaleY); - } - - public PointF getScale() { - return sprite.getScale(); - } - - public PointF getPosition() { - return sprite.getTranslation(); - } - - public int getRotation() { - return sprite.getRotation(); - } - - public boolean isHardwareMode() { - return Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && hardwareMode; - } - - /** - * Draw in surface using hardware canvas. True by default - */ - public void setHardwareMode(boolean hardwareMode) { - this.hardwareMode = hardwareMode; - } - - private void startRender() { - running = true; - thread = Executors.newSingleThreadExecutor(); - thread.execute(() -> { - while (running) { - final Status status = renderingStatus; - if (status == Status.RENDER1 || status == Status.RENDER2) { - final Canvas canvas; - try { - if (isHardwareMode()) { - canvas = status == Status.RENDER1 ? surface.lockHardwareCanvas() : surface2.lockHardwareCanvas(); - } else { - canvas = status == Status.RENDER1 ? surface.lockCanvas(null) : surface2.lockCanvas(null); + + public View getView() { + return view; + } + + /** + * Set the view to render onto the GL texture. + * + * Pass an {@link ObservableWebView} for accurate dirty-detection with WebView. + * For all other View subclasses, ViewTreeObserver.OnDrawListener is used. + * + * Passing the same instance that is already set is a no-op. + */ + public void setView(final View view) { + if (this.view == view) + return; + + unregisterDirtyListener(this.view); + stopRender(); + + this.view = view; + + if (view != null) { + if (view.getMeasuredWidth() <= 0 || view.getMeasuredHeight() <= 0) { + view.measure(View.MeasureSpec.makeMeasureSpec(getPreviewWidth(), View.MeasureSpec.EXACTLY), + View.MeasureSpec.makeMeasureSpec(getPreviewHeight(), View.MeasureSpec.EXACTLY)); + view.layout(0, 0, view.getMeasuredWidth(), view.getMeasuredHeight()); + } + sprite.setView(view); + dirty = true; // always render at least one frame for the new view + registerDirtyListener(view); + startRender(); + } + } + + /** + * Force a redraw on the next render cycle regardless of dirty state. + * Useful when WebView content changes via JavaScript without triggering + * invalidate() + * (rare, but possible with some JS frameworks). + */ + public void markDirty() { + dirty = true; + } + + // ------------------------------------------------------------------------- + // Dirty listener + // ------------------------------------------------------------------------- + + private void registerDirtyListener(View view) { + if (view == null) + return; + if (view instanceof ObservableWebView) { + ((ObservableWebView) view).setOnInvalidateListener(() -> dirty = true); + } else if (view instanceof ViewGroup) { + ViewGroup group = (ViewGroup) view; + for (int i = 0; i < group.getChildCount(); i++) { + registerDirtyListener(group.getChildAt(i)); + } + } + // Always register OnDrawListener for the view itself + try { + view.getViewTreeObserver().addOnDrawListener(onDrawListener); + } catch (Exception ignored) { + } + } + + private void unregisterDirtyListener(View view) { + if (view == null) + return; + if (view instanceof ObservableWebView) { + ((ObservableWebView) view).clearOnInvalidateListener(); + } else if (view instanceof ViewGroup) { + ViewGroup group = (ViewGroup) view; + for (int i = 0; i < group.getChildCount(); i++) { + unregisterDirtyListener(group.getChildAt(i)); } - } catch (IllegalStateException e) { - continue; - } - - sprite.calculateDefaultScale(getPreviewWidth(), getPreviewHeight()); - PointF canvasPosition = sprite.getCanvasPosition(getPreviewWidth(), getPreviewHeight()); - PointF canvasScale = sprite.getCanvasScale(getPreviewWidth(), getPreviewHeight()); - PointF rotationAxis = sprite.getRotationAxis(); - int rotation = sprite.getRotation(); - - canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR); - canvas.translate(canvasPosition.x, canvasPosition.y); - canvas.scale(canvasScale.x, canvasScale.y); - canvas.rotate(rotation, rotationAxis.x, rotationAxis.y); - try { - view.draw(canvas); - if (status == Status.RENDER1) { - surface.unlockCanvasAndPost(canvas); - renderingStatus = Status.DONE1; - } else { - surface2.unlockCanvasAndPost(canvas); - renderingStatus = Status.DONE2; + } + try { + view.getViewTreeObserver().removeOnDrawListener(onDrawListener); + } catch (Exception ignored) { + } + } + + // ------------------------------------------------------------------------- + // Transform / property setters & getters + // ------------------------------------------------------------------------- + + /** + * @param x Position in percent + * @param y Position in percent + */ + public void setPosition(float x, float y) { + sprite.translate(x, y); + } + + public void setPosition(TranslateTo positionTo) { + sprite.translate(positionTo); + } + + public void setScale(float scaleX, float scaleY) { + sprite.scale(scaleX, scaleY); + } + + public void setRotation(int rotation) { + sprite.setRotation(rotation); + } + + public PointF getPosition() { + return sprite.getTranslation(); + } + + public PointF getScale() { + return sprite.getScale(); + } + + public int getRotation() { + return sprite.getRotation(); + } + + public boolean isHardwareMode() { + return Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && hardwareMode; + } + + /** + * Draw in surface using hardware canvas. True by default + */ + public void setHardwareMode(boolean hardwareMode) { + this.hardwareMode = hardwareMode; + } + + public boolean isRender() { + return render; + } + + /** + * Enable or disable rendering. When false the render thread stays alive + * but skips all drawing until re-enabled. + */ + public void setRender(boolean render) { + this.render = render; + if (render) + dirty = true; // force a frame immediately on re-enable + } + + // ------------------------------------------------------------------------- + // Render thread + // ------------------------------------------------------------------------- + + private void startRender() { + running = true; + final int generation = ++renderGeneration; + thread = Executors.newSingleThreadExecutor(); + thread.execute(() -> { + Log.d(TAG, "render thread started, gen=" + generation); + while (running && renderGeneration == generation) { + final long frameStart = System.currentTimeMillis(); + final Status status = renderingStatus; + + if (status == Status.RENDER1 || status == Status.RENDER2) { + + // If rendering is disabled or we have no new content, + // idle cheaply yielding to the system. + if (!render || !dirty) { + try { + Thread.sleep(FRAME_INTERVAL_MS); + } catch (InterruptedException e) { + break; + } + continue; + } + + // Clear before locking so any invalidation during the draw + // isn't silently dropped + dirty = false; + + // Lock the appropriate surface canvas + final Canvas canvas; + try { + if (isHardwareMode()) { + canvas = (status == Status.RENDER1) + ? surface.lockHardwareCanvas() + : surface2.lockHardwareCanvas(); + } else { + canvas = (status == Status.RENDER1) + ? surface.lockCanvas(null) + : surface2.lockCanvas(null); + } + } catch (IllegalStateException e) { + // Surface not ready yet — back off briefly + try { + Thread.sleep(FRAME_INTERVAL_MS); + } catch (InterruptedException ie) { + break; + } + continue; + } + + // Start with a clean slate + canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR); + + if (render) { + // Apply sprite transform + sprite.calculateDefaultScale(getPreviewWidth(), getPreviewHeight()); + PointF canvasPosition = sprite.getCanvasPosition(getPreviewWidth(), getPreviewHeight()); + PointF canvasScale = sprite.getCanvasScale(getPreviewWidth(), getPreviewHeight()); + PointF rotationAxis = sprite.getRotationAxis(); + int rotation = sprite.getRotation(); + + canvas.translate(canvasPosition.x, canvasPosition.y); + canvas.scale(canvasScale.x, canvasScale.y); + canvas.rotate(rotation, rotationAxis.x, rotationAxis.y); + + // Draw on this thread; fall back to main thread if the view + // requires it + try { + Log.d(TAG, "draw view, gen=" + generation); + view.draw(canvas); + postCanvas(canvas, status); + } catch (Exception e) { + final CountDownLatch latch = new CountDownLatch(1); + pendingMainThreadDraw = () -> { + try { + if (renderGeneration != generation) { + Log.w(TAG, "Main thread fallback aborted due to generation change. Gen: " + + generation + ", Global: " + renderGeneration); + return; + } + Log.d(TAG, "draw view (main thread fallback), gen=" + generation); + view.draw(canvas); + postCanvas(canvas, status); + } catch (Exception ex) { + Log.e(TAG, "Main thread fallback draw failed", ex); + // CRITICAL: Must still unlock even if draw fails + try { + if (status == Status.RENDER1) + surface.unlockCanvasAndPost(canvas); + else + surface2.unlockCanvasAndPost(canvas); + } catch (Exception ignored) { + } + } finally { + latch.countDown(); + } + }; + mainHandler.post(pendingMainThreadDraw); + try { + if (!latch.await(FRAME_INTERVAL_MS * 2, TimeUnit.MILLISECONDS)) { + Log.w(TAG, + "Main thread fallback latch timed out. Possible deadlock or slow UI thread."); + } + } catch (InterruptedException ie) { + break; + } + pendingMainThreadDraw = null; + } + } else { + // Rendering disabled: post the clear canvas to keep GL textures valid + postCanvas(canvas, status); + } + + // Sleep for the remaining frame budget + long sleepMs = (render ? FRAME_INTERVAL_MS : (FRAME_INTERVAL_MS * 2)) + - (System.currentTimeMillis() - frameStart); + if (sleepMs > 0) { + try { + Thread.sleep(sleepMs); + } catch (InterruptedException e) { + break; + } + } + + } else { + // Not in a renderable state — sleep before rechecking + try { + Thread.sleep(FRAME_INTERVAL_MS); + } catch (InterruptedException e) { + break; + } + } } - //Sometimes draw could crash if you don't use main thread. Ensuring you can render always - } catch (Exception e) { - mainHandler.post(() -> { - view.draw(canvas); - if (status == Status.RENDER1) { - surface.unlockCanvasAndPost(canvas); - renderingStatus = Status.DONE1; - } else { - surface2.unlockCanvasAndPost(canvas); - renderingStatus = Status.DONE2; - } - }); - } + Log.d(TAG, "render thread exiting, gen=" + generation); + }); + + } + + private void postCanvas(Canvas canvas, Status status) { + if (status == Status.RENDER1) { + surface.unlockCanvasAndPost(canvas); + renderingStatus = Status.DONE1; + } else { + surface2.unlockCanvasAndPost(canvas); + renderingStatus = Status.DONE2; } - else { - // not rendering, no need to try again immediately - try { - Thread.sleep(10); - } catch (InterruptedException e) { + } + + private void stopRender() { + running = false; + + // Unregister dirty listener before bumping generation + unregisterDirtyListener(view); - } + // Bump generation — any live threads and their queued callbacks will + // see the mismatch and self-terminate / bail without drawing. + // Doing this before removing callbacks prevents new ones from starting. + renderGeneration++; + + // Cancel only our own pending main-thread callback + if (pendingMainThreadDraw != null) { + mainHandler.removeCallbacks(pendingMainThreadDraw); + pendingMainThreadDraw = null; } - } - }); - } - - private void stopRender() { - running = false; - if (thread != null) { - thread.shutdownNow(); - thread = null; + + if (thread != null) { + thread.shutdownNow(); + try { + thread.awaitTermination(500, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + thread = null; + } + + renderingStatus = Status.DONE1; } - renderingStatus = Status.DONE1; - } } \ No newline at end of file diff --git a/encoder/src/main/java/com/pedro/encoder/input/video/Camera2ApiManager.kt b/encoder/src/main/java/com/pedro/encoder/input/video/Camera2ApiManager.kt index d3b720420..6da2e467f 100644 --- a/encoder/src/main/java/com/pedro/encoder/input/video/Camera2ApiManager.kt +++ b/encoder/src/main/java/com/pedro/encoder/input/video/Camera2ApiManager.kt @@ -190,11 +190,34 @@ class Camera2ApiManager(context: Context) : CameraDevice.StateCallback() { @Throws(IllegalStateException::class, Exception::class) private fun drawSurface(cameraDevice: CameraDevice, surfaces: List): CaptureRequest { - val builderInputSurface = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW) + val builderInputSurface = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_RECORD) for (surface in surfaces) builderInputSurface.addTarget(surface) builderInputSurface.set(CaptureRequest.CONTROL_MODE, CameraMetadata.CONTROL_MODE_AUTO) val validFps = min(60, fps) - builderInputSurface.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(validFps, validFps)) + // Find best FPS range instead of forcing strict [30, 30] which causes HAL duplication stutter + var bestRange = Range(validFps, validFps) + try { + val facing = if (cameraId == "1") Facing.FRONT else Facing.BACK + val supportedRanges = getSupportedFps(null, facing) + + // Look for a range that maxes out at our target FPS, but allows dipping to save light (e.g. [24, 30] or [15, 30]) + for (range in supportedRanges) { + if (range.upper == validFps && range.lower < validFps) { + // Try to avoid dropping too low (e.g. [15, 30] can be too choppy, prefer [24, 30]) + if (range.lower >= 24) { + bestRange = range + break + } else if (bestRange.lower == validFps || range.lower > bestRange.lower) { + bestRange = range + } + } + } + Log.i(TAG, "Selected dynamic FPS range: $bestRange for target $validFps") + } catch (e: Exception) { + Log.e(TAG, "Error finding dynamic FPS range", e) + } + + builderInputSurface.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, bestRange) this.builderInputSurface = builderInputSurface return builderInputSurface.build() } @@ -605,7 +628,12 @@ class Camera2ApiManager(context: Context) : CameraDevice.StateCallback() { builderInputSurface.set(CaptureRequest.CONTROL_AF_TRIGGER, CameraMetadata.CONTROL_AF_TRIGGER_CANCEL) builderInputSurface.set(CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_OFF) applyRequest(builderInputSurface) - if (supportedFocusModes.contains(CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE)) { + // Prefer CONTINUOUS_VIDEO: same smooth AF as CONTINUOUS_PICTURE but defers + // fine adjustments to not interrupt frame delivery (no dropped frames on focus hunt). + if (supportedFocusModes.contains(CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_VIDEO)) { + builderInputSurface.set(CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_VIDEO) + isAutoFocusEnabled = true + } else if (supportedFocusModes.contains(CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE)) { builderInputSurface.set(CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE) isAutoFocusEnabled = true } else if (supportedFocusModes.contains(CaptureRequest.CONTROL_AF_MODE_AUTO)) { diff --git a/encoder/src/main/java/com/pedro/encoder/utils/SpsColorPatcher.java b/encoder/src/main/java/com/pedro/encoder/utils/SpsColorPatcher.java new file mode 100644 index 000000000..4306ae9a4 --- /dev/null +++ b/encoder/src/main/java/com/pedro/encoder/utils/SpsColorPatcher.java @@ -0,0 +1,418 @@ +/* + * Copyright (C) 2024 pedroSG94. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedro.encoder.utils; + +import android.media.MediaFormat; +import android.util.Log; + +import java.nio.ByteBuffer; + +/** + * Patches H.264 / H.265 SPS NAL units to embed BT.709 colour metadata. + * + * Hardware encoders typically ignore MediaFormat KEY_COLOR_* when generating + * the SPS VUI, so we must fix the raw bytes directly. + * + * Call sites: + * - Recording path: patchMediaFormatColorToBt709(videoFormat) before + * mediaMuxer.addTrack() + * - Streaming path: patchSpsNalColorToBt709(spsBytes, isHevc) before + * onVideoInfo() + */ +public final class SpsColorPatcher { + + private static final String TAG = "COLOR_PATCH"; + + private SpsColorPatcher() { + } + + /** + * Patches csd-0 in a MediaFormat in-place before mediaMuxer.addTrack(). + * Auto-detects H.264 vs H.265 from the MIME type. + */ + public static void patchMediaFormatColorToBt709(MediaFormat format) { + if (format == null || !format.containsKey("csd-0")) + return; + ByteBuffer csd0 = format.getByteBuffer("csd-0"); + String mime = format.containsKey(MediaFormat.KEY_MIME) ? format.getString(MediaFormat.KEY_MIME) : ""; + ByteBuffer patched; + if ("video/hevc".equals(mime)) { + patched = patchSpsNalColorToBt709(csd0, true); + } else { + // Try H.264 first; fall back to H.265 scan + patched = patchSpsNalColorToBt709(csd0, false); + if (patched == csd0) + patched = patchSpsNalColorToBt709(csd0, true); + } + if (patched != csd0) { + format.setByteBuffer("csd-0", patched); + Log.d(TAG, "csd-0 patched with BT.709 colour info"); + } + } + + /** + * Patches colour_primaries / transfer_characteristics / matrix_coefficients + * to BT.709 (1, 1, 1) in a raw SPS NAL ByteBuffer. + * + * Works for both: + * - A full csd-0 buffer containing one or more NAL units (recording) + * - A single raw SPS NAL ByteBuffer (streaming, passed to onVideoInfo) + * + * @param spsData ByteBuffer containing the NAL bytes + * @param isHevc true for H.265/HEVC, false for H.264/AVC + * @return Patched ByteBuffer (new allocation), or original if patching + * failed/not needed + */ + public static ByteBuffer patchSpsNalColorToBt709(ByteBuffer spsData, boolean isHevc) { + if (spsData == null || spsData.remaining() < 4) + return spsData; + byte[] data = new byte[spsData.remaining()]; + int savedPos = spsData.position(); + spsData.get(data); + spsData.position(savedPos); + + return isHevc + ? patchH265(data) + : patchH264(data); + } + + // ------------------------------------------------------------------------- + // H.264 SPS patcher + // ------------------------------------------------------------------------- + + private static ByteBuffer patchH264(byte[] data) { + // Find SPS NAL unit (NAL type 7) + int spsStart = -1; + for (int i = 0; i < data.length; i++) { + if ((data[i] & 0x1F) == 7) { + spsStart = i + 1; + break; + } + } + if (spsStart < 0) { + Log.w(TAG, "H.264 SPS NAL (type 7) not found"); + return ByteBuffer.wrap(data); + } + + byte[] patched = data.clone(); + BitReader br = new BitReader(patched, spsStart); + try { + int profile_idc = br.readBits(8); + br.skipBits(16); // constraint flags + level_idc + br.readUVLC(); // seq_parameter_set_id + + if (profile_idc == 100 || profile_idc == 110 || profile_idc == 122 || profile_idc == 244 + || profile_idc == 44 || profile_idc == 83 || profile_idc == 86 + || profile_idc == 118 || profile_idc == 128 || profile_idc == 138) { + int chroma = br.readUVLC(); + if (chroma == 3) + br.skipBits(1); + br.readUVLC(); + br.readUVLC(); + br.skipBits(1); + if (br.readBits(1) != 0) { + int cnt = (chroma != 3) ? 8 : 12; + for (int i = 0; i < cnt; i++) + if (br.readBits(1) != 0) + br.skipScalingList(i < 6 ? 16 : 64); + } + } + + br.readUVLC(); // log2_max_frame_num_minus4 + int poc = br.readUVLC(); + if (poc == 0) { + br.readUVLC(); + } else if (poc == 1) { + br.skipBits(1); + br.readSVLC(); + br.readSVLC(); + int n = br.readUVLC(); + for (int i = 0; i < n; i++) + br.readSVLC(); + } + + br.readUVLC(); + br.skipBits(1); // max_num_ref_frames, gaps_flag + br.readUVLC(); + br.readUVLC(); // pic_width/height + if (br.readBits(1) == 0) + br.skipBits(1); // frame_mbs_only_flag / mb_adaptive + br.skipBits(1); // direct_8x8_inference_flag + if (br.readBits(1) != 0) { + br.readUVLC(); + br.readUVLC(); + br.readUVLC(); + br.readUVLC(); + } // crop + + if (br.readBits(1) == 0) { + Log.w(TAG, "H.264 no VUI"); + return ByteBuffer.wrap(data); + } + + if (br.readBits(1) != 0) { + if (br.readBits(8) == 255) + br.skipBits(32); + } // aspect ratio + if (br.readBits(1) != 0) + br.skipBits(1); // overscan + + if (br.readBits(1) != 0) { // video_signal_type_present_flag + br.skipBits(4); // video_format + full_range + if (br.readBits(1) != 0) { // colour_description_present_flag + br.writeBits(1, 8); // colour_primaries = BT.709 + br.writeBits(1, 8); // transfer_characteristics = BT.709 + br.writeBits(1, 8); // matrix_coefficients = BT.709 + Log.d(TAG, "H.264 SPS colour patched → BT.709 at bit " + br.getBitPos()); + return ByteBuffer.wrap(patched); + } + Log.w(TAG, "H.264 colour_description_present_flag=0"); + } else { + Log.w(TAG, "H.264 video_signal_type_present_flag=0"); + } + } catch (Exception e) { + Log.e(TAG, "H.264 SPS parse error", e); + } + return ByteBuffer.wrap(data); + } + + // ------------------------------------------------------------------------- + // H.265 SPS patcher + // ------------------------------------------------------------------------- + + private static ByteBuffer patchH265(byte[] data) { + // Find H.265 SPS NAL unit (nal_unit_type = 33, 2-byte HEVC NAL header) + int spsStart = -1; + for (int i = 0; i < data.length - 1; i++) { + if (((data[i] >> 1) & 0x3F) == 33) { + spsStart = i + 2; + break; + } + } + if (spsStart < 0) { + Log.w(TAG, "H.265 SPS NAL (type 33) not found"); + return ByteBuffer.wrap(data); + } + + byte[] patched = data.clone(); + BitReader br = new BitReader(patched, spsStart); + try { + br.skipBits(4); // sps_video_parameter_set_id + int M = br.readBits(3); // sps_max_sub_layers_minus1 + br.skipBits(1); // sps_temporal_id_nesting_flag + + // profile_tier_level + br.skipBits(96); + br.skipBits(8); // general profile (96b) + level_idc (8b) + boolean[] profPresent = new boolean[M], levelPresent = new boolean[M]; + for (int i = 0; i < M; i++) { + profPresent[i] = br.readBits(1) != 0; + levelPresent[i] = br.readBits(1) != 0; + } + if (M > 0) + br.skipBits(2 * (8 - M)); + for (int i = 0; i < M; i++) { + if (profPresent[i]) + br.skipBits(96); + if (levelPresent[i]) + br.skipBits(8); + } + + br.readUVLC(); // sps_seq_parameter_set_id + int chroma = br.readUVLC(); + if (chroma == 3) + br.skipBits(1); + br.readUVLC(); + br.readUVLC(); // width, height + if (br.readBits(1) != 0) { + br.readUVLC(); + br.readUVLC(); + br.readUVLC(); + br.readUVLC(); + } // conf window + br.readUVLC(); + br.readUVLC(); // bit depth luma/chroma + int log2MaxPoc = br.readUVLC() + 4; + int start = br.readBits(1) != 0 ? 0 : M; + for (int i = start; i <= M; i++) { + br.readUVLC(); + br.readUVLC(); + br.readUVLC(); + } + for (int i = 0; i < 6; i++) + br.readUVLC(); // log2/diff block/transform sizes + depth inter/intra + if (br.readBits(1) != 0 && br.readBits(1) != 0) + skipH265ScalingListData(br); + br.skipBits(2); // amp_enabled + sample_adaptive_offset + if (br.readBits(1) != 0) { + br.skipBits(8); + br.readUVLC(); + br.readUVLC(); + br.skipBits(1); + } // pcm + + int numStRps = br.readUVLC(); + int[] deltaPocs = new int[numStRps]; + for (int i = 0; i < numStRps; i++) + deltaPocs[i] = skipH265StRefPicSet(br, i, deltaPocs); + if (br.readBits(1) != 0) { + int numLt = br.readUVLC(); + for (int i = 0; i < numLt; i++) { + br.skipBits(log2MaxPoc); + br.skipBits(1); + } + } + br.skipBits(2); // temporal_mvp + strong_intra_smoothing + + if (br.readBits(1) == 0) { + Log.w(TAG, "H.265 no VUI"); + return ByteBuffer.wrap(data); + } + + if (br.readBits(1) != 0) { + if (br.readBits(8) == 255) + br.skipBits(32); + } // aspect ratio + if (br.readBits(1) != 0) + br.skipBits(1); // overscan + + if (br.readBits(1) != 0) { // video_signal_type_present_flag + br.skipBits(4); + if (br.readBits(1) != 0) { // colour_description_present_flag + br.writeBits(1, 8); + br.writeBits(1, 8); + br.writeBits(1, 8); + Log.d(TAG, "H.265 SPS colour patched → BT.709 at bit " + br.getBitPos()); + return ByteBuffer.wrap(patched); + } + Log.w(TAG, "H.265 colour_description_present_flag=0"); + } else { + Log.w(TAG, "H.265 video_signal_type_present_flag=0"); + } + } catch (Exception e) { + Log.e(TAG, "H.265 SPS parse error", e); + } + return ByteBuffer.wrap(data); + } + + private static int skipH265StRefPicSet(BitReader br, int idx, int[] deltaPocs) { + if (idx != 0 && br.readBits(1) != 0) { + if (idx == deltaPocs.length) + br.readUVLC(); + br.skipBits(1); + br.readUVLC(); + int numPics = deltaPocs[idx - 1] + 1; + for (int j = 0; j < numPics; j++) + if (br.readBits(1) == 0) + br.skipBits(1); + return numPics; + } + int neg = br.readUVLC(), pos = br.readUVLC(); + for (int i = 0; i < neg; i++) { + br.readUVLC(); + br.skipBits(1); + } + for (int i = 0; i < pos; i++) { + br.readUVLC(); + br.skipBits(1); + } + return neg + pos; + } + + private static void skipH265ScalingListData(BitReader br) { + for (int s = 0; s < 4; s++) { + int nm = (s == 3) ? 2 : 6; + for (int m = 0; m < nm; m++) { + if (br.readBits(1) == 0) { + br.readUVLC(); + } else { + int coef = Math.min(64, 1 << (4 + (s << 1))); + if (s > 1) + br.readSVLC(); + for (int i = 0; i < coef; i++) + br.readSVLC(); + } + } + } + } + + // ------------------------------------------------------------------------- + // Minimal bit-level reader/writer (Exp-Golomb aware) + // ------------------------------------------------------------------------- + + static final class BitReader { + private final byte[] buf; + private int bitPos; + + BitReader(byte[] buf, int startByte) { + this.buf = buf; + this.bitPos = startByte * 8; + } + + int getBitPos() { + return bitPos; + } + + int readBits(int n) { + int v = 0; + for (int i = 0; i < n; i++) { + int mask = 1 << (7 - (bitPos % 8)); + v = (v << 1) | ((buf[bitPos / 8] & mask) != 0 ? 1 : 0); + bitPos++; + } + return v; + } + + void writeBits(int value, int n) { + for (int i = n - 1; i >= 0; i--) { + int shift = 7 - (bitPos % 8); + if (((value >> i) & 1) != 0) + buf[bitPos / 8] |= (byte) (1 << shift); + else + buf[bitPos / 8] &= (byte) ~(1 << shift); + bitPos++; + } + } + + void skipBits(int n) { + bitPos += n; + } + + int readUVLC() { + int z = 0; + while (readBits(1) == 0) + z++; + return z == 0 ? 0 : (1 << z) - 1 + readBits(z); + } + + int readSVLC() { + int c = readUVLC(); + return (c % 2 == 0) ? -(c / 2) : (c + 1) / 2; + } + + void skipScalingList(int size) { + int last = 8, next = 8; + for (int j = 0; j < size; j++) { + if (next != 0) { + int d = readSVLC(); + next = (last + d + 256) % 256; + } + last = (next == 0) ? last : next; + } + } + } +} diff --git a/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java b/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java index ab5351246..6c3858c91 100644 --- a/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java +++ b/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java @@ -39,6 +39,7 @@ import com.pedro.encoder.input.video.FpsLimiter; import com.pedro.encoder.input.video.GetCameraData; import com.pedro.encoder.utils.CodecUtil; +import com.pedro.encoder.utils.SpsColorPatcher; import com.pedro.encoder.utils.yuv.YUVUtil; import java.nio.ByteBuffer; @@ -168,6 +169,14 @@ public boolean prepareVideoEncoder(int width, int height, int fps, int bitRate, // MediaFormat.KEY_LEVEL, API > 23 videoFormat.setInteger("level", this.level); } + // Set BT.709 color metadata so the encoder embeds correct VUI in the SPS NAL unit. + // Without this, devices default to smpte170m/bt470bg which ffprobe/players read incorrectly. + // KEY_COLOR_STANDARD / KEY_COLOR_TRANSFER / KEY_COLOR_RANGE added in API 24. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + videoFormat.setInteger(MediaFormat.KEY_COLOR_STANDARD, MediaFormat.COLOR_STANDARD_BT709); // primaries + matrix = BT.709 + videoFormat.setInteger(MediaFormat.KEY_COLOR_TRANSFER, MediaFormat.COLOR_TRANSFER_SDR_VIDEO); // transfer = BT.709 (gamma) + videoFormat.setInteger(MediaFormat.KEY_COLOR_RANGE, MediaFormat.COLOR_RANGE_LIMITED); // TV range (16-235) + } setCallback(); codec.configure(videoFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); running = false; @@ -334,7 +343,7 @@ private boolean sendSPSandPPS(MediaFormat mediaFormat) { ByteBuffer bufferInfo = mediaFormat.getByteBuffer("csd-0"); if (bufferInfo != null) { List byteBufferList = VideoEncoderHelper.extractVpsSpsPpsFromH265(bufferInfo.duplicate()); - oldSps = byteBufferList.get(1); + oldSps = SpsColorPatcher.patchSpsNalColorToBt709(byteBufferList.get(1), true); oldPps = byteBufferList.get(2); oldVps = byteBufferList.get(0); getVideoData.onVideoInfo(oldSps, oldPps, oldVps); @@ -345,7 +354,7 @@ private boolean sendSPSandPPS(MediaFormat mediaFormat) { ByteBuffer sps = mediaFormat.getByteBuffer("csd-0"); ByteBuffer pps = mediaFormat.getByteBuffer("csd-1"); if (sps != null && pps != null) { - oldSps = sps.duplicate(); + oldSps = SpsColorPatcher.patchSpsNalColorToBt709(sps.duplicate(), false); oldPps = pps.duplicate(); oldVps = null; getVideoData.onVideoInfo(oldSps, oldPps, oldVps); @@ -468,8 +477,15 @@ protected void checkBuffer(@NonNull ByteBuffer byteBuffer, @NonNull MediaCodec.B } } if (timestampMode == TimestampMode.CLOCK) { - if (formatVideoEncoder == FormatVideoEncoder.SURFACE) { + if (formatVideoEncoder != FormatVideoEncoder.SURFACE) { + // Buffer mode: synthesize PTS from wall clock. bufferInfo.presentationTimeUs = TimeUtils.getCurrentTimeMicro() - presentTimeUs; + } else { + // Surface mode: EGL timestamp is camera sensor time (nanoseconds from boot ÷ 1000). + // It has clean, jitter-free intervals — but it's a huge absolute value that breaks RTMP. + // Rebase to relative by subtracting the first frame's PTS → clean intervals, starts at 0. + if (firstTimestamp == 0) firstTimestamp = bufferInfo.presentationTimeUs; + bufferInfo.presentationTimeUs -= firstTimestamp; } } else { if (firstTimestamp == 0) firstTimestamp = bufferInfo.presentationTimeUs; diff --git a/encoder/src/main/res/raw/android_view_fragment.glsl b/encoder/src/main/res/raw/android_view_fragment.glsl index 4039e1d5c..94709a585 100644 --- a/encoder/src/main/res/raw/android_view_fragment.glsl +++ b/encoder/src/main/res/raw/android_view_fragment.glsl @@ -3,12 +3,17 @@ precision mediump float; uniform samplerExternalOES uSamplerView; uniform sampler2D uSampler; +uniform float uDrawView; varying vec2 vTextureCoord; void main() { vec4 color = texture2D(uSampler, vTextureCoord); - vec4 viewColor = texture2D(uSamplerView, vec2(vTextureCoord.x, 1.0 - vTextureCoord.y)); - color.rgb *= 1.0 - viewColor.a; - gl_FragColor = color + viewColor; + if (uDrawView > 0.5) { + vec4 viewColor = texture2D(uSamplerView, vec2(vTextureCoord.x, 1.0 - vTextureCoord.y)); + color.rgb *= 1.0 - viewColor.a; + gl_FragColor = color + viewColor; + } else { + gl_FragColor = color; + } } \ No newline at end of file diff --git a/library/src/main/java/com/pedro/library/view/GlStreamInterface.kt b/library/src/main/java/com/pedro/library/view/GlStreamInterface.kt index 2a830c203..e12948b3c 100644 --- a/library/src/main/java/com/pedro/library/view/GlStreamInterface.kt +++ b/library/src/main/java/com/pedro/library/view/GlStreamInterface.kt @@ -21,6 +21,8 @@ import android.graphics.Point import android.graphics.SurfaceTexture import android.graphics.SurfaceTexture.OnFrameAvailableListener import android.os.Build +import android.os.Handler +import android.os.HandlerThread import android.view.Surface import androidx.annotation.RequiresApi import com.pedro.common.newSingleThreadExecutor @@ -64,6 +66,8 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener, private val surfaceManagerPreview = SurfaceManager() private val multiPreviewSurfaceManagers = ConcurrentHashMap() private val mainRender = MainRender() + private var prevSensorTs = 0L // for PTS interval logging + private var prevWallTs = 0L // wall-clock comparison (what PTS would be WITHOUT the fix) private var encoderWidth = 0 private var encoderHeight = 0 @@ -92,6 +96,7 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener, private var renderErrorCallback: RenderErrorCallback? = null private var previewViewPort: ViewPort? = null private var streamViewPort: ViewPort? = null + private var surfaceHandlerThread: HandlerThread? = null private val sensorRotationManager = SensorRotationManager(context, true, true) { orientation, isPortrait -> if (autoHandleOrientation && shouldHandleOrientation) { @@ -147,27 +152,33 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener, } override fun addMediaCodecSurface(surface: Surface) { - if (surfaceManager.isReady) { - surfaceManagerEncoder.release() - surfaceManagerEncoder.eglSetup(surface, surfaceManager) + executor?.execute { + if (surfaceManager.isReady) { + surfaceManagerEncoder.release() + surfaceManagerEncoder.eglSetup(surface, surfaceManager) + } } } override fun removeMediaCodecSurface() { - threadQueue.clear() - surfaceManagerEncoder.release() + executor?.execute { + surfaceManagerEncoder.release() + } } override fun addMediaCodecRecordSurface(surface: Surface) { - if (surfaceManager.isReady) { - surfaceManagerEncoderRecord.release() - surfaceManagerEncoderRecord.eglSetup(surface, surfaceManager) + executor?.execute { + if (surfaceManager.isReady) { + surfaceManagerEncoderRecord.release() + surfaceManagerEncoderRecord.eglSetup(surface, surfaceManager) + } } } override fun removeMediaCodecRecordSurface() { - threadQueue.clear() - surfaceManagerEncoderRecord.release() + executor?.execute { + surfaceManagerEncoderRecord.release() + } } override fun takePhoto(takePhotoCallback: TakePhotoCallback?) { @@ -179,6 +190,10 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener, executor?.shutdownNow() executor = null executor = newSingleThreadExecutor(threadQueue) + surfaceHandlerThread?.quitSafely() + surfaceHandlerThread = HandlerThread("GlStreamHandler") + surfaceHandlerThread?.start() + val surfaceHandler = Handler(surfaceHandlerThread!!.looper) val width = max(encoderWidth, encoderRecordWidth) val height = max(encoderHeight, encoderRecordHeight) surfaceManager.release() @@ -190,7 +205,7 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener, surfaceManager.makeCurrent() mainRender.initGl(context, width, height, width, height) running.set(true) - mainRender.getSurfaceTexture().setOnFrameAvailableListener(this) + mainRender.getSurfaceTexture().setOnFrameAvailableListener(this, surfaceHandler) forceRender.start { executor?.execute { try { @@ -205,6 +220,8 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener, override fun stop() { running.set(false) + surfaceHandlerThread?.quitSafely() + surfaceHandlerThread = null threadQueue.clear() executor?.shutdownNow() executor = null @@ -221,9 +238,16 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener, mainRender.release() } + private var prevLimitFps = false + private fun draw(forced: Boolean) { if (!isRunning) return + val drawTotalStart = System.nanoTime() val limitFps = fpsLimiter.limitFPS() + if (limitFps != prevLimitFps) { + android.util.Log.d("FPS_LIMIT", "limitFps → $limitFps (was $prevLimitFps)") + prevLimitFps = limitFps + } if (!forced) forceRender.frameAvailable() if (!filterQueue.isEmpty() && mainRender.isReady()) { @@ -265,6 +289,9 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener, if (surfaceManagerEncoder.makeCurrent()) { mainRender.drawScreenEncoder(w, h, orientation, streamOrientation, isStreamVerticalFlip, isStreamHorizontalFlip, streamViewPort) + val sensorTs = mainRender.getSurfaceTexture().timestamp + val wallTs = System.nanoTime() + surfaceManagerEncoder.setPresentationTime(sensorTs) surfaceManagerEncoder.swapBuffer() } } @@ -275,6 +302,8 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener, if (surfaceManagerEncoderRecord.makeCurrent()) { mainRender.drawScreenEncoder(w, h, orientation, streamOrientation, isStreamVerticalFlip, isStreamHorizontalFlip, streamViewPort) + // Fix: same timestamp fix for the dedicated record surface + surfaceManagerEncoderRecord.setPresentationTime(mainRender.getSurfaceTexture().timestamp) surfaceManagerEncoderRecord.swapBuffer() } } diff --git a/library/src/main/java/com/pedro/library/view/ObservableWebView.java b/library/src/main/java/com/pedro/library/view/ObservableWebView.java new file mode 100644 index 000000000..43bf2b4c1 --- /dev/null +++ b/library/src/main/java/com/pedro/library/view/ObservableWebView.java @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2024 pedroSG94. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedro.library.view; + +import android.content.Context; +import android.util.AttributeSet; +import android.webkit.WebView; + +/** + * A WebView subclass that exposes a dirty-detection callback via invalidate(). + * + * WebView renders through its own compositor thread and does not reliably + * trigger + * ViewTreeObserver.OnDrawListener when used off-screen (e.g. drawn to a + * Surface). + * Instead, the compositor calls invalidate() on the View whenever a new frame + * is + * ready. By overriding invalidate() here we get a precise, zero-overhead signal + * that the WebView content has changed and needs to be redrawn onto the + * Surface. + * + * Usage: + * ObservableWebView webView = new ObservableWebView(context); + * androidViewFilterRender.setView(webView); + * // AndroidViewFilterRender detects ObservableWebView automatically and + * // registers the dirty listener via setOnInvalidateListener(). + */ +public class ObservableWebView extends WebView { + + /** + * Called from the compositor thread whenever WebView has new pixels to show. + * Must be volatile so writes from the compositor thread are visible to the + * render thread immediately. + */ + private volatile Runnable onInvalidateListener; + + public ObservableWebView(Context context) { + super(context); + } + + public ObservableWebView(Context context, AttributeSet attrs) { + super(context, attrs); + } + + public ObservableWebView(Context context, AttributeSet attrs, int defStyleAttr) { + super(context, attrs, defStyleAttr); + } + + /** + * Set a listener that will be called every time WebView invalidates itself. + * The listener runs on whatever thread called invalidate() — typically the + * WebView compositor thread — so keep it fast and thread-safe. + */ + public void setOnInvalidateListener(Runnable listener) { + this.onInvalidateListener = listener; + } + + /** + * Remove the invalidate listener. Call this before releasing the view to + * avoid callbacks after the render pipeline has been torn down. + */ + public void clearOnInvalidateListener() { + this.onInvalidateListener = null; + } + + /** + * Overrides the single non-deprecated invalidate() entry point. + * On API 28+, all partial-invalidation overloads delegate here internally, + * so one override is sufficient to catch every invalidation. + */ + @Override + public void invalidate() { + super.invalidate(); + // Local copy guards against the listener being cleared on another thread + // between our null-check and our call. + Runnable listener = onInvalidateListener; + if (listener != null) + listener.run(); + } +} \ No newline at end of file diff --git a/library/src/main/java/com/pedro/library/view/OpenGlView.kt b/library/src/main/java/com/pedro/library/view/OpenGlView.kt index e6b47bf40..814798c45 100644 --- a/library/src/main/java/com/pedro/library/view/OpenGlView.kt +++ b/library/src/main/java/com/pedro/library/view/OpenGlView.kt @@ -20,6 +20,8 @@ import android.graphics.Point import android.graphics.SurfaceTexture import android.graphics.SurfaceTexture.OnFrameAvailableListener import android.os.Build +import android.os.Handler +import android.os.HandlerThread import android.util.AttributeSet import android.view.Surface import android.view.SurfaceHolder @@ -74,6 +76,7 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf private val fpsLimiter = FpsLimiter() private val forceRenderer = ForceRenderer() private var renderErrorCallback: RenderErrorCallback? = null + private var surfaceHandlerThread: HandlerThread? = null constructor(context: Context?) : super(context) { holder.addCallback(this) @@ -273,27 +276,33 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf } override fun addMediaCodecSurface(surface: Surface) { - if (surfaceManager.isReady) { - surfaceManagerEncoder.release() - surfaceManagerEncoder.eglSetup(surface, surfaceManager) + executor?.secureSubmit { + if (surfaceManager.isReady) { + surfaceManagerEncoder.release() + surfaceManagerEncoder.eglSetup(surface, surfaceManager) + } } } override fun removeMediaCodecSurface() { - threadQueue.clear() - surfaceManagerEncoder.release() + executor?.secureSubmit { + surfaceManagerEncoder.release() + } } override fun addMediaCodecRecordSurface(surface: Surface) { - if (surfaceManager.isReady) { - surfaceManagerEncoderRecord.release() - surfaceManagerEncoderRecord.eglSetup(surface, surfaceManager) + executor?.secureSubmit { + if (surfaceManager.isReady) { + surfaceManagerEncoderRecord.release() + surfaceManagerEncoderRecord.eglSetup(surface, surfaceManager) + } } } override fun removeMediaCodecRecordSurface() { - threadQueue.clear() - surfaceManagerEncoderRecord.release() + executor?.secureSubmit { + surfaceManagerEncoderRecord.release() + } } override fun start() { @@ -301,6 +310,10 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf executor?.shutdownNow() executor = null executor = newSingleThreadExecutor(threadQueue) + surfaceHandlerThread?.quitSafely() + surfaceHandlerThread = HandlerThread("OpenGlViewHandler") + surfaceHandlerThread?.start() + val surfaceHandler = Handler(surfaceHandlerThread!!.looper) executor?.secureSubmit { surfaceManager.release() surfaceManager.eglSetup(holder.surface) @@ -309,7 +322,7 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf surfaceManager.makeCurrent() mainRender.initGl(context, encoderWidth, encoderHeight, encoderWidth, encoderHeight) running.set(true) - mainRender.getSurfaceTexture().setOnFrameAvailableListener(this) + mainRender.getSurfaceTexture().setOnFrameAvailableListener(this, surfaceHandler) forceRenderer.start { executor?.execute { try { @@ -324,6 +337,8 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf override fun stop() { running.set(false) + surfaceHandlerThread?.quitSafely() + surfaceHandlerThread = null threadQueue.clear() executor?.shutdownNow() executor = null