From 1fd6c1c82ac96c8ed6ca2ca59e9c5a30eaa50fc0 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 14 Nov 2024 20:50:41 +0100 Subject: [PATCH] Apply filters to camera capture --- .../scrcpy/opengl/OpenGLRunner.java | 15 +++++- .../scrcpy/video/CameraCapture.java | 50 +++++++++++++++++-- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/opengl/OpenGLRunner.java b/server/src/main/java/com/genymobile/scrcpy/opengl/OpenGLRunner.java index 92074140..906960fd 100644 --- a/server/src/main/java/com/genymobile/scrcpy/opengl/OpenGLRunner.java +++ b/server/src/main/java/com/genymobile/scrcpy/opengl/OpenGLRunner.java @@ -11,6 +11,7 @@ import android.opengl.EGLExt; import android.opengl.EGLSurface; import android.opengl.GLES11Ext; import android.opengl.GLES20; +import android.opengl.Matrix; import android.os.Handler; import android.os.HandlerThread; import android.view.Surface; @@ -28,6 +29,7 @@ public final class OpenGLRunner { private EGLSurface eglSurface; private final OpenGLFilter filter; + private final boolean ignoreTransformMatrix; private SurfaceTexture surfaceTexture; private Surface inputSurface; @@ -35,8 +37,13 @@ public final class OpenGLRunner { private boolean stopped; - public OpenGLRunner(OpenGLFilter filter) { + public OpenGLRunner(OpenGLFilter filter, boolean ignoreTransformMatrix) { this.filter = filter; + this.ignoreTransformMatrix = ignoreTransformMatrix; + } + + public OpenGLRunner(OpenGLFilter filter) { + this(filter, false); } public static synchronized void initOnce() { @@ -195,7 +202,11 @@ public final class OpenGLRunner { surfaceTexture.updateTexImage(); float[] matrix = new float[16]; - surfaceTexture.getTransformMatrix(matrix); + if (ignoreTransformMatrix) { + Matrix.setIdentityM(matrix, 0); + } else { + surfaceTexture.getTransformMatrix(matrix); + } filter.draw(textureId, matrix); diff --git a/server/src/main/java/com/genymobile/scrcpy/video/CameraCapture.java b/server/src/main/java/com/genymobile/scrcpy/video/CameraCapture.java index ee4085e9..9f2d6d71 100644 --- a/server/src/main/java/com/genymobile/scrcpy/video/CameraCapture.java +++ b/server/src/main/java/com/genymobile/scrcpy/video/CameraCapture.java @@ -3,7 +3,12 @@ package com.genymobile.scrcpy.video; import com.genymobile.scrcpy.AndroidVersions; import com.genymobile.scrcpy.Options; import com.genymobile.scrcpy.device.ConfigurationException; +import com.genymobile.scrcpy.device.Orientation; import com.genymobile.scrcpy.device.Size; +import com.genymobile.scrcpy.opengl.AffineOpenGLFilter; +import com.genymobile.scrcpy.opengl.OpenGLFilter; +import com.genymobile.scrcpy.opengl.OpenGLRunner; +import com.genymobile.scrcpy.util.AffineMatrix; import com.genymobile.scrcpy.util.HandlerExecutor; import com.genymobile.scrcpy.util.Ln; import com.genymobile.scrcpy.util.LogUtils; @@ -48,9 +53,15 @@ public class CameraCapture extends SurfaceCapture { private final CameraAspectRatio aspectRatio; private final int fps; private final boolean highSpeed; + private final Rect crop; + private Orientation captureOrientation; private String cameraId; - private Size size; + private Size captureSize; + private Size videoSize; // after OpenGL transforms + + private AffineMatrix transform; + private OpenGLRunner glRunner; private HandlerThread cameraThread; private Handler cameraHandler; @@ -67,6 +78,9 @@ public class CameraCapture extends SurfaceCapture { this.aspectRatio = options.getCameraAspectRatio(); this.fps = options.getCameraFps(); this.highSpeed = options.getCameraHighSpeed(); + this.crop = options.getCrop(); + this.captureOrientation = options.getCaptureOrientation(); + assert captureOrientation != null; } @Override @@ -92,13 +106,32 @@ public class CameraCapture extends SurfaceCapture { @Override public void prepare() throws IOException { try { - size = selectSize(cameraId, explicitSize, maxSize, aspectRatio, highSpeed); - if (size == null) { + captureSize = selectSize(cameraId, explicitSize, maxSize, aspectRatio, highSpeed); + if (captureSize == null) { throw new IOException("Could not select camera size"); } } catch (CameraAccessException e) { throw new IOException(e); } + + VideoFilter filter = new VideoFilter(captureSize); + + if (crop != null) { + filter.addCrop(crop, false); + } + + if (captureOrientation != Orientation.Orient0) { + filter.addOrientation(captureOrientation); + } + + transform = filter.getInverseTransform(); + videoSize = filter.getOutputSize().limit(maxSize).round8(); + + if (transform != null) { + // The transform matrix returned by SurfaceTexture is incorrect for camera capture (it often contains an additional unexpected 90° + // rotation), so it is disabled (see start()). A vertical flip must be applied, though. + transform = AffineMatrix.vflip().multiply(transform); + } } private static String selectCamera(String explicitCameraId, CameraFacing cameraFacing) throws CameraAccessException, ConfigurationException { @@ -214,6 +247,15 @@ public class CameraCapture extends SurfaceCapture { @Override public void start(Surface surface) throws IOException { + if (transform != null) { + assert glRunner == null; + OpenGLFilter glFilter = new AffineOpenGLFilter(transform); + // The transform matrix returned by SurfaceTexture is incorrect for camera capture (it often contains an additional unexpected 90° + // rotation), so disable it. A vertical flip is necessary though (see prepare()). + glRunner = new OpenGLRunner(glFilter, true); + surface = glRunner.start(captureSize, videoSize, surface); + } + try { CameraCaptureSession session = createCaptureSession(cameraDevice, surface); CaptureRequest request = createCaptureRequest(surface); @@ -235,7 +277,7 @@ public class CameraCapture extends SurfaceCapture { @Override public Size getSize() { - return size; + return videoSize; } @Override