Apply filters to camera capture

server/src/main/java/com/genymobile/scrcpy/opengl/OpenGLRunner.java

@@ -11,6 +11,7 @@ import android.opengl.EGLExt;
 import android.opengl.EGLSurface;
 import android.opengl.GLES11Ext;
 import android.opengl.GLES20;
+import android.opengl.Matrix;
 import android.os.Handler;
 import android.os.HandlerThread;
 import android.view.Surface;
@@ -28,6 +29,7 @@ public final class OpenGLRunner {
     private EGLSurface eglSurface;
 
     private final OpenGLFilter filter;
+    private final boolean ignoreTransformMatrix;
 
     private SurfaceTexture surfaceTexture;
     private Surface inputSurface;
@@ -35,8 +37,13 @@ public final class OpenGLRunner {
 
     private boolean stopped;
 
-    public OpenGLRunner(OpenGLFilter filter) {
+    public OpenGLRunner(OpenGLFilter filter, boolean ignoreTransformMatrix) {
         this.filter = filter;
+        this.ignoreTransformMatrix = ignoreTransformMatrix;
+    }
+
+    public OpenGLRunner(OpenGLFilter filter) {
+        this(filter, false);
     }
 
     public static synchronized void initOnce() {
@@ -195,7 +202,11 @@ public final class OpenGLRunner {
 
         surfaceTexture.updateTexImage();
         float[] matrix = new float[16];
-        surfaceTexture.getTransformMatrix(matrix);
+        if (ignoreTransformMatrix) {
+            Matrix.setIdentityM(matrix, 0);
+        } else {
+            surfaceTexture.getTransformMatrix(matrix);
+        }
 
         filter.draw(textureId, matrix);
 

server/src/main/java/com/genymobile/scrcpy/video/CameraCapture.java

@@ -3,7 +3,12 @@ package com.genymobile.scrcpy.video;
 import com.genymobile.scrcpy.AndroidVersions;
 import com.genymobile.scrcpy.Options;
 import com.genymobile.scrcpy.device.ConfigurationException;
+import com.genymobile.scrcpy.device.Orientation;
 import com.genymobile.scrcpy.device.Size;
+import com.genymobile.scrcpy.opengl.AffineOpenGLFilter;
+import com.genymobile.scrcpy.opengl.OpenGLFilter;
+import com.genymobile.scrcpy.opengl.OpenGLRunner;
+import com.genymobile.scrcpy.util.AffineMatrix;
 import com.genymobile.scrcpy.util.HandlerExecutor;
 import com.genymobile.scrcpy.util.Ln;
 import com.genymobile.scrcpy.util.LogUtils;
@@ -48,9 +53,15 @@ public class CameraCapture extends SurfaceCapture {
     private final CameraAspectRatio aspectRatio;
     private final int fps;
     private final boolean highSpeed;
+    private final Rect crop;
+    private Orientation captureOrientation;
 
     private String cameraId;
-    private Size size;
+    private Size captureSize;
+    private Size videoSize; // after OpenGL transforms
+
+    private AffineMatrix transform;
+    private OpenGLRunner glRunner;
 
     private HandlerThread cameraThread;
     private Handler cameraHandler;
@@ -67,6 +78,9 @@ public class CameraCapture extends SurfaceCapture {
         this.aspectRatio = options.getCameraAspectRatio();
         this.fps = options.getCameraFps();
         this.highSpeed = options.getCameraHighSpeed();
+        this.crop = options.getCrop();
+        this.captureOrientation = options.getCaptureOrientation();
+        assert captureOrientation != null;
     }
 
     @Override
@@ -92,13 +106,32 @@ public class CameraCapture extends SurfaceCapture {
     @Override
     public void prepare() throws IOException {
         try {
-            size = selectSize(cameraId, explicitSize, maxSize, aspectRatio, highSpeed);
-            if (size == null) {
+            captureSize = selectSize(cameraId, explicitSize, maxSize, aspectRatio, highSpeed);
+            if (captureSize == null) {
                 throw new IOException("Could not select camera size");
             }
         } catch (CameraAccessException e) {
             throw new IOException(e);
         }
+
+        VideoFilter filter = new VideoFilter(captureSize);
+
+        if (crop != null) {
+            filter.addCrop(crop, false);
+        }
+
+        if (captureOrientation != Orientation.Orient0) {
+            filter.addOrientation(captureOrientation);
+        }
+
+        transform = filter.getInverseTransform();
+        videoSize = filter.getOutputSize().limit(maxSize).round8();
+
+        if (transform != null) {
+            // The transform matrix returned by SurfaceTexture is incorrect for camera capture (it often contains an additional unexpected 90°
+            // rotation), so it is disabled (see start()). A vertical flip must be applied, though.
+            transform = AffineMatrix.vflip().multiply(transform);
+        }
     }
 
     private static String selectCamera(String explicitCameraId, CameraFacing cameraFacing) throws CameraAccessException, ConfigurationException {
@@ -214,6 +247,15 @@ public class CameraCapture extends SurfaceCapture {
 
     @Override
     public void start(Surface surface) throws IOException {
+        if (transform != null) {
+            assert glRunner == null;
+            OpenGLFilter glFilter = new AffineOpenGLFilter(transform);
+            // The transform matrix returned by SurfaceTexture is incorrect for camera capture (it often contains an additional unexpected 90°
+            // rotation), so disable it. A vertical flip is necessary though (see prepare()).
+            glRunner = new OpenGLRunner(glFilter, true);
+            surface = glRunner.start(captureSize, videoSize, surface);
+        }
+
         try {
             CameraCaptureSession session = createCaptureSession(cameraDevice, surface);
             CaptureRequest request = createCaptureRequest(surface);
@@ -235,7 +277,7 @@ public class CameraCapture extends SurfaceCapture {
 
     @Override
     public Size getSize() {
-        return size;
+        return videoSize;
     }
 
     @Override