Fail-fast camera mirroring on Android 11 and older

Co-authored-by: Romain Vimont <rom@rom1v.com>
Signed-off-by: Romain Vimont <rom@rom1v.com>

app/data/bash-completion/scrcpy

@@ -10,6 +10,7 @@ _scrcpy() {
         --audio-source=
         --audio-output-buffer=
         -b --video-bit-rate=
+        --camera-ar=
         --camera-id=
         --camera-facing=
         --camera-size=
@@ -153,6 +154,7 @@ _scrcpy() {
         |--audio-codec-options \
         |--audio-encoder \
         |--audio-output-buffer \
+        |--camera-ar \
         |--camera-id \
         |--camera-size \
         |--crop \

app/data/zsh-completion/_scrcpy

@@ -17,6 +17,7 @@ arguments=(
     '--audio-source=[Select the audio source]:source:(output mic)'
     '--audio-output-buffer=[Configure the size of the SDL audio output buffer (in milliseconds)]'
     {-b,--video-bit-rate=}'[Encode the video at the given bit-rate]'
+    '--camera-ar=[Select the camera size by its aspect ratio]'
     '--camera-id=[Specify the camera id to mirror]'
     '--camera-facing=[Select the device camera by its facing direction]:facing:(front back external)'
     '--camera-size=[Specify an explicit camera capture size]'

app/scrcpy.1

@@ -75,6 +75,12 @@ Encode the video at the given bit rate, expressed in bits/s. Unit suffixes are s
 
 Default is 8M (8000000).
 
+.TP
+.BI "\-\-camera\-ar " ar
+Select the camera size by its aspect ratio (+/- 10%).
+
+Possible values are "sensor" (use the camera sensor aspect ratio), "<num>:<den>" (e.g. "4:3") and "<value>" (e.g. "1.6").
+
 .TP
 .BI "\-\-camera\-id " id
 Specify the device camera id to mirror.
@@ -452,6 +458,8 @@ The available encoders can be listed by \-\-list\-encoders.
 .BI "\-\-video\-source " source
 Select the video source (display or camera).
 
+Camera mirroring requires Android 12+.
+
 Default is display.
 
 .TP

app/src/cli.c

@@ -87,6 +87,7 @@ enum {
     OPT_CAMERA_ID,
     OPT_CAMERA_SIZE,
     OPT_CAMERA_FACING,
+    OPT_CAMERA_AR,
 };
 
 struct sc_option {
@@ -203,6 +204,15 @@ static const struct sc_option options[] = {
         .longopt = "bit-rate",
         .argdesc = "value",
     },
+    {
+        .longopt_id = OPT_CAMERA_AR,
+        .longopt = "camera-ar",
+        .argdesc = "ar",
+        .text = "Select the camera size by its aspect ratio (+/- 10%).\n"
+                "Possible values are \"sensor\" (use the camera sensor aspect "
+                "ratio), \"<num>:<den>\" (e.g. \"4:3\") or \"<value>\" (e.g. "
+                "\"1.6\")."
+    },
     {
         .longopt_id = OPT_CAMERA_ID,
         .longopt = "camera-id",
@@ -733,6 +743,7 @@ static const struct sc_option options[] = {
         .longopt = "video-source",
         .argdesc = "source",
         .text = "Select the video source (display or camera).\n"
+                "Camera mirroring requires Android 12+.\n"
                 "Default is display.",
     },
     {
@@ -2129,6 +2140,9 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
                     return false;
                 }
                 break;
+            case OPT_CAMERA_AR:
+                opts->camera_ar = optarg;
+                break;
             case OPT_CAMERA_ID:
                 opts->camera_id = optarg;
                 break;
@@ -2250,9 +2264,16 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
             return false;
         }
 
-        if (!opts->camera_size) {
-            LOGE("Camera size must be specified by --camera-size=WIDTHxHEIGHT");
-            return false;
+        if (opts->camera_size) {
+            if (opts->camera_ar) {
+                LOGE("Could not specify both --camera-size and -m/--max-size");
+                return false;
+            }
+
+            if (opts->camera_ar) {
+                LOGE("Could not specify both --camera-size and --camera-ar");
+                return false;
+            }
         }
 
         if (opts->control) {
@@ -2260,6 +2281,7 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
             opts->control = false;
         }
     } else if (opts->camera_id
+            || opts->camera_ar
             || opts->camera_facing != SC_CAMERA_FACING_ANY
             || opts->camera_size) {
         LOGE("Camera options are only available with --video-source=camera");

app/src/options.c

@@ -13,6 +13,7 @@ const struct scrcpy_options scrcpy_options_default = {
     .audio_encoder = NULL,
     .camera_id = NULL,
     .camera_size = NULL,
+    .camera_ar = NULL,
     .log_level = SC_LOG_LEVEL_INFO,
     .video_codec = SC_CODEC_H264,
     .audio_codec = SC_CODEC_OPUS,

app/src/options.h

@@ -132,6 +132,7 @@ struct scrcpy_options {
     const char *audio_encoder;
     const char *camera_id;
     const char *camera_size;
+    const char *camera_ar;
     enum sc_log_level log_level;
     enum sc_codec video_codec;
     enum sc_codec audio_codec;

app/src/scrcpy.c

@@ -375,6 +375,7 @@ scrcpy(struct scrcpy_options *options) {
         .audio_encoder = options->audio_encoder,
         .camera_id = options->camera_id,
         .camera_size = options->camera_size,
+        .camera_ar = options->camera_ar,
         .force_adb_forward = options->force_adb_forward,
         .power_off_on_close = options->power_off_on_close,
         .clipboard_autosync = options->clipboard_autosync,

app/src/server.c

@@ -77,6 +77,7 @@ sc_server_params_destroy(struct sc_server_params *params) {
     free((char *) params->audio_encoder);
     free((char *) params->tcpip_dst);
     free((char *) params->camera_id);
+    free((char *) params->camera_ar);
 }
 
 static bool
@@ -105,6 +106,7 @@ sc_server_params_copy(struct sc_server_params *dst,
     COPY(audio_encoder);
     COPY(tcpip_dst);
     COPY(camera_id);
+    COPY(camera_ar);
 #undef COPY
 
     return true;
@@ -303,6 +305,9 @@ execute_server(struct sc_server *server,
         ADD_PARAM("camera_facing=%s",
             sc_server_get_camera_facing_name(params->camera_facing));
     }
+    if (params->camera_ar) {
+        ADD_PARAM("camera_ar=%s", params->camera_ar);
+    }
     if (params->show_touches) {
         ADD_PARAM("show_touches=true");
     }

app/src/server.h

@@ -36,6 +36,7 @@ struct sc_server_params {
     const char *audio_encoder;
     const char *camera_id;
     const char *camera_size;
+    const char *camera_ar;
     struct sc_port_range port_range;
     uint32_t tunnel_host;
     uint16_t tunnel_port;

server/src/main/java/com/genymobile/scrcpy/CameraAspectRatio.java

@@ -0,0 +1,37 @@
+package com.genymobile.scrcpy;
+
+public final class CameraAspectRatio {
+    private static final float SENSOR = -1;
+
+    private float ar;
+
+    private CameraAspectRatio(float ar) {
+        this.ar = ar;
+    }
+
+    public static CameraAspectRatio fromFloat(float ar) {
+        if (ar < 0) {
+            throw new IllegalArgumentException("Invalid aspect ratio: " + ar);
+        }
+        return new CameraAspectRatio(ar);
+    }
+
+    public static CameraAspectRatio fromFraction(int w, int h) {
+        if (w <= 0 || h <= 0) {
+            throw new IllegalArgumentException("Invalid aspect ratio: " + w + ":" + h);
+        }
+        return new CameraAspectRatio((float) w / h);
+    }
+
+    public static CameraAspectRatio sensorAspectRatio() {
+        return new CameraAspectRatio(SENSOR);
+    }
+
+    public boolean isSensor() {
+        return ar == SENSOR;
+    }
+
+    public float getAspectRatio() {
+        return ar;
+    }
+}

server/src/main/java/com/genymobile/scrcpy/CameraCapture.java

@@ -4,6 +4,7 @@ import com.genymobile.scrcpy.wrappers.ServiceManager;
 
 import android.annotation.SuppressLint;
 import android.annotation.TargetApi;
+import android.graphics.Rect;
 import android.hardware.camera2.CameraAccessException;
 import android.hardware.camera2.CameraCaptureSession;
 import android.hardware.camera2.CameraCharacteristics;
@@ -13,6 +14,8 @@ import android.hardware.camera2.CaptureFailure;
 import android.hardware.camera2.CaptureRequest;
 import android.hardware.camera2.params.OutputConfiguration;
 import android.hardware.camera2.params.SessionConfiguration;
+import android.hardware.camera2.params.StreamConfigurationMap;
+import android.media.MediaCodec;
 import android.os.Build;
 import android.os.Handler;
 import android.os.HandlerThread;
@@ -20,34 +23,25 @@ import android.view.Surface;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Stream;
 
 public class CameraCapture extends SurfaceCapture {
 
-    public static class CameraSelection {
-        private String explicitCameraId;
-        private CameraFacing cameraFacing;
-
-        public CameraSelection(String explicitCameraId, CameraFacing cameraFacing) {
-            this.explicitCameraId = explicitCameraId;
-            this.cameraFacing = cameraFacing;
-        }
-
-        boolean hasId() {
-            return explicitCameraId != null;
-        }
-
-        boolean hasProperties() {
-            return cameraFacing != null;
-        }
-    }
-
-    private final CameraSelection cameraSelection;
+    private final String explicitCameraId;
+    private final CameraFacing cameraFacing;
     private final Size explicitSize;
+    private int maxSize;
+    private final CameraAspectRatio aspectRatio;
+
+    private String cameraId;
+    private Size size;
 
     private HandlerThread cameraThread;
     private Handler cameraHandler;
@@ -56,9 +50,12 @@ public class CameraCapture extends SurfaceCapture {
 
     private final AtomicBoolean disconnected = new AtomicBoolean();
 
-    public CameraCapture(CameraSelection cameraSelection, Size explicitSize) {
-        this.cameraSelection = cameraSelection;
+    public CameraCapture(String explicitCameraId, CameraFacing cameraFacing, Size explicitSize, int maxSize, CameraAspectRatio aspectRatio) {
+        this.explicitCameraId = explicitCameraId;
+        this.cameraFacing = cameraFacing;
         this.explicitSize = explicitSize;
+        this.maxSize = maxSize;
+        this.aspectRatio = aspectRatio;
     }
 
     @Override
@@ -69,11 +66,16 @@ public class CameraCapture extends SurfaceCapture {
         cameraExecutor = new HandlerExecutor(cameraHandler);
 
         try {
-            String cameraId = selectCamera(cameraSelection);
+            cameraId = selectCamera(explicitCameraId, cameraFacing);
             if (cameraId == null) {
                 throw new IOException("No matching camera found");
             }
 
+            size = selectSize(cameraId, explicitSize, maxSize, aspectRatio);
+            if (size == null) {
+                throw new IOException("Could not select camera size");
+            }
+
             Ln.i("Using camera '" + cameraId + "'");
             cameraDevice = openCamera(cameraId);
         } catch (CameraAccessException | InterruptedException e) {
@@ -81,15 +83,15 @@ public class CameraCapture extends SurfaceCapture {
         }
     }
 
-    private String selectCamera(CameraSelection cameraSelection) throws CameraAccessException {
-        if (cameraSelection.hasId()) {
-            return cameraSelection.explicitCameraId;
+    private static String selectCamera(String explicitCameraId, CameraFacing cameraFacing) throws CameraAccessException {
+        if (explicitCameraId != null) {
+            return explicitCameraId;
         }
 
         CameraManager cameraManager = ServiceManager.getCameraManager();
 
         String[] cameraIds = cameraManager.getCameraIdList();
-        if (!cameraSelection.hasProperties()) {
+        if (cameraFacing == null) {
             // Use the first one
             return cameraIds.length > 0 ? cameraIds[0] : null;
         }
@@ -97,21 +99,89 @@ public class CameraCapture extends SurfaceCapture {
         for (String cameraId : cameraIds) {
             CameraCharacteristics characteristics = cameraManager.getCameraCharacteristics(cameraId);
 
-            if (cameraSelection.cameraFacing != null) {
-                int facing = characteristics.get(CameraCharacteristics.LENS_FACING);
-                if (cameraSelection.cameraFacing.value() != facing) {
-                    // Does not match
-                    continue;
-                }
+            int facing = characteristics.get(CameraCharacteristics.LENS_FACING);
+            if (cameraFacing.value() == facing) {
+                return cameraId;
             }
-
-            return cameraId;
         }
 
         // Not found
         return null;
     }
 
+    @TargetApi(Build.VERSION_CODES.N)
+    private static Size selectSize(String cameraId, Size explicitSize, int maxSize, CameraAspectRatio aspectRatio) throws CameraAccessException {
+        if (explicitSize != null) {
+            return explicitSize;
+        }
+
+        CameraManager cameraManager = ServiceManager.getCameraManager();
+        CameraCharacteristics characteristics = cameraManager.getCameraCharacteristics(cameraId);
+
+        StreamConfigurationMap configs = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
+        android.util.Size[] sizes = configs.getOutputSizes(MediaCodec.class);
+        Stream<android.util.Size> stream = Arrays.stream(sizes);
+        if (maxSize > 0) {
+            stream = stream.filter(it -> it.getWidth() <= maxSize && it.getHeight() <= maxSize);
+        }
+
+        Float targetAspectRatio = resolveAspectRatio(aspectRatio, characteristics);
+        if (targetAspectRatio != null) {
+            stream = stream.filter(it -> {
+                float ar = ((float) it.getWidth() / it.getHeight());
+                float arRatio = ar / targetAspectRatio;
+                // Accept if the aspect ratio is the target aspect ratio + or - 10%
+                return arRatio >= 0.9f && arRatio <= 1.1f;
+            });
+        }
+
+        Optional<android.util.Size> selected = stream.max((s1, s2) -> {
+            // Greater width is better
+            int cmp = Integer.compare(s1.getWidth(), s2.getWidth());
+            if (cmp != 0) {
+                return cmp;
+            }
+
+            if (targetAspectRatio != null) {
+                // Closer to the target aspect ratio is better
+                float ar1 = ((float) s1.getWidth() / s1.getHeight());
+                float arRatio1 = ar1 / targetAspectRatio;
+                float distance1 = Math.abs(1 - arRatio1);
+
+                float ar2 = ((float) s2.getWidth() / s2.getHeight());
+                float arRatio2 = ar2 / targetAspectRatio;
+                float distance2 = Math.abs(1 - arRatio2);
+
+                // Reverse the order because lower distance is better
+                return Float.compare(distance2, distance1);
+            }
+
+            // Greater height is better
+            return Integer.compare(s1.getHeight(), s2.getHeight());
+        });
+
+        if (selected.isPresent()) {
+            android.util.Size size = selected.get();
+            return new Size(size.getWidth(), size.getHeight());
+        }
+
+        // Not found
+        return null;
+    }
+
+    private static Float resolveAspectRatio(CameraAspectRatio ratio, CameraCharacteristics characteristics) {
+        if (ratio == null) {
+            return null;
+        }
+
+        if (ratio.isSensor()) {
+            Rect activeSize = characteristics.get(CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE);
+            return (float) activeSize.width() / activeSize.height();
+        }
+
+        return ratio.getAspectRatio();
+    }
+
     @Override
     public void start(Surface surface) throws IOException {
         try {
@@ -137,24 +207,33 @@ public class CameraCapture extends SurfaceCapture {
 
     @Override
     public Size getSize() {
-        return explicitSize;
+        return size;
     }
 
     @Override
-    public boolean setMaxSize(int size) {
-        return false;
+    public boolean setMaxSize(int maxSize) {
+        if (explicitSize != null) {
+            return false;
+        }
+
+        this.maxSize = maxSize;
+        try {
+            size = selectSize(cameraId, null, maxSize, aspectRatio);
+            return true;
+        } catch (CameraAccessException e) {
+            Ln.w("Could not select camera size", e);
+            return false;
+        }
     }
 
     @SuppressLint("MissingPermission")
     @TargetApi(Build.VERSION_CODES.S)
     private CameraDevice openCamera(String id) throws CameraAccessException, InterruptedException {
-        Ln.v("Open Camera: " + id);
-
         CompletableFuture<CameraDevice> future = new CompletableFuture<>();
         ServiceManager.getCameraManager().openCamera(id, new CameraDevice.StateCallback() {
             @Override
             public void onOpened(CameraDevice camera) {
-                Ln.v("Open Camera Success");
+                Ln.d("Camera opened successfully");
                 future.complete(camera);
             }
 
@@ -197,17 +276,13 @@ public class CameraCapture extends SurfaceCapture {
 
     @TargetApi(Build.VERSION_CODES.S)
     private CameraCaptureSession createCaptureSession(CameraDevice camera, Surface surface) throws CameraAccessException, InterruptedException {
-        Ln.d("Create Capture Session");
-
         CompletableFuture<CameraCaptureSession> future = new CompletableFuture<>();
-        // replace by createCaptureSession(SessionConfiguration)
         OutputConfiguration outputConfig = new OutputConfiguration(surface);
         List<OutputConfiguration> outputs = Arrays.asList(outputConfig);
         SessionConfiguration sessionConfig = new SessionConfiguration(SessionConfiguration.SESSION_REGULAR, outputs, cameraExecutor,
                 new CameraCaptureSession.StateCallback() {
                     @Override
                     public void onConfigured(CameraCaptureSession session) {
-                        Ln.d("Create Capture Session Success");
                         future.complete(session);
                     }
 
@@ -252,4 +327,4 @@ public class CameraCapture extends SurfaceCapture {
     public boolean isClosed() {
         return disconnected.get();
     }
-}
+}

server/src/main/java/com/genymobile/scrcpy/Options.java

@@ -27,6 +27,7 @@ public class Options {
     private String cameraId;
     private Size cameraSize;
     private CameraFacing cameraFacing;
+    private CameraAspectRatio cameraAspectRatio;
     private boolean showTouches;
     private boolean stayAwake;
     private List<CodecOption> videoCodecOptions;
@@ -131,6 +132,10 @@ public class Options {
         return cameraFacing;
     }
 
+    public CameraAspectRatio getCameraAspectRatio() {
+        return cameraAspectRatio;
+    }
+
     public boolean getShowTouches() {
         return showTouches;
     }
@@ -361,7 +366,9 @@ public class Options {
                     }
                     break;
                 case "camera_size":
-                    options.cameraSize = parseSize(value);
+                    if (!value.isEmpty()) {
+                        options.cameraSize = parseSize(value);
+                    }
                     break;
                 case "camera_facing":
                     if (!value.isEmpty()) {
@@ -372,6 +379,11 @@ public class Options {
                         options.cameraFacing = facing;
                     }
                     break;
+                case "camera_ar":
+                    if (!value.isEmpty()) {
+                        options.cameraAspectRatio = parseCameraAspectRatio(value);
+                    }
+                    break;
                 case "send_device_meta":
                     options.sendDeviceMeta = Boolean.parseBoolean(value);
                     break;
@@ -416,9 +428,6 @@ public class Options {
     }
 
     private static Size parseSize(String size) {
-        if (size.isEmpty()) {
-            return null;
-        }
         // input format: "<width>x<height>"
         String[] tokens = size.split("x");
         if (tokens.length != 2) {
@@ -428,4 +437,20 @@ public class Options {
         int height = Integer.parseInt(tokens[1]);
         return new Size(width, height);
     }
+
+    private static CameraAspectRatio parseCameraAspectRatio(String ar) {
+        if ("sensor".equals(ar)) {
+            return CameraAspectRatio.sensorAspectRatio();
+        }
+
+        String[] tokens = ar.split(":");
+        if (tokens.length == 2) {
+            int w = Integer.parseInt(tokens[0]);
+            int h = Integer.parseInt(tokens[1]);
+            return CameraAspectRatio.fromFraction(w, h);
+        }
+
+        float floatAr = Float.parseFloat(tokens[0]);
+        return CameraAspectRatio.fromFloat(floatAr);
+    }
 }

server/src/main/java/com/genymobile/scrcpy/Server.java

@@ -2,7 +2,6 @@ package com.genymobile.scrcpy;
 
 import android.os.BatteryManager;
 import android.os.Build;
-import android.os.Looper;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -89,6 +88,12 @@ public final class Server {
 
     private static void scrcpy(Options options) throws IOException, ConfigurationException {
         Ln.i("Device: [" + Build.MANUFACTURER + "] " + Build.BRAND + " " + Build.MODEL + " (Android " + Build.VERSION.RELEASE + ")");
+
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S && options.getVideoSource() == VideoSource.CAMERA) {
+            Ln.e("Camera mirroring is not supported before Android 12");
+            throw new ConfigurationException("Camera mirroring is not supported");
+        }
+
         final Device device = new Device(options);
 
         Thread initThread = startInitThread(options);
@@ -138,9 +143,8 @@ public final class Server {
                 if (options.getVideoSource() == VideoSource.DISPLAY) {
                     surfaceCapture = new ScreenCapture(device);
                 } else {
-                    CameraCapture.CameraSelection cameraSelection = new CameraCapture.CameraSelection(options.getCameraId(),
-                            options.getCameraFacing());
-                    surfaceCapture = new CameraCapture(cameraSelection, options.getCameraSize());
+                    surfaceCapture = new CameraCapture(options.getCameraId(), options.getCameraFacing(), options.getCameraSize(),
+                            options.getMaxSize(), options.getCameraAspectRatio());
                 }
                 SurfaceEncoder surfaceEncoder = new SurfaceEncoder(surfaceCapture, videoStreamer, options.getVideoBitRate(), options.getMaxFps(),
                         options.getVideoCodecOptions(), options.getVideoEncoder(), options.getDownsizeOnError());

server/src/main/java/com/genymobile/scrcpy/VideoSource.java

@@ -1,7 +1,5 @@
 package com.genymobile.scrcpy;
 
-import android.media.MediaRecorder;
-
 public enum VideoSource {
     DISPLAY("display"),
     CAMERA("camera");

server/src/main/java/com/genymobile/scrcpy/Workarounds.java

@@ -32,7 +32,7 @@ public final class Workarounds {
         Workarounds.prepareMainLooper();
 
         boolean mustFillAppInfo = false;
-        boolean mustFillBaseContext = true;
+        boolean mustFillBaseContext = false;
         boolean mustFillAppContext = false;
 
         if (Build.BRAND.equalsIgnoreCase("meizu")) {
@@ -66,6 +66,7 @@ public final class Workarounds {
 
         if (camera) {
             mustFillAppInfo = true;
+            mustFillBaseContext = true;
         }
 
         if (mustFillAppInfo) {

server/src/main/java/com/genymobile/scrcpy/wrappers/ServiceManager.java

@@ -1,7 +1,6 @@
 package com.genymobile.scrcpy.wrappers;
 
 import com.genymobile.scrcpy.FakeContext;
-import com.genymobile.scrcpy.Workarounds;
 
 import android.annotation.SuppressLint;
 import android.content.Context;