Add support for high frame rate camera capture

Add --camera-high-speed to enable high frame rate camera capture. If
the option is enabled, then --camera-fps is mandatory.

PR #4213 <https://github.com/Genymobile/scrcpy/pull/4213>

Co-authored-by: Romain Vimont <rom@rom1v.com>
Signed-off-by: Andrew Gunnerson <accounts+github@chiller3.com>
Signed-off-by: Romain Vimont <rom@rom1v.com>

server/src/main/java/com/genymobile/scrcpy/CameraCapture.java

@@ -8,6 +8,7 @@ import android.graphics.Rect;
 import android.hardware.camera2.CameraAccessException;
 import android.hardware.camera2.CameraCaptureSession;
 import android.hardware.camera2.CameraCharacteristics;
+import android.hardware.camera2.CameraConstrainedHighSpeedCaptureSession;
 import android.hardware.camera2.CameraDevice;
 import android.hardware.camera2.CameraManager;
 import android.hardware.camera2.CaptureFailure;
@@ -40,6 +41,7 @@ public class CameraCapture extends SurfaceCapture {
     private int maxSize;
     private final CameraAspectRatio aspectRatio;
     private final int fps;
+    private final boolean highSpeed;
 
     private String cameraId;
     private Size size;
@@ -51,13 +53,15 @@ public class CameraCapture extends SurfaceCapture {
 
     private final AtomicBoolean disconnected = new AtomicBoolean();
 
-    public CameraCapture(String explicitCameraId, CameraFacing cameraFacing, Size explicitSize, int maxSize, CameraAspectRatio aspectRatio, int fps) {
+    public CameraCapture(String explicitCameraId, CameraFacing cameraFacing, Size explicitSize, int maxSize, CameraAspectRatio aspectRatio, int fps,
+            boolean highSpeed) {
         this.explicitCameraId = explicitCameraId;
         this.cameraFacing = cameraFacing;
         this.explicitSize = explicitSize;
         this.maxSize = maxSize;
         this.aspectRatio = aspectRatio;
         this.fps = fps;
+        this.highSpeed = highSpeed;
     }
 
     @Override
@@ -73,7 +77,7 @@ public class CameraCapture extends SurfaceCapture {
                 throw new IOException("No matching camera found");
             }
 
-            size = selectSize(cameraId, explicitSize, maxSize, aspectRatio);
+            size = selectSize(cameraId, explicitSize, maxSize, aspectRatio, highSpeed);
             if (size == null) {
                 throw new IOException("Could not select camera size");
             }
@@ -112,7 +116,8 @@ public class CameraCapture extends SurfaceCapture {
     }
 
     @TargetApi(Build.VERSION_CODES.N)
-    private static Size selectSize(String cameraId, Size explicitSize, int maxSize, CameraAspectRatio aspectRatio) throws CameraAccessException {
+    private static Size selectSize(String cameraId, Size explicitSize, int maxSize, CameraAspectRatio aspectRatio, boolean highSpeed)
+            throws CameraAccessException {
         if (explicitSize != null) {
             return explicitSize;
         }
@@ -121,7 +126,7 @@ public class CameraCapture extends SurfaceCapture {
         CameraCharacteristics characteristics = cameraManager.getCameraCharacteristics(cameraId);
 
         StreamConfigurationMap configs = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
-        android.util.Size[] sizes = configs.getOutputSizes(MediaCodec.class);
+        android.util.Size[] sizes = highSpeed ? configs.getHighSpeedVideoSizes() : configs.getOutputSizes(MediaCodec.class);
         Stream<android.util.Size> stream = Arrays.stream(sizes);
         if (maxSize > 0) {
             stream = stream.filter(it -> it.getWidth() <= maxSize && it.getHeight() <= maxSize);
@@ -221,7 +226,7 @@ public class CameraCapture extends SurfaceCapture {
 
         this.maxSize = maxSize;
         try {
-            size = selectSize(cameraId, null, maxSize, aspectRatio);
+            size = selectSize(cameraId, null, maxSize, aspectRatio, highSpeed);
             return size != null;
         } catch (CameraAccessException e) {
             Ln.w("Could not select camera size", e);
@@ -282,7 +287,9 @@ public class CameraCapture extends SurfaceCapture {
         CompletableFuture<CameraCaptureSession> future = new CompletableFuture<>();
         OutputConfiguration outputConfig = new OutputConfiguration(surface);
         List<OutputConfiguration> outputs = Arrays.asList(outputConfig);
-        SessionConfiguration sessionConfig = new SessionConfiguration(SessionConfiguration.SESSION_REGULAR, outputs, cameraExecutor,
+
+        int sessionType = highSpeed ? SessionConfiguration.SESSION_HIGH_SPEED : SessionConfiguration.SESSION_REGULAR;
+        SessionConfiguration sessionConfig = new SessionConfiguration(sessionType, outputs, cameraExecutor,
                 new CameraCaptureSession.StateCallback() {
                     @Override
                     public void onConfigured(CameraCaptureSession session) {
@@ -317,7 +324,7 @@ public class CameraCapture extends SurfaceCapture {
 
     @TargetApi(Build.VERSION_CODES.S)
     private void setRepeatingRequest(CameraCaptureSession session, CaptureRequest request) throws CameraAccessException, InterruptedException {
-        session.setRepeatingRequest(request, new CameraCaptureSession.CaptureCallback() {
+        CameraCaptureSession.CaptureCallback callback = new CameraCaptureSession.CaptureCallback() {
             @Override
             public void onCaptureStarted(CameraCaptureSession session, CaptureRequest request, long timestamp, long frameNumber) {
                 // Called for each frame captured, do nothing
@@ -327,7 +334,15 @@ public class CameraCapture extends SurfaceCapture {
             public void onCaptureFailed(CameraCaptureSession session, CaptureRequest request, CaptureFailure failure) {
                 Ln.w("Camera capture failed: frame " + failure.getFrameNumber());
             }
-        }, cameraHandler);
+        };
+
+        if (highSpeed) {
+            CameraConstrainedHighSpeedCaptureSession highSpeedSession = (CameraConstrainedHighSpeedCaptureSession) session;
+            List<CaptureRequest> requests = highSpeedSession.createHighSpeedRequestList(request);
+            highSpeedSession.setRepeatingBurst(requests, callback, cameraHandler);
+        } else {
+            session.setRepeatingRequest(request, callback, cameraHandler);
+        }
     }
 
     @Override

server/src/main/java/com/genymobile/scrcpy/LogUtils.java

@@ -12,6 +12,7 @@ import android.media.MediaCodec;
 import android.util.Range;
 
 import java.util.List;
+import java.util.SortedSet;
 import java.util.TreeSet;
 
 public final class LogUtils {
@@ -103,18 +104,27 @@ public final class LogUtils {
 
                     // Capture frame rates for low-FPS mode are the same for every resolution
                     Range<Integer>[] lowFpsRanges = characteristics.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES);
-                    TreeSet<Integer> uniqueLowFps = new TreeSet<>();
-                    for (Range<Integer> range : lowFpsRanges) {
-                        uniqueLowFps.add(range.getUpper());
-                    }
+                    SortedSet<Integer> uniqueLowFps = getUniqueSet(lowFpsRanges);
                     builder.append("fps=").append(uniqueLowFps).append(')');
 
                     if (includeSizes) {
                         StreamConfigurationMap configs = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
+
                         android.util.Size[] sizes = configs.getOutputSizes(MediaCodec.class);
                         for (android.util.Size size : sizes) {
                             builder.append("\n        - ").append(size.getWidth()).append('x').append(size.getHeight());
                         }
+
+                        android.util.Size[] highSpeedSizes = configs.getHighSpeedVideoSizes();
+                        if (highSpeedSizes.length > 0) {
+                            builder.append("\n      High speed capture (--camera-high-speed):");
+                            for (android.util.Size size : highSpeedSizes) {
+                                Range<Integer>[] highFpsRanges = configs.getHighSpeedVideoFpsRanges();
+                                SortedSet<Integer> uniqueHighFps = getUniqueSet(highFpsRanges);
+                                builder.append("\n        - ").append(size.getWidth()).append("x").append(size.getHeight());
+                                builder.append(" (fps=").append(uniqueHighFps).append(')');
+                            }
+                        }
                     }
                 }
             }
@@ -123,4 +133,12 @@ public final class LogUtils {
         }
         return builder.toString();
     }
+
+    private static SortedSet<Integer> getUniqueSet(Range<Integer>[] ranges) {
+        SortedSet<Integer> set = new TreeSet<>();
+        for (Range<Integer> range : ranges) {
+            set.add(range.getUpper());
+        }
+        return set;
+    }
 }

server/src/main/java/com/genymobile/scrcpy/Options.java

@@ -29,6 +29,7 @@ public class Options {
     private CameraFacing cameraFacing;
     private CameraAspectRatio cameraAspectRatio;
     private int cameraFps;
+    private boolean cameraHighSpeed;
     private boolean showTouches;
     private boolean stayAwake;
     private List<CodecOption> videoCodecOptions;
@@ -141,6 +142,10 @@ public class Options {
         return cameraFps;
     }
 
+    public boolean getCameraHighSpeed() {
+        return cameraHighSpeed;
+    }
+
     public boolean getShowTouches() {
         return showTouches;
     }
@@ -392,6 +397,9 @@ public class Options {
                 case "camera_fps":
                     options.cameraFps = Integer.parseInt(value);
                     break;
+                case "camera_high_speed":
+                    options.cameraHighSpeed = Boolean.parseBoolean(value);
+                    break;
                 case "send_device_meta":
                     options.sendDeviceMeta = Boolean.parseBoolean(value);
                     break;

server/src/main/java/com/genymobile/scrcpy/Server.java

@@ -144,7 +144,7 @@ public final class Server {
                     surfaceCapture = new ScreenCapture(device);
                 } else {
                     surfaceCapture = new CameraCapture(options.getCameraId(), options.getCameraFacing(), options.getCameraSize(),
-                            options.getMaxSize(), options.getCameraAspectRatio(), options.getCameraFps());
+                            options.getMaxSize(), options.getCameraAspectRatio(), options.getCameraFps(), options.getCameraHighSpeed());
                 }
                 SurfaceEncoder surfaceEncoder = new SurfaceEncoder(surfaceCapture, videoStreamer, options.getVideoBitRate(), options.getMaxFps(),
                         options.getVideoCodecOptions(), options.getVideoEncoder(), options.getDownsizeOnError());