Extract SurfaceCapture from ScreenEncoder

Extract an interface SurfaceCapture from ScreenEncoder, representing a
video source which can be rendered to a Surface for encoding.

Split ScreenEncoder into:
 - ScreenCapture, implementing SurfaceCapture to capture the device
   screen,
 - SurfaceEncoder, to encode any SurfaceCapture.

This separation prepares the introduction of another SurfaceCapture
implementation to capture the camera instead of the device screen.

Co-authored-by: Romain Vimont <rom@rom1v.com>

.gitignore

@@ -7,4 +7,3 @@ build/
 .gradle/
 /x/
 local.properties
-/scrcpy-server

app/src/cli.c

@@ -2022,12 +2022,6 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
         opts->audio_playback = false;
     }
 
-    if (!opts->video_playback && !otg) {
-        // If video playback is disabled and OTG are disabled, then there is
-        // no way to control the device.
-        opts->control = false;
-    }
-
     if (opts->video && !opts->video_playback && !opts->record_filename
             && !v4l2) {
         LOGI("No video playback, no recording, no V4L2 sink: video disabled");

app/src/display.c

@@ -53,7 +53,7 @@ sc_display_init(struct sc_display *display, SDL_Window *window, bool mipmaps) {
                 display->mipmaps = true;
             } else {
                 LOGW("Trilinear filtering disabled "
-                     "(OpenGL 3.0+ or ES 2.0+ required");
+                     "(OpenGL 3.0+ or ES 2.0+ required)");
             }
         } else {
             LOGI("Trilinear filtering disabled");

app/src/scrcpy.c

@@ -252,7 +252,9 @@ sc_audio_demuxer_on_ended(struct sc_demuxer *demuxer,
 
     // Contrary to the video demuxer, keep mirroring if only the audio fails
     // (unless --require-audio is set).
-    if (status == SC_DEMUXER_STATUS_ERROR
+    if (status == SC_DEMUXER_STATUS_EOS) {
+        PUSH_EVENT(SC_EVENT_DEVICE_DISCONNECTED);
+    } else if (status == SC_DEMUXER_STATUS_ERROR
             || (status == SC_DEMUXER_STATUS_DISABLED
                 && options->require_audio)) {
         PUSH_EVENT(SC_EVENT_DEMUXER_ERROR);
@@ -448,9 +450,7 @@ scrcpy(struct scrcpy_options *options) {
 
     struct sc_file_pusher *fp = NULL;
 
-    // control implies video playback
-    assert(!options->control || options->video_playback);
-    if (options->control) {
+    if (options->video_playback && options->control) {
         if (!sc_file_pusher_init(&s->file_pusher, serial,
                                  options->push_target)) {
             goto end;

server/src/main/java/com/genymobile/scrcpy/ScreenCapture.java

@@ -0,0 +1,83 @@
+package com.genymobile.scrcpy;
+
+import com.genymobile.scrcpy.wrappers.SurfaceControl;
+
+import android.graphics.Rect;
+import android.os.Build;
+import android.os.IBinder;
+import android.view.Surface;
+
+public class ScreenCapture extends SurfaceCapture implements Device.RotationListener, Device.FoldListener {
+
+    private final Device device;
+    private IBinder display;
+
+    public ScreenCapture(Device device) {
+        this.device = device;
+    }
+
+    @Override
+    public void init() {
+        display = createDisplay();
+        device.setRotationListener(this);
+        device.setFoldListener(this);
+    }
+
+    @Override
+    public void release() {
+        device.setRotationListener(null);
+        device.setFoldListener(null);
+        SurfaceControl.destroyDisplay(display);
+    }
+
+    @Override
+    public Size getSize() {
+        return device.getScreenInfo().getVideoSize();
+    }
+
+    @Override
+    public void setMaxSize(int size) {
+        device.setMaxSize(size);
+    }
+
+    @Override
+    public void setSurface(Surface surface) {
+        ScreenInfo screenInfo = device.getScreenInfo();
+        Rect contentRect = screenInfo.getContentRect();
+
+        // does not include the locked video orientation
+        Rect unlockedVideoRect = screenInfo.getUnlockedVideoSize().toRect();
+        int videoRotation = screenInfo.getVideoRotation();
+        int layerStack = device.getLayerStack();
+        setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack);
+    }
+
+    @Override
+    public void onFoldChanged(int displayId, boolean folded) {
+        requestReset();
+    }
+
+    @Override
+    public void onRotationChanged(int rotation) {
+        requestReset();
+    }
+
+    private static IBinder createDisplay() {
+        // Since Android 12 (preview), secure displays could not be created with shell permissions anymore.
+        // On Android 12 preview, SDK_INT is still R (not S), but CODENAME is "S".
+        boolean secure = Build.VERSION.SDK_INT < Build.VERSION_CODES.R || (Build.VERSION.SDK_INT == Build.VERSION_CODES.R && !"S".equals(
+                Build.VERSION.CODENAME));
+        return SurfaceControl.createDisplay("scrcpy", secure);
+    }
+
+    private static void setDisplaySurface(IBinder display, Surface surface, int orientation, Rect deviceRect, Rect displayRect, int layerStack) {
+        SurfaceControl.openTransaction();
+        try {
+            SurfaceControl.setDisplaySurface(display, surface);
+            SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect);
+            SurfaceControl.setDisplayLayerStack(display, layerStack);
+        } finally {
+            SurfaceControl.closeTransaction();
+        }
+    }
+}

server/src/main/java/com/genymobile/scrcpy/Server.java

@@ -132,7 +132,8 @@ public final class Server {
             if (video) {
                 Streamer videoStreamer = new Streamer(connection.getVideoFd(), options.getVideoCodec(), options.getSendCodecMeta(),
                         options.getSendFrameMeta());
-                ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getVideoBitRate(), options.getMaxFps(),
+                ScreenCapture screenCapture = new ScreenCapture(device);
+                SurfaceEncoder screenEncoder = new SurfaceEncoder(screenCapture, videoStreamer, options.getVideoBitRate(), options.getMaxFps(),
                         options.getVideoCodecOptions(), options.getVideoEncoder(), options.getDownsizeOnError());
                 asyncProcessors.add(screenEncoder);
             }

server/src/main/java/com/genymobile/scrcpy/SurfaceCapture.java

@@ -0,0 +1,61 @@
+package com.genymobile.scrcpy;
+
+import android.view.Surface;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * A video source which can be rendered on a Surface for encoding.
+ */
+public abstract class SurfaceCapture {
+
+    private final AtomicBoolean resetCapture = new AtomicBoolean();
+
+    /**
+     * Request the encoding session to be restarted, for example if the capture implementation detects that the video source size has changed (on
+     * device rotation for example).
+     */
+    protected void requestReset() {
+        resetCapture.set(true);
+    }
+
+    /**
+     * Consume the reset request (intended to be called by the encoder).
+     *
+     * @return {@code true} if a reset request was pending, {@code false} otherwise.
+     */
+    public boolean consumeReset() {
+        return resetCapture.getAndSet(false);
+    }
+
+    /**
+     * Called once before the capture starts.
+     */
+    public abstract void init();
+
+    /**
+     * Called after the capture ends (if and only if {@link #init()} has been called).
+     */
+    public abstract void release();
+
+    /**
+     * Return the video size
+     *
+     * @return the video size
+     */
+    public abstract Size getSize();
+
+    /**
+     * Set the maximum capture size (set by the encoder if it does not support the current size).
+     *
+     * @param size Maximum size
+     */
+    public abstract void setMaxSize(int size);
+
+    /**
+     * Set the target surface.
+     *
+     * @param surface the surface which will be encoded
+     */
+    public abstract void setSurface(Surface surface);
+}

server/src/main/java/com/genymobile/scrcpy/SurfaceEncoder.java

@@ -17,7 +17,7 @@ import java.nio.ByteBuffer;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-public class ScreenEncoder implements Device.RotationListener, Device.FoldListener, AsyncProcessor {
+public class SurfaceEncoder implements AsyncProcessor {
 
     private static final int DEFAULT_I_FRAME_INTERVAL = 10; // seconds
     private static final int REPEAT_FRAME_DELAY_US = 100_000; // repeat after 100ms
@@ -27,9 +27,7 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
     private static final int[] MAX_SIZE_FALLBACK = {2560, 1920, 1600, 1280, 1024, 800};
     private static final int MAX_CONSECUTIVE_ERRORS = 3;
 
-    private final AtomicBoolean resetCapture = new AtomicBoolean();
-
-    private final Device device;
+    private final SurfaceCapture capture;
     private final Streamer streamer;
     private final String encoderName;
     private final List<CodecOption> codecOptions;
@@ -43,9 +41,9 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
     private Thread thread;
     private final AtomicBoolean stopped = new AtomicBoolean();
 
-    public ScreenEncoder(Device device, Streamer streamer, int videoBitRate, int maxFps, List<CodecOption> codecOptions, String encoderName,
+    public SurfaceEncoder(SurfaceCapture capture, Streamer streamer, int videoBitRate, int maxFps, List<CodecOption> codecOptions, String encoderName,
             boolean downsizeOnError) {
-        this.device = device;
+        this.capture = capture;
         this.streamer = streamer;
         this.videoBitRate = videoBitRate;
         this.maxFps = maxFps;
@@ -54,51 +52,29 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
         this.downsizeOnError = downsizeOnError;
     }
 
-    @Override
-    public void onFoldChanged(int displayId, boolean folded) {
-        resetCapture.set(true);
-    }
-
-    @Override
-    public void onRotationChanged(int rotation) {
-        resetCapture.set(true);
-    }
-
-    private boolean consumeResetCapture() {
-        return resetCapture.getAndSet(false);
-    }
-
     private void streamScreen() throws IOException, ConfigurationException {
         Codec codec = streamer.getCodec();
         MediaCodec mediaCodec = createMediaCodec(codec, encoderName);
         MediaFormat format = createFormat(codec.getMimeType(), videoBitRate, maxFps, codecOptions);
-        IBinder display = createDisplay();
-        device.setRotationListener(this);
-        device.setFoldListener(this);
 
-        streamer.writeVideoHeader(device.getScreenInfo().getVideoSize());
+        capture.init();
 
-        boolean alive;
         try {
-            do {
-                ScreenInfo screenInfo = device.getScreenInfo();
-                Rect contentRect = screenInfo.getContentRect();
+            streamer.writeVideoHeader(capture.getSize());
 
-                // include the locked video orientation
-                Rect videoRect = screenInfo.getVideoSize().toRect();
-                format.setInteger(MediaFormat.KEY_WIDTH, videoRect.width());
-                format.setInteger(MediaFormat.KEY_HEIGHT, videoRect.height());
+            boolean alive;
+
+            do {
+                Size size = capture.getSize();
+                format.setInteger(MediaFormat.KEY_WIDTH, size.getWidth());
+                format.setInteger(MediaFormat.KEY_HEIGHT, size.getHeight());
 
                 Surface surface = null;
                 try {
                     mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
                     surface = mediaCodec.createInputSurface();
 
-                    // does not include the locked video orientation
-                    Rect unlockedVideoRect = screenInfo.getUnlockedVideoSize().toRect();
-                    int videoRotation = screenInfo.getVideoRotation();
-                    int layerStack = device.getLayerStack();
-                    setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack);
+                    capture.setSurface(surface);
 
                     mediaCodec.start();
 
@@ -107,7 +83,7 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
                     mediaCodec.stop();
                 } catch (IllegalStateException | IllegalArgumentException e) {
                     Ln.e("Encoding error: " + e.getClass().getName() + ": " + e.getMessage());
-                    if (!prepareRetry(device, screenInfo)) {
+                    if (!prepareRetry(size)) {
                         throw e;
                     }
                     Ln.i("Retrying...");
@@ -121,13 +97,11 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
             } while (alive);
         } finally {
             mediaCodec.release();
-            device.setRotationListener(null);
-            device.setFoldListener(null);
-            SurfaceControl.destroyDisplay(display);
+            capture.release();
         }
     }
 
-    private boolean prepareRetry(Device device, ScreenInfo screenInfo) {
+    private boolean prepareRetry(Size currentSize) {
         if (firstFrameSent) {
             ++consecutiveErrors;
             if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
@@ -147,7 +121,7 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
 
         // Downsizing on error is only enabled if an encoding failure occurs before the first frame (downsizing later could be surprising)
 
-        int newMaxSize = chooseMaxSizeFallback(screenInfo.getVideoSize());
+        int newMaxSize = chooseMaxSizeFallback(currentSize);
         if (newMaxSize == 0) {
             // Must definitively fail
             return false;
@@ -155,7 +129,7 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
 
         // Retry with a smaller device size
         Ln.i("Retrying with -m" + newMaxSize + "...");
-        device.setMaxSize(newMaxSize);
+        capture.setMaxSize(newMaxSize);
         return true;
     }
 
@@ -176,14 +150,14 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
         boolean alive = true;
         MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
 
-        while (!consumeResetCapture() && !eof) {
+        while (!capture.consumeReset() && !eof) {
             if (stopped.get()) {
                 alive = false;
                 break;
             }
             int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1);
             try {
-                if (consumeResetCapture()) {
+                if (capture.consumeReset()) {
                     // must restart encoding with new size
                     break;
                 }
@@ -267,8 +241,8 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
     private static IBinder createDisplay() {
         // Since Android 12 (preview), secure displays could not be created with shell permissions anymore.
         // On Android 12 preview, SDK_INT is still R (not S), but CODENAME is "S".
-        boolean secure = Build.VERSION.SDK_INT < Build.VERSION_CODES.R || (Build.VERSION.SDK_INT == Build.VERSION_CODES.R && !"S"
-                .equals(Build.VERSION.CODENAME));
+        boolean secure = Build.VERSION.SDK_INT < Build.VERSION_CODES.R || (Build.VERSION.SDK_INT == Build.VERSION_CODES.R && !"S".equals(
+                Build.VERSION.CODENAME));
         return SurfaceControl.createDisplay("scrcpy", secure);
     }