Move Workarounds call

Co-authored-by: Romain Vimont <rom@rom1v.com>
Signed-off-by: Romain Vimont <rom@rom1v.com>

app/src/recorder.c

@@ -11,8 +11,6 @@
 /** Downcast packet_sink to recorder */
 #define DOWNCAST(SINK) container_of(SINK, struct sc_recorder, packet_sink)
 
-#define SC_PTS_ORIGIN_NONE UINT64_C(-1)
-
 static const AVRational SCRCPY_TIME_BASE = {1, 1000000}; // timestamps in us
 
 static const AVOutputFormat *
@@ -130,8 +128,6 @@ sc_recorder_write(struct sc_recorder *recorder, AVPacket *packet) {
         return true;
     }
 
-    LOGI("==== %" PRIu64, packet->pts);
-
     sc_recorder_rescale_packet(recorder, packet);
     return av_write_frame(recorder->ctx, packet) >= 0;
 }
@@ -173,18 +169,6 @@ run_recorder(void *data) {
 
         sc_mutex_unlock(&recorder->mutex);
 
-        if (recorder->pts_origin == SC_PTS_ORIGIN_NONE
-                && rec->packet->pts != AV_NOPTS_VALUE) {
-            // First PTS received
-            recorder->pts_origin = rec->packet->pts;
-        }
-
-        if (rec->packet->pts != AV_NOPTS_VALUE) {
-            // Set PTS relatve to the origin
-            rec->packet->pts -= recorder->pts_origin;
-            rec->packet->dts = rec->packet->pts;
-        }
-
         // recorder->previous is only written from this thread, no need to lock
         struct sc_record_packet *previous = recorder->previous;
         recorder->previous = rec;
@@ -259,7 +243,6 @@ sc_recorder_open(struct sc_recorder *recorder, const AVCodec *input_codec) {
     recorder->failed = false;
     recorder->header_written = false;
     recorder->previous = NULL;
-    recorder->pts_origin = SC_PTS_ORIGIN_NONE;
 
     const char *format_name = sc_recorder_get_format_name(recorder->format);
     assert(format_name);

app/src/recorder.h

@@ -28,8 +28,6 @@ struct sc_recorder {
     struct sc_size declared_frame_size;
     bool header_written;
 
-    uint64_t pts_origin;
-
     sc_thread thread;
     sc_mutex mutex;
     sc_cond queue_cond;

server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java

@@ -1,196 +0,0 @@
-package com.genymobile.scrcpy;
-
-import com.genymobile.scrcpy.wrappers.ServiceManager;
-
-import android.annotation.SuppressLint;
-import android.annotation.TargetApi;
-import android.content.ComponentName;
-import android.content.Intent;
-import android.media.AudioFormat;
-import android.media.AudioRecord;
-import android.media.AudioTimestamp;
-import android.media.MediaCodec;
-import android.media.MediaFormat;
-import android.media.MediaRecorder;
-import android.os.Build;
-import android.os.Handler;
-import android.os.HandlerThread;
-import android.os.SystemClock;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-public final class AudioEncoder {
-
-    private static final String MIMETYPE = MediaFormat.MIMETYPE_AUDIO_OPUS;
-    private static final int SAMPLE_RATE = 48000;
-    private static final int CHANNELS = 2;
-    private static final int BIT_RATE = 128000;
-
-    private static int BUFFER_MS = 15; // milliseconds
-    private static final int BUFFER_SIZE = SAMPLE_RATE * CHANNELS * BUFFER_MS / 1000;
-
-    private AudioRecord recorder;
-    private MediaCodec mediaCodec;
-    private HandlerThread thread;
-    private final AtomicBoolean interrupted = new AtomicBoolean();
-    private final Semaphore endSemaphore = new Semaphore(0); // blocks until encoding is ended
-
-    private static AudioFormat createAudioFormat() {
-        AudioFormat.Builder builder = new AudioFormat.Builder();
-        builder.setEncoding(AudioFormat.ENCODING_PCM_16BIT);
-        builder.setSampleRate(SAMPLE_RATE);
-        builder.setChannelMask(CHANNELS == 2 ? AudioFormat.CHANNEL_IN_STEREO : AudioFormat.CHANNEL_IN_MONO);
-        return builder.build();
-    }
-
-    @TargetApi(Build.VERSION_CODES.M)
-    @SuppressLint({"WrongConstant", "MissingPermission"})
-    private static AudioRecord createAudioRecord() {
-        AudioRecord.Builder builder = new AudioRecord.Builder();
-        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
-            // On older APIs, Workarounds.fillAppInfo() must be called beforehand
-            builder.setContext(FakeContext.get());
-        }
-        builder.setAudioSource(MediaRecorder.AudioSource.REMOTE_SUBMIX);
-        builder.setAudioFormat(createAudioFormat());
-        builder.setBufferSizeInBytes(1024 * 1024);
-        return builder.build();
-    }
-
-    private static MediaFormat createFormat() {
-        MediaFormat format = new MediaFormat();
-        format.setString(MediaFormat.KEY_MIME, MIMETYPE);
-        format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
-        format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, CHANNELS);
-        format.setInteger(MediaFormat.KEY_SAMPLE_RATE, SAMPLE_RATE);
-        return format;
-    }
-
-
-
-    @TargetApi(Build.VERSION_CODES.M)
-    public void start() throws IOException {
-        mediaCodec = MediaCodec.createEncoderByType(MIMETYPE); // may throw IOException
-
-        recorder = createAudioRecord();
-
-        MediaFormat format = createFormat();
-        mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
-
-        recorder.startRecording();
-
-        thread = new HandlerThread("AudioEncoder");
-        thread.start();
-
-        class AudioEncoderCallbacks extends MediaCodec.Callback {
-
-            private final AudioTimestamp timestamp = new AudioTimestamp();
-            private long nextPts;
-            private boolean eofSignaled;
-            private boolean ended;
-
-            private void notifyEnded() {
-                assert(!ended);
-                ended = true;
-                endSemaphore.release();
-            }
-
-            @TargetApi(Build.VERSION_CODES.N)
-            @Override
-            public void onInputBufferAvailable(MediaCodec codec, int index) {
-                if (eofSignaled) {
-                    return;
-                }
-
-                ByteBuffer inputBuffer = codec.getInputBuffer(index);
-                int r = recorder.read(inputBuffer, BUFFER_SIZE);
-
-                long pts;
-
-                int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC);
-                if (ret == AudioRecord.SUCCESS) {
-                    pts = timestamp.nanoTime / 1000;
-                } else {
-                    if (nextPts == 0) {
-                        Ln.w("Could not get any audio timestamp");
-                    }
-                    // compute from previous timestamp and packet size
-                    pts = nextPts;
-                }
-
-                long durationMs = r * 1000 / CHANNELS / SAMPLE_RATE;
-                nextPts = pts + durationMs;
-
-                int flags = 0;
-                if (interrupted.get()) {
-                    flags = flags | MediaCodec.BUFFER_FLAG_END_OF_STREAM;
-                    eofSignaled = true;
-                }
-
-                codec.queueInputBuffer(index, 0, r, pts, flags);
-            }
-
-            @Override
-            public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo bufferInfo) {
-                if (ended) {
-                    return;
-                }
-
-                ByteBuffer codecBuffer = codec.getOutputBuffer(index);
-                try {
-                    boolean isConfig = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0;
-                    long pts = bufferInfo.presentationTimeUs;
-                    Ln.i("Audio packet: pts=" + pts  + " " + codecBuffer.remaining() + " bytes");
-                } finally {
-                    codec.releaseOutputBuffer(index, false);
-                }
-
-                boolean eof = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
-                if (eof) {
-                    notifyEnded();
-                }
-            }
-
-            @Override
-            public void onError(MediaCodec codec, MediaCodec.CodecException e) {
-                Ln.e("MediaCodec error", e);
-                if (!ended) {
-                    notifyEnded();
-                }
-            }
-
-            @Override
-            public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
-                // ignore
-            }
-        }
-
-        mediaCodec.setCallback(new AudioEncoderCallbacks(), new Handler(thread.getLooper()));
-        mediaCodec.start();
-    }
-
-    private void waitEnded() {
-        try {
-            endSemaphore.acquire();
-        } catch (InterruptedException e) {
-            // ignore
-        }
-    }
-
-    public void stop() {
-        Ln.i("==== STOP");
-        if (thread != null) {
-            interrupted.set(true);
-            waitEnded();
-            thread.interrupt();
-            thread = null;
-            mediaCodec.stop();
-            mediaCodec.release();
-            recorder.stop();
-            Ln.i("==== STOPPED");
-        }
-    }
-}

server/src/main/java/com/genymobile/scrcpy/Controller.java

@@ -24,8 +24,6 @@ public class Controller {
 
     private static final ScheduledExecutorService EXECUTOR = Executors.newSingleThreadScheduledExecutor();
 
-    private Thread thread;
-
     private final Device device;
     private final DesktopConnection connection;
     private final DeviceMessageSender sender;
@@ -64,7 +62,7 @@ public class Controller {
         }
     }
 
-    private void control() throws IOException {
+    public void control() throws IOException {
         // on start, power on the device
         if (powerOn && !Device.isScreenOn()) {
             device.pressReleaseKeycode(KeyEvent.KEYCODE_POWER, Device.INJECT_MODE_ASYNC);
@@ -84,27 +82,6 @@ public class Controller {
         }
     }
 
-    public void start() {
-        thread = new Thread(() -> {
-            try {
-                control();
-            } catch (IOException e) {
-                // this is expected on close
-                Ln.d("Controller stopped");
-            }
-        });
-        thread.start();
-        sender.start();
-    }
-
-    public void stop() {
-        if (thread != null) {
-            thread.interrupt();
-            thread = null;
-        }
-        sender.stop();
-    }
-
     public DeviceMessageSender getSender() {
         return sender;
     }

server/src/main/java/com/genymobile/scrcpy/DeviceMessageSender.java

@@ -6,8 +6,6 @@ public final class DeviceMessageSender {
 
     private final DesktopConnection connection;
 
-    private Thread thread;
-
     private String clipboardText;
 
     private long ack;
@@ -26,7 +24,7 @@ public final class DeviceMessageSender {
         notify();
     }
 
-    private void loop() throws IOException, InterruptedException {
+    public void loop() throws IOException, InterruptedException {
         while (!Thread.currentThread().isInterrupted()) {
             String text;
             long sequence;
@@ -51,22 +49,4 @@ public final class DeviceMessageSender {
             }
         }
     }
-    public void start() {
-        thread = new Thread(() -> {
-            try {
-                loop();
-            } catch (IOException | InterruptedException e) {
-                // this is expected on close
-                Ln.d("Device message sender stopped");
-            }
-        });
-        thread.start();
-    }
-
-    public void stop() {
-        if (thread != null) {
-            thread.interrupt();
-            thread = null;
-        }
-    }
 }

server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java

@@ -42,6 +42,7 @@ public class ScreenEncoder implements Device.RotationListener {
     private final int maxFps;
     private final boolean sendFrameMeta;
     private final boolean downsizeOnError;
+    private long ptsOrigin;
 
     private boolean firstFrameSent;
     private int consecutiveErrors;
@@ -206,7 +207,10 @@ public class ScreenEncoder implements Device.RotationListener {
         if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
             pts = PACKET_FLAG_CONFIG; // non-media data packet
         } else {
-            pts = bufferInfo.presentationTimeUs;
+            if (ptsOrigin == 0) {
+                ptsOrigin = bufferInfo.presentationTimeUs;
+            }
+            pts = bufferInfo.presentationTimeUs - ptsOrigin;
             if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0) {
                 pts |= PACKET_FLAG_KEY_FRAME;
             }

server/src/main/java/com/genymobile/scrcpy/Server.java

@@ -4,7 +4,6 @@ import android.graphics.Rect;
 import android.media.MediaCodecInfo;
 import android.os.BatteryManager;
 import android.os.Build;
-import android.provider.MediaStore;
 
 import java.io.IOException;
 import java.util.List;
@@ -67,27 +66,18 @@ public final class Server {
 
         Thread initThread = startInitThread(options);
 
+        Workarounds.prepareMainLooper();
+        if (Build.BRAND.equalsIgnoreCase("meizu")) {
+            // <https://github.com/Genymobile/scrcpy/issues/240>
+            // <https://github.com/Genymobile/scrcpy/issues/2656>
+            Workarounds.fillAppInfo();
+        }
+
         int uid = options.getUid();
         boolean tunnelForward = options.isTunnelForward();
         boolean control = options.getControl();
-        boolean audio = Build.VERSION.SDK_INT >= Build.VERSION_CODES.R; // TODO option
         boolean sendDummyByte = options.getSendDummyByte();
 
-        Workarounds.prepareMainLooper();
-
-        // <https://github.com/Genymobile/scrcpy/issues/240>
-        // <https://github.com/Genymobile/scrcpy/issues/2656>
-        boolean mustFillAppInfo = Build.BRAND.equalsIgnoreCase("meizu");
-
-        // Before Android 11, audio is not supported.
-        // Since Android 12, we can properly set a context on the AudioRecord.
-        // Only on Android 11 we must fill app info for the AudioRecord to work.
-        mustFillAppInfo |= audio && Build.VERSION.SDK_INT == Build.VERSION_CODES.R;
-
-        if (mustFillAppInfo) {
-            Workarounds.fillAppInfo();
-        }
-
         try (DesktopConnection connection = DesktopConnection.open(uid, tunnelForward, control, sendDummyByte)) {
             if (options.getSendDeviceMeta()) {
                 Size videoSize = device.getScreenInfo().getVideoSize();
@@ -96,19 +86,16 @@ public final class Server {
             ScreenEncoder screenEncoder = new ScreenEncoder(options.getSendFrameMeta(), options.getBitRate(), options.getMaxFps(), codecOptions,
                     options.getEncoderName(), options.getDownsizeOnError());
 
-            Controller controller = null;
+            Thread controllerThread = null;
+            Thread deviceMessageSenderThread = null;
             if (control) {
-                controller = new Controller(device, connection, options.getClipboardAutosync(), options.getPowerOn());
-                controller.start();
+                final Controller controller = new Controller(device, connection, options.getClipboardAutosync(), options.getPowerOn());
 
-                final Controller controllerRef = controller;
-                device.setClipboardListener(text -> controllerRef.getSender().pushClipboardText(text));
-            }
+                // asynchronous
+                controllerThread = startController(controller);
+                deviceMessageSenderThread = startDeviceMessageSender(controller.getSender());
 
-            AudioEncoder audioEncoder = null;
-            if (audio) {
-                audioEncoder = new AudioEncoder();
-                audioEncoder.start();
+                device.setClipboardListener(text -> controller.getSender().pushClipboardText(text));
             }
 
             try {
@@ -119,11 +106,11 @@ public final class Server {
                 Ln.d("Screen streaming stopped");
             } finally {
                 initThread.interrupt();
-                if (controller != null) {
-                    controller.stop();
+                if (controllerThread != null) {
+                    controllerThread.interrupt();
                 }
-                if (audioEncoder != null) {
-                    audioEncoder.stop();
+                if (deviceMessageSenderThread != null) {
+                    deviceMessageSenderThread.interrupt();
                 }
             }
         }
@@ -135,6 +122,32 @@ public final class Server {
         return thread;
     }
 
+    private static Thread startController(final Controller controller) {
+        Thread thread = new Thread(() -> {
+            try {
+                controller.control();
+            } catch (IOException e) {
+                // this is expected on close
+                Ln.d("Controller stopped");
+            }
+        });
+        thread.start();
+        return thread;
+    }
+
+    private static Thread startDeviceMessageSender(final DeviceMessageSender sender) {
+        Thread thread = new Thread(() -> {
+            try {
+                sender.loop();
+            } catch (IOException | InterruptedException e) {
+                // this is expected on close
+                Ln.d("Device message sender stopped");
+            }
+        });
+        thread.start();
+        return thread;
+    }
+
     private static Options createOptions(String... args) {
         if (args.length < 1) {
             throw new IllegalArgumentException("Missing client version");

server/src/main/java/com/genymobile/scrcpy/Workarounds.java

@@ -3,7 +3,6 @@ package com.genymobile.scrcpy;
 import android.annotation.SuppressLint;
 import android.app.Application;
 import android.app.Instrumentation;
-import android.content.ContextWrapper;
 import android.content.pm.ApplicationInfo;
 import android.os.Looper;
 
@@ -61,10 +60,7 @@ public final class Workarounds {
             mBoundApplicationField.setAccessible(true);
             mBoundApplicationField.set(activityThread, appBindData);
 
-            Application app = Application.class.newInstance();
-            Field baseField = ContextWrapper.class.getDeclaredField("mBase");
-            baseField.setAccessible(true);
-            baseField.set(app, FakeContext.get());
+            Application app = Instrumentation.newApplication(Application.class, FakeContext.get());
 
             // activityThread.mInitialApplication = app;
             Field mInitialApplicationField = activityThreadClass.getDeclaredField("mInitialApplication");