From d20b68ab038af7ebd6099162f1ed6377346ef0e0 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 10 Nov 2024 13:47:02 +0100 Subject: [PATCH] Handle virtual display rotation Listen to display size changes and rotate the virtual display accordingly. Note: use `git show -b` to Show this commit ignoring whitespace changes. --- .../scrcpy/video/DisplaySizeMonitor.java | 100 +++++++++++------- .../scrcpy/video/NewDisplayCapture.java | 74 ++++++++++--- 2 files changed, 123 insertions(+), 51 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/video/DisplaySizeMonitor.java b/server/src/main/java/com/genymobile/scrcpy/video/DisplaySizeMonitor.java index 4c2c4f27..d5f92479 100644 --- a/server/src/main/java/com/genymobile/scrcpy/video/DisplaySizeMonitor.java +++ b/server/src/main/java/com/genymobile/scrcpy/video/DisplaySizeMonitor.java @@ -27,6 +27,7 @@ public class DisplaySizeMonitor { // detect it directly, so register a RotationWatcher and a DisplayFoldListener as a fallback, until we receive the first event from // DisplayListener (which proves that it works). private boolean displayListenerWorks; // only accessed from the display listener thread + private boolean fallbacksUnregistered; // a register call after an unregister must be ignored (protected by this) private IRotationWatcher rotationWatcher; private IDisplayFoldListener displayFoldListener; @@ -37,6 +38,11 @@ public class DisplaySizeMonitor { private Listener listener; public void start(int displayId, Listener listener) { + start(displayId, listener, true, true, false); + } + + public void start(int displayId, Listener listener, boolean useRotationWatcherFallback, boolean useFoldListenerFallback, + boolean delayRotationWatcherFallback) { // Once started, the listener and the displayId must never change assert listener != null; this.listener = listener; @@ -48,10 +54,6 @@ public class DisplaySizeMonitor { handlerThread.start(); Handler handler = new Handler(handlerThread.getLooper()); - if (Build.VERSION.SDK_INT == AndroidVersions.API_34_ANDROID_14) { - registerDisplayListenerFallbacks(); - } - displayListenerHandle = ServiceManager.getDisplayManager().registerDisplayListener(eventDisplayId -> { if (Ln.isEnabled(Ln.Level.VERBOSE)) { Ln.v("DisplaySizeMonitor: onDisplayChanged(" + eventDisplayId + ")"); @@ -69,6 +71,10 @@ public class DisplaySizeMonitor { checkDisplaySizeChanged(); } }, handler); + + if (Build.VERSION.SDK_INT == AndroidVersions.API_34_ANDROID_14) { + registerDisplayListenerFallbacks(useRotationWatcherFallback, useFoldListenerFallback, delayRotationWatcherFallback); + } } /** @@ -134,48 +140,68 @@ public class DisplaySizeMonitor { } } - private void registerDisplayListenerFallbacks() { - rotationWatcher = new IRotationWatcher.Stub() { - @Override - public void onRotationChanged(int rotation) { - if (Ln.isEnabled(Ln.Level.VERBOSE)) { - Ln.v("DisplaySizeMonitor: onRotationChanged(" + rotation + ")"); - } + private synchronized void registerDisplayListenerFallbacks(boolean useRotationWatcherFallback, boolean useFoldListenerFallback, + boolean delayRotationWatcherFallback) { + if (fallbacksUnregistered) { + return; + } - checkDisplaySizeChanged(); + if (useRotationWatcherFallback) { + rotationWatcher = new IRotationWatcher.Stub() { + @Override + public void onRotationChanged(int rotation) { + if (Ln.isEnabled(Ln.Level.VERBOSE)) { + Ln.v("DisplaySizeMonitor: onRotationChanged(" + rotation + ")"); + } + + checkDisplaySizeChanged(); + } + }; + + if (delayRotationWatcherFallback) { + // Hack: If the virtual display was just created, registering a rotation watcher immediately fails with an error because the + // display id does not exist yet. + new Handler(handlerThread.getLooper()).postDelayed(() -> { + ServiceManager.getWindowManager().registerRotationWatcher(rotationWatcher, displayId); + }, 50); + } else { + ServiceManager.getWindowManager().registerRotationWatcher(rotationWatcher, displayId); } - }; - ServiceManager.getWindowManager().registerRotationWatcher(rotationWatcher, displayId); + } - // Build.VERSION.SDK_INT >= AndroidVersions.API_29_ANDROID_10 (but implied by == API_34_ANDROID 14) - displayFoldListener = new IDisplayFoldListener.Stub() { + if (useFoldListenerFallback) { + // Build.VERSION.SDK_INT >= AndroidVersions.API_29_ANDROID_10 (but implied by == API_34_ANDROID 14) + displayFoldListener = new IDisplayFoldListener.Stub() { - private boolean first = true; + private boolean first = true; - @Override - public void onDisplayFoldChanged(int displayId, boolean folded) { - if (first) { - // An event is posted on registration to signal the initial state. Ignore it to avoid restarting encoding. - first = false; - return; + @Override + public void onDisplayFoldChanged(int displayId, boolean folded) { + if (first) { + // An event is posted on registration to signal the initial state. Ignore it to avoid restarting encoding. + first = false; + return; + } + + if (Ln.isEnabled(Ln.Level.VERBOSE)) { + Ln.v("DisplaySizeMonitor: onDisplayFoldChanged(" + displayId + ", " + folded + ")"); + } + + if (DisplaySizeMonitor.this.displayId != displayId) { + // Ignore events related to other display ids + return; + } + + checkDisplaySizeChanged(); } - - if (Ln.isEnabled(Ln.Level.VERBOSE)) { - Ln.v("DisplaySizeMonitor: onDisplayFoldChanged(" + displayId + ", " + folded + ")"); - } - - if (DisplaySizeMonitor.this.displayId != displayId) { - // Ignore events related to other display ids - return; - } - - checkDisplaySizeChanged(); - } - }; - ServiceManager.getWindowManager().registerDisplayFoldListener(displayFoldListener); + }; + ServiceManager.getWindowManager().registerDisplayFoldListener(displayFoldListener); + } } private synchronized void unregisterDisplayListenerFallbacks() { + fallbacksUnregistered = true; + if (rotationWatcher != null) { ServiceManager.getWindowManager().unregisterRotationWatcher(rotationWatcher); rotationWatcher = null; diff --git a/server/src/main/java/com/genymobile/scrcpy/video/NewDisplayCapture.java b/server/src/main/java/com/genymobile/scrcpy/video/NewDisplayCapture.java index cc54876a..f9463911 100644 --- a/server/src/main/java/com/genymobile/scrcpy/video/NewDisplayCapture.java +++ b/server/src/main/java/com/genymobile/scrcpy/video/NewDisplayCapture.java @@ -6,10 +6,13 @@ import com.genymobile.scrcpy.control.PositionMapper; import com.genymobile.scrcpy.device.DisplayInfo; import com.genymobile.scrcpy.device.NewDisplay; import com.genymobile.scrcpy.device.Size; +import com.genymobile.scrcpy.opengl.AffineOpenGLFilter; +import com.genymobile.scrcpy.opengl.OpenGLFilter; +import com.genymobile.scrcpy.opengl.OpenGLRunner; +import com.genymobile.scrcpy.util.AffineMatrix; import com.genymobile.scrcpy.util.Ln; import com.genymobile.scrcpy.wrappers.ServiceManager; -import android.hardware.display.DisplayManager; import android.hardware.display.VirtualDisplay; import android.os.Build; import android.view.Surface; @@ -19,8 +22,8 @@ import java.io.IOException; public class NewDisplayCapture extends SurfaceCapture { // Internal fields copied from android.hardware.display.DisplayManager - private static final int VIRTUAL_DISPLAY_FLAG_PUBLIC = DisplayManager.VIRTUAL_DISPLAY_FLAG_PUBLIC; - private static final int VIRTUAL_DISPLAY_FLAG_OWN_CONTENT_ONLY = DisplayManager.VIRTUAL_DISPLAY_FLAG_OWN_CONTENT_ONLY; + private static final int VIRTUAL_DISPLAY_FLAG_PUBLIC = android.hardware.display.DisplayManager.VIRTUAL_DISPLAY_FLAG_PUBLIC; + private static final int VIRTUAL_DISPLAY_FLAG_OWN_CONTENT_ONLY = android.hardware.display.DisplayManager.VIRTUAL_DISPLAY_FLAG_OWN_CONTENT_ONLY; private static final int VIRTUAL_DISPLAY_FLAG_SUPPORTS_TOUCH = 1 << 6; private static final int VIRTUAL_DISPLAY_FLAG_ROTATES_WITH_CONTENT = 1 << 7; private static final int VIRTUAL_DISPLAY_FLAG_DESTROY_CONTENT_ON_REMOVAL = 1 << 8; @@ -35,12 +38,18 @@ public class NewDisplayCapture extends SurfaceCapture { private final VirtualDisplayListener vdListener; private final NewDisplay newDisplay; + private final DisplaySizeMonitor displaySizeMonitor = new DisplaySizeMonitor(); + + private AffineMatrix displayTransform; + private OpenGLRunner glRunner; + private Size mainDisplaySize; private int mainDisplayDpi; private int maxSize; // only used if newDisplay.getSize() != null private VirtualDisplay virtualDisplay; - private Size size; + private Size size; // the logical size of the display (including rotation) + private Size physicalSize; // the physical size of the display (without rotation) private int dpi; public NewDisplayCapture(VirtualDisplayListener vdListener, Options options) { @@ -69,11 +78,27 @@ public class NewDisplayCapture extends SurfaceCapture { @Override public void prepare() { - if (!newDisplay.hasExplicitSize()) { - size = mainDisplaySize.limit(maxSize).round8(); - } - if (!newDisplay.hasExplicitDpi()) { - dpi = scaleDpi(mainDisplaySize, mainDisplayDpi, size); + if (virtualDisplay == null) { + if (!newDisplay.hasExplicitSize()) { + size = mainDisplaySize.limit(maxSize).round8(); + } + if (!newDisplay.hasExplicitDpi()) { + dpi = scaleDpi(mainDisplaySize, mainDisplayDpi, size); + } + + physicalSize = size; + // Set the current display size to avoid an unnecessary call to invalidate() + displaySizeMonitor.setSessionDisplaySize(size); + } else { + DisplayInfo displayInfo = ServiceManager.getDisplayManager().getDisplayInfo(virtualDisplay.getDisplay().getDisplayId()); + size = displayInfo.getSize(); + dpi = displayInfo.getDpi(); + + VideoFilter displayFilter = new VideoFilter(size); + displayFilter.addRotation(displayInfo.getRotation()); + // The display info gives the oriented size, but the virtual display video always remains in the origin orientation + displayTransform = displayFilter.getInverseTransform(); + physicalSize = displayFilter.getOutputSize(); } } @@ -100,28 +125,49 @@ public class NewDisplayCapture extends SurfaceCapture { .createNewVirtualDisplay("scrcpy", size.getWidth(), size.getHeight(), dpi, surface, flags); virtualDisplayId = virtualDisplay.getDisplay().getDisplayId(); Ln.i("New display: " + size.getWidth() + "x" + size.getHeight() + "/" + dpi + " (id=" + virtualDisplayId + ")"); + + // Disable DisplayFoldListener fallback, and delay registering the RotationWatcher as a workaround + displaySizeMonitor.start(virtualDisplayId, this::invalidate, true, false, true); } catch (Exception e) { Ln.e("Could not create display", e); throw new AssertionError("Could not create display"); } - - if (vdListener != null) { - PositionMapper positionMapper = new PositionMapper(size, null); - vdListener.onNewVirtualDisplay(virtualDisplayId, positionMapper); - } } @Override public void start(Surface surface) throws IOException { + if (displayTransform != null) { + assert glRunner == null; + OpenGLFilter glFilter = new AffineOpenGLFilter(displayTransform); + glRunner = new OpenGLRunner(glFilter); + surface = glRunner.start(physicalSize, size, surface); + } + if (virtualDisplay == null) { startNew(surface); } else { virtualDisplay.setSurface(surface); } + + if (vdListener != null) { + // The virtual display rotation must only be applied to video, it is already taken into account when injecting events! + PositionMapper positionMapper = PositionMapper.create(size, null, size); + vdListener.onNewVirtualDisplay(virtualDisplay.getDisplay().getDisplayId(), positionMapper); + } + } + + @Override + public void stop() { + if (glRunner != null) { + glRunner.stopAndRelease(); + glRunner = null; + } } @Override public void release() { + displaySizeMonitor.stopAndRelease(); + if (virtualDisplay != null) { virtualDisplay.release(); virtualDisplay = null;