Fix touch events HiDPI-scaling [TODO: rebase on dev]

Touch events were HiDPI-scaled twice:
 - once because the position (provided as floats between 0 and 1) were
   converted in pixels using the drawable size (not the window size)
 - once due to screen_convert_to_frame_coords()

One possible fix could be to compute the position in pixels from the
window size instead, but this would unnecessarily round the event
position to the nearest window coordinates (instead of drawable
coordinates).

Instead, expose two separate functions to convert to frame coordinates
from either window or drawable coordinates.

Fixes #1536 <https://github.com/Genymobile/scrcpy/issues/1536>
Refs #15 <https://github.com/Genymobile/scrcpy/issues/15>
Refs e40532a376

README.md

@@ -301,7 +301,7 @@ ssh -CN -L5037:localhost:5037 -L27183:localhost:27183 your_remote_computer
 From another terminal:
 
 ```bash
-scrcpy --force-adb-forwrad
+scrcpy --force-adb-forward
 ```
 
 
@@ -417,7 +417,7 @@ The secondary display may only be controlled if the device runs at least Android
 
 #### Stay awake
 
-To prevent the device to sleep after some delay:
+To prevent the device to sleep after some delay when the device is plugged in:
 
 ```bash
 scrcpy --stay-awake
@@ -550,11 +550,11 @@ scrcpy --push-target /sdcard/foo/bar/
 
 ### Audio forwarding
 
-Audio is not forwarded by _scrcpy_. Use [USBaudio] (Linux-only).
+Audio is not forwarded by _scrcpy_. Use [sndcpy].
 
 Also see [issue #14].
 
-[USBaudio]: https://github.com/rom1v/usbaudio
+[sndcpy]: https://github.com/rom1v/sndcpy
 [issue #14]: https://github.com/Genymobile/scrcpy/issues/14
 
 

app/scrcpy.1

@@ -167,7 +167,7 @@ Default is "info" for release builds, "debug" for debug builds.
 
 .TP
 .B \-w, \-\-stay-awake
-Keep the device on while scrcpy is running.
+Keep the device on while scrcpy is running, when the device is plugged in.
 
 .TP
 .B \-\-window\-borderless

app/src/cli.c

@@ -159,7 +159,8 @@ scrcpy_print_usage(const char *arg0) {
 #endif
         "\n"
         "    -w, --stay-awake\n"
-        "        Keep the device on while scrcpy is running.\n"
+        "        Keep the device on while scrcpy is running, when the device\n"
+        "        is plugged in.\n"
         "\n"
         "    --window-borderless\n"
         "        Disable window decorations (display borderless window).\n"

app/src/control_msg.c

@@ -66,9 +66,6 @@ control_msg_serialize(const struct control_msg *msg, unsigned char *buf) {
             buffer_write32be(&buf[17],
                              (uint32_t) msg->inject_scroll_event.vscroll);
             return 21;
-        case CONTROL_MSG_TYPE_GET_CLIPBOARD:
-            buf[1] = msg->get_clipboard.copy;
-            return 2;
         case CONTROL_MSG_TYPE_SET_CLIPBOARD: {
             buf[1] = !!msg->set_clipboard.paste;
             size_t len = write_string(msg->set_clipboard.text,
@@ -82,6 +79,7 @@ control_msg_serialize(const struct control_msg *msg, unsigned char *buf) {
         case CONTROL_MSG_TYPE_BACK_OR_SCREEN_ON:
         case CONTROL_MSG_TYPE_EXPAND_NOTIFICATION_PANEL:
         case CONTROL_MSG_TYPE_COLLAPSE_NOTIFICATION_PANEL:
+        case CONTROL_MSG_TYPE_GET_CLIPBOARD:
         case CONTROL_MSG_TYPE_ROTATE_DEVICE:
             // no additional data
             return 1;

app/src/control_msg.h

@@ -60,9 +60,6 @@ struct control_msg {
             int32_t hscroll;
             int32_t vscroll;
         } inject_scroll_event;
-        struct {
-            bool copy;
-        } get_clipboard;
         struct {
             char *text; // owned, to be freed by SDL_free()
             bool paste;

app/src/event_converter.c

@@ -92,8 +92,6 @@ convert_keycode(SDL_Keycode from, enum android_keycode *to, uint16_t mod,
         MAP(SDLK_LEFT,         AKEYCODE_DPAD_LEFT);
         MAP(SDLK_DOWN,         AKEYCODE_DPAD_DOWN);
         MAP(SDLK_UP,           AKEYCODE_DPAD_UP);
-        MAP(SDLK_LSHIFT,       AKEYCODE_SHIFT_LEFT);
-        MAP(SDLK_RSHIFT,       AKEYCODE_SHIFT_RIGHT);
     }
 
     if (!(mod & (KMOD_NUM | KMOD_SHIFT))) {

app/src/input_manager.c

@@ -102,10 +102,9 @@ collapse_notification_panel(struct controller *controller) {
 }
 
 static void
-request_device_clipboard(struct controller *controller, bool copy) {
+request_device_clipboard(struct controller *controller) {
     struct control_msg msg;
     msg.type = CONTROL_MSG_TYPE_GET_CLIPBOARD;
-    msg.get_clipboard.copy = copy;
 
     if (!controller_push_msg(controller, &msg)) {
         LOGW("Could not request device clipboard");
@@ -353,8 +352,7 @@ input_manager_process_key(struct input_manager *im,
                 return;
             case SDLK_c:
                 if (control && cmd && !shift && !repeat && down) {
-                    // press COPY and get the clipboard content
-                    request_device_clipboard(controller, true);
+                    request_device_clipboard(controller);
                 }
                 return;
             case SDLK_v:
@@ -429,7 +427,7 @@ convert_mouse_motion(const SDL_MouseMotionEvent *from, struct screen *screen,
     to->inject_touch_event.pointer_id = POINTER_ID_MOUSE;
     to->inject_touch_event.position.screen_size = screen->frame_size;
     to->inject_touch_event.position.point =
-        screen_convert_to_frame_coords(screen, from->x, from->y);
+        screen_convert_window_to_frame_coords(screen, from->x, from->y);
     to->inject_touch_event.pressure = 1.f;
     to->inject_touch_event.buttons = convert_mouse_buttons(from->state);
 
@@ -467,15 +465,15 @@ convert_touch(const SDL_TouchFingerEvent *from, struct screen *screen,
     to->inject_touch_event.pointer_id = from->fingerId;
     to->inject_touch_event.position.screen_size = screen->frame_size;
 
-    int ww;
-    int wh;
-    SDL_GL_GetDrawableSize(screen->window, &ww, &wh);
+    int dw;
+    int dh;
+    SDL_GL_GetDrawableSize(screen->window, &dw, &dh);
 
     // SDL touch event coordinates are normalized in the range [0; 1]
-    int32_t x = from->x * ww;
-    int32_t y = from->y * wh;
+    int32_t x = from->x * dw;
+    int32_t y = from->y * dh;
     to->inject_touch_event.position.point =
-        screen_convert_to_frame_coords(screen, x, y);
+        screen_convert_drawable_to_frame_coords(screen, x, y);
 
     to->inject_touch_event.pressure = from->pressure;
     to->inject_touch_event.buttons = 0;
@@ -505,7 +503,7 @@ convert_mouse_button(const SDL_MouseButtonEvent *from, struct screen *screen,
     to->inject_touch_event.pointer_id = POINTER_ID_MOUSE;
     to->inject_touch_event.position.screen_size = screen->frame_size;
     to->inject_touch_event.position.point =
-        screen_convert_to_frame_coords(screen, from->x, from->y);
+        screen_convert_window_to_frame_coords(screen, from->x, from->y);
     to->inject_touch_event.pressure = 1.f;
     to->inject_touch_event.buttons =
         convert_mouse_buttons(SDL_BUTTON(from->button));
@@ -570,7 +568,8 @@ convert_mouse_wheel(const SDL_MouseWheelEvent *from, struct screen *screen,
 
     struct position position = {
         .screen_size = screen->frame_size,
-        .point = screen_convert_to_frame_coords(screen, mouse_x, mouse_y),
+        .point = screen_convert_window_to_frame_coords(screen,
+                                                       mouse_x, mouse_y),
     };
 
     to->type = CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT;

app/src/screen.c

@@ -579,14 +579,14 @@ screen_handle_window_event(struct screen *screen,
 }
 
 struct point
-screen_convert_to_frame_coords(struct screen *screen, int32_t x, int32_t y) {
+screen_convert_drawable_to_frame_coords(struct screen *screen,
+                                        int32_t x, int32_t y) {
     unsigned rotation = screen->rotation;
     assert(rotation < 4);
 
     int32_t w = screen->content_size.width;
     int32_t h = screen->content_size.height;
 
-    screen_hidpi_scale_coords(screen, &x, &y);
 
     x = (int64_t) (x - screen->rect.x) * w / screen->rect.w;
     y = (int64_t) (y - screen->rect.y) * h / screen->rect.h;
@@ -615,6 +615,13 @@ screen_convert_to_frame_coords(struct screen *screen, int32_t x, int32_t y) {
     return result;
 }
 
+struct point
+screen_convert_window_to_frame_coords(struct screen *screen,
+                                      int32_t x, int32_t y) {
+    screen_hidpi_scale_coords(screen, &x, &y);
+    return screen_convert_drawable_to_frame_coords(screen, x, y);
+}
+
 void
 screen_hidpi_scale_coords(struct screen *screen, int32_t *x, int32_t *y) {
     // take the HiDPI scaling (dw/ww and dh/wh) into account

app/src/screen.h

@@ -126,7 +126,14 @@ screen_handle_window_event(struct screen *screen, const SDL_WindowEvent *event);
 // convert point from window coordinates to frame coordinates
 // x and y are expressed in pixels
 struct point
-screen_convert_to_frame_coords(struct screen *screen, int32_t x, int32_t y);
+screen_convert_window_to_frame_coords(struct screen *screen,
+                                      int32_t x, int32_t y);
+
+// convert point from drawable coordinates to frame coordinates
+// x and y are expressed in pixels
+struct point
+screen_convert_drawable_to_frame_coords(struct screen *screen,
+                                        int32_t x, int32_t y);
 
 // Convert coordinates from window to drawable.
 // Events are expressed in window coordinates, but content is expressed in

app/tests/test_control_msg_serialize.c

@@ -185,18 +185,14 @@ static void test_serialize_collapse_notification_panel(void) {
 static void test_serialize_get_clipboard(void) {
     struct control_msg msg = {
         .type = CONTROL_MSG_TYPE_GET_CLIPBOARD,
-        .get_clipboard = {
-            .copy = true,
-        },
     };
 
     unsigned char buf[CONTROL_MSG_SERIALIZED_MAX_SIZE];
     int size = control_msg_serialize(&msg, buf);
-    assert(size == 2);
+    assert(size == 1);
 
     const unsigned char expected[] = {
         CONTROL_MSG_TYPE_GET_CLIPBOARD,
-        1, // copy
     };
     assert(!memcmp(buf, expected, sizeof(expected)));
 }

server/build_without_gradle.sh

@@ -42,6 +42,8 @@ echo "Generating java from aidl..."
 cd "$SERVER_DIR/src/main/aidl"
 "$ANDROID_HOME/build-tools/$BUILD_TOOLS/aidl" -o"$CLASSES_DIR" \
     android/view/IRotationWatcher.aidl
+"$ANDROID_HOME/build-tools/$BUILD_TOOLS/aidl" -o"$CLASSES_DIR" \
+    android/content/IOnPrimaryClipChangedListener.aidl
 
 echo "Compiling java sources..."
 cd ../java
@@ -55,6 +57,7 @@ cd "$CLASSES_DIR"
 "$ANDROID_HOME/build-tools/$BUILD_TOOLS/dx" --dex \
     --output "$BUILD_DIR/classes.dex" \
     android/view/*.class \
+    android/content/*.class \
     com/genymobile/scrcpy/*.class \
     com/genymobile/scrcpy/wrappers/*.class
 

server/src/main/java/com/genymobile/scrcpy/ControlMessage.java

@@ -18,7 +18,6 @@ public final class ControlMessage {
     public static final int TYPE_ROTATE_DEVICE = 10;
 
     public static final int FLAGS_PASTE = 1;
-    public static final int FLAGS_COPY = 2;
 
     private int type;
     private String text;
@@ -82,15 +81,6 @@ public final class ControlMessage {
         return msg;
     }
 
-    public static ControlMessage createGetClipboard(boolean copy) {
-        ControlMessage msg = new ControlMessage();
-        msg.type = TYPE_GET_CLIPBOARD;
-        if (copy) {
-            msg.flags = FLAGS_COPY;
-        }
-        return msg;
-    }
-
     /**
      * @param mode one of the {@code Device.SCREEN_POWER_MODE_*} constants
      */

server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java

@@ -12,7 +12,6 @@ public class ControlMessageReader {
     static final int INJECT_TOUCH_EVENT_PAYLOAD_LENGTH = 27;
     static final int INJECT_SCROLL_EVENT_PAYLOAD_LENGTH = 20;
     static final int SET_SCREEN_POWER_MODE_PAYLOAD_LENGTH = 1;
-    static final int GET_CLIPBOARD_PAYLOAD_LENGTH = 1;
     static final int SET_CLIPBOARD_FIXED_PAYLOAD_LENGTH = 1;
 
     public static final int CLIPBOARD_TEXT_MAX_LENGTH = 4092; // 4096 - 1 (type) - 1 (parse flag) - 2 (length)
@@ -68,9 +67,6 @@ public class ControlMessageReader {
             case ControlMessage.TYPE_INJECT_SCROLL_EVENT:
                 msg = parseInjectScrollEvent();
                 break;
-            case ControlMessage.TYPE_GET_CLIPBOARD:
-                msg = parseGetClipboard();
-                break;
             case ControlMessage.TYPE_SET_CLIPBOARD:
                 msg = parseSetClipboard();
                 break;
@@ -80,6 +76,7 @@ public class ControlMessageReader {
             case ControlMessage.TYPE_BACK_OR_SCREEN_ON:
             case ControlMessage.TYPE_EXPAND_NOTIFICATION_PANEL:
             case ControlMessage.TYPE_COLLAPSE_NOTIFICATION_PANEL:
+            case ControlMessage.TYPE_GET_CLIPBOARD:
             case ControlMessage.TYPE_ROTATE_DEVICE:
                 msg = ControlMessage.createEmpty(type);
                 break;
@@ -151,14 +148,6 @@ public class ControlMessageReader {
         return ControlMessage.createInjectScrollEvent(position, hScroll, vScroll);
     }
 
-    private ControlMessage parseGetClipboard() {
-        if (buffer.remaining() < GET_CLIPBOARD_PAYLOAD_LENGTH) {
-            return null;
-        }
-        boolean copy = buffer.get() != 0;
-        return ControlMessage.createGetClipboard(copy);
-    }
-
     private ControlMessage parseSetClipboard() {
         if (buffer.remaining() < SET_CLIPBOARD_FIXED_PAYLOAD_LENGTH) {
             return null;

server/src/main/java/com/genymobile/scrcpy/Controller.java

@@ -104,8 +104,10 @@ public class Controller {
                 device.collapsePanels();
                 break;
             case ControlMessage.TYPE_GET_CLIPBOARD:
-                boolean copy = (msg.getFlags() & ControlMessage.FLAGS_COPY) != 0;
-                getClipboard(copy);
+                String clipboardText = device.getClipboardText();
+                if (clipboardText != null) {
+                    sender.pushClipboardText(clipboardText);
+                }
                 break;
             case ControlMessage.TYPE_SET_CLIPBOARD:
                 boolean paste = (msg.getFlags() & ControlMessage.FLAGS_PASTE) != 0;
@@ -227,19 +229,6 @@ public class Controller {
         return device.injectKeycode(keycode);
     }
 
-    private void getClipboard(boolean copy) {
-        // On Android >= 7, also press the COPY key if requested
-        if (copy && Build.VERSION.SDK_INT >= Build.VERSION_CODES.N && device.supportsInputEvents()) {
-            // Must wait until the COPY has been executed
-            device.injectCopyKeycode();
-        }
-
-        String clipboardText = device.getClipboardText();
-        if (clipboardText != null) {
-            sender.pushClipboardText(clipboardText);
-        }
-    }
-
     private boolean setClipboard(String text, boolean paste) {
         boolean ok = device.setClipboardText(text);
         if (ok) {
@@ -248,7 +237,7 @@ public class Controller {
 
         // On Android >= 7, also press the PASTE key if requested
         if (paste && Build.VERSION.SDK_INT >= Build.VERSION_CODES.N && device.supportsInputEvents()) {
-            device.injectPasteKeycode();
+            device.injectKeycode(KeyEvent.KEYCODE_PASTE);
         }
 
         return ok;

server/src/main/java/com/genymobile/scrcpy/Device.java

@@ -167,35 +167,15 @@ public final class Device {
         return injectEvent(event, InputManager.INJECT_INPUT_EVENT_MODE_ASYNC);
     }
 
-    public boolean injectKeyEvent(int action, int keyCode, int repeat, int metaState, int mode) {
+    public boolean injectKeyEvent(int action, int keyCode, int repeat, int metaState) {
         long now = SystemClock.uptimeMillis();
         KeyEvent event = new KeyEvent(now, now, action, keyCode, repeat, metaState, KeyCharacterMap.VIRTUAL_KEYBOARD, 0, 0,
                 InputDevice.SOURCE_KEYBOARD);
-        return injectEvent(event, mode);
-    }
-
-    public boolean injectKeyEvent(int action, int keyCode, int repeat, int metaState) {
-        return injectKeyEvent(action, keyCode, repeat, metaState, InputManager.INJECT_INPUT_EVENT_MODE_ASYNC);
-    }
-
-    public boolean injectKeycode(int keyCode, int mode) {
-        return injectKeyEvent(KeyEvent.ACTION_DOWN, keyCode, 0, 0) && injectKeyEvent(KeyEvent.ACTION_UP, keyCode, 0, 0, mode);
+        return injectEvent(event);
     }
 
     public boolean injectKeycode(int keyCode) {
-        return injectKeycode(keyCode, InputManager.INJECT_INPUT_EVENT_MODE_ASYNC);
-    }
-
-    public boolean injectCopyKeycode() {
-        isSettingClipboard.set(true);
-        // Must wait until the COPY has been executed
-        boolean ret = injectKeycode(KeyEvent.KEYCODE_COPY, InputManager.INJECT_INPUT_EVENT_MODE_WAIT_FOR_FINISH);
-        isSettingClipboard.set(false);
-        return ret;
-    }
-
-    public boolean injectPasteKeycode() {
-        return injectKeycode(KeyEvent.KEYCODE_PASTE);
+        return injectKeyEvent(KeyEvent.ACTION_DOWN, keyCode, 0, 0) && injectKeyEvent(KeyEvent.ACTION_UP, keyCode, 0, 0);
     }
 
     public boolean isScreenOn() {

server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java

@@ -200,7 +200,6 @@ public class ControlMessageReaderTest {
         ByteArrayOutputStream bos = new ByteArrayOutputStream();
         DataOutputStream dos = new DataOutputStream(bos);
         dos.writeByte(ControlMessage.TYPE_GET_CLIPBOARD);
-        dos.writeByte(1); // copy
 
         byte[] packet = bos.toByteArray();
 
@@ -208,7 +207,6 @@ public class ControlMessageReaderTest {
         ControlMessage event = reader.next();
 
         Assert.assertEquals(ControlMessage.TYPE_GET_CLIPBOARD, event.getType());
-        Assert.assertEquals(1, (event.getFlags() & ControlMessage.FLAGS_COPY));
     }
 
     @Test