hid_mouse_initialization

This will allow the mouse processor to handle relative motion easily.

LICENSE

@@ -188,7 +188,7 @@
       identification within third-party archives.
 
    Copyright (C) 2018 Genymobile
-   Copyright (C) 2018-2021 Romain Vimont
+   Copyright (C) 2018-2022 Romain Vimont
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.id.md

@@ -672,7 +672,7 @@ Baca [halaman pengembang].
 ## Lisensi
 
     Copyright (C) 2018 Genymobile
-    Copyright (C) 2018-2021 Romain Vimont
+    Copyright (C) 2018-2022 Romain Vimont
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

README.it.md

@@ -790,7 +790,7 @@ Leggi la [pagina per sviluppatori].
 ## Licenza (in inglese)
 
     Copyright (C) 2018 Genymobile
-    Copyright (C) 2018-2021 Romain Vimont
+    Copyright (C) 2018-2022 Romain Vimont
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

README.jp.md

@@ -776,7 +776,7 @@ _⁴Android 7以上のみ._
 ## ライセンス
 
     Copyright (C) 2018 Genymobile
-    Copyright (C) 2018-2021 Romain Vimont
+    Copyright (C) 2018-2022 Romain Vimont
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

README.ko.md

@@ -475,7 +475,7 @@ _²화면이 꺼진 상태에서 우클릭 시 다시 켜지며, 그 외의 상
 ## 라이선스
 
     Copyright (C) 2018 Genymobile
-    Copyright (C) 2018-2021 Romain Vimont
+    Copyright (C) 2018-2022 Romain Vimont
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

README.md

@@ -1017,7 +1017,7 @@ Read the [developers page].
 ## Licence
 
     Copyright (C) 2018 Genymobile
-    Copyright (C) 2018-2021 Romain Vimont
+    Copyright (C) 2018-2022 Romain Vimont
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

README.pt-br.md

@@ -857,7 +857,7 @@ Leia a [página dos desenvolvedores][developers page].
 ## Licença
 
     Copyright (C) 2018 Genymobile
-    Copyright (C) 2018-2021 Romain Vimont
+    Copyright (C) 2018-2022 Romain Vimont
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

README.sp.md

@@ -720,7 +720,7 @@ Lea la [hoja de desarrolladores (en inglés)](DEVELOP.md).
 ## Licencia
 
     Copyright (C) 2018 Genymobile
-    Copyright (C) 2018-2021 Romain Vimont
+    Copyright (C) 2018-2022 Romain Vimont
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

README.tr.md

@@ -801,7 +801,7 @@ Bakınız [FAQ](FAQ.md).
 ## Lisans
 
     Copyright (C) 2018 Genymobile
-    Copyright (C) 2018-2021 Romain Vimont
+    Copyright (C) 2018-2022 Romain Vimont
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

README.zh-Hans.md

@@ -842,7 +842,7 @@ ADB=/path/to/adb scrcpy
 ## 许可协议
 
     Copyright (C) 2018 Genymobile
-    Copyright (C) 2018-2021 Romain Vimont
+    Copyright (C) 2018-2022 Romain Vimont
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

README.zh-Hant.md

@@ -679,7 +679,7 @@ _³只支援 Android 7+。_
 ## Licence
 
     Copyright (C) 2018 Genymobile
-    Copyright (C) 2018-2021 Romain Vimont
+    Copyright (C) 2018-2022 Romain Vimont
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

app/scrcpy.1

@@ -446,7 +446,7 @@ Copyright \(co 2018 Genymobile
 Genymobile
 .UE
 
-Copyright \(co 2018\-2021
+Copyright \(co 2018\-2022
 .MT rom@rom1v.com
 Romain Vimont
 .ME

app/src/cli.c

@@ -214,6 +214,17 @@ static const struct sc_option options[] = {
         .text = "Limit the frame rate of screen capture (officially supported "
                 "since Android 10, but may work on earlier versions).",
     },
+    {
+        .shortopt = 'M',
+        .longopt = "hid-mouse",
+        .text = "Simulate a physical mouse by using HID over AOAv2.\n"
+                "In this mode, the computer mouse is captured to control the "
+                "device directly (relative mouse mode).\n"
+                "LAlt, LSuper or RSuper toggle the capture mode, to give "
+                "control of the mouse back to the computer.\n"
+                "It may only work over USB, and is currently only supported "
+                "on Linux.\n",
+    },
     {
         .shortopt = 'm',
         .longopt = "max-size",
@@ -240,11 +251,8 @@ static const struct sc_option options[] = {
     {
         .shortopt = 'N',
         .longopt = "no-display",
-        .text = "Do not display device (only when screen recording "
-#ifdef HAVE_V4L2
-                "or V4L2 sink "
-#endif
-                "is enabled).",
+        .text = "Do not display device (only when screen recording or V4L2 "
+                "sink is enabled).",
     },
     {
         .longopt_id = OPT_NO_KEY_REPEAT,
@@ -381,14 +389,14 @@ static const struct sc_option options[] = {
                 "Default is 0 (not forced): the local port used for "
                 "establishing the tunnel will be used.",
     },
-#ifdef HAVE_V4L2
     {
         .longopt_id = OPT_V4L2_SINK,
         .longopt = "v4l2-sink",
         .argdesc = "/dev/videoN",
         .text = "Output to v4l2loopback device.\n"
                 "It requires to lock the video orientation (see "
-                "--lock-video-orientation).",
+                "--lock-video-orientation).\n"
+                "This feature is only available on Linux.",
     },
     {
         .longopt_id = OPT_V4L2_BUFFER,
@@ -398,9 +406,9 @@ static const struct sc_option options[] = {
                 "frames. This increases latency to compensate for jitter.\n"
                 "This option is similar to --display-buffer, but specific to "
                 "V4L2 sink.\n"
+                "This feature is only available on Linux.\n"
                 "Default is 0 (no buffering).",
     },
-#endif
     {
         .shortopt = 'V',
         .longopt = "verbosity",
@@ -1300,7 +1308,13 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
                 args->help = true;
                 break;
             case 'K':
+#ifdef HAVE_AOA_HID
                 opts->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_HID;
+#else
+                LOGE("HID over AOA (-K/--hid-keyboard) is not supported on "
+                     "this platform. It is only available on Linux.");
+                return false;
+#endif
                 break;
             case OPT_MAX_FPS:
                 if (!parse_max_fps(optarg, &opts->max_fps)) {
@@ -1312,6 +1326,15 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
                     return false;
                 }
                 break;
+            case 'M':
+#ifdef HAVE_AOA_HID
+                opts->mouse_input_mode = SC_MOUSE_INPUT_MODE_HID;
+#else
+                LOGE("HID over AOA (-M/--hid-mouse) is not supported on this"
+                     "platform. It is only available on Linux.");
+                return false;
+#endif
+                break;
             case OPT_LOCK_VIDEO_ORIENTATION:
                 if (!parse_lock_video_orientation(optarg,
                         &opts->lock_video_orientation)) {
@@ -1464,16 +1487,24 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
                 opts->tcpip = true;
                 opts->tcpip_dst = optarg;
                 break;
-#ifdef HAVE_V4L2
             case OPT_V4L2_SINK:
+#ifdef HAVE_V4L2
                 opts->v4l2_device = optarg;
+#else
+                LOGE("V4L2 (--v4l2-sink) is only available on Linux.");
+                return false;
+#endif
                 break;
             case OPT_V4L2_BUFFER:
+#ifdef HAVE_V4L2
                 if (!parse_buffering_time(optarg, &opts->v4l2_buffer)) {
                     return false;
                 }
-                break;
+#else
+                LOGE("V4L2 (--v4l2-buffer) is only available on Linux.");
+                return false;
 #endif
+                break;
             default:
                 // getopt prints the error message on stderr
                 return false;

app/src/control_msg.c

@@ -119,7 +119,8 @@ control_msg_serialize(const struct control_msg *msg, unsigned char *buf) {
                              (uint32_t) msg->inject_scroll_event.hscroll);
             buffer_write32be(&buf[17],
                              (uint32_t) msg->inject_scroll_event.vscroll);
-            return 21;
+            buffer_write32be(&buf[21], msg->inject_scroll_event.buttons);
+            return 25;
         case CONTROL_MSG_TYPE_BACK_OR_SCREEN_ON:
             buf[1] = msg->inject_keycode.action;
             return 2;
@@ -192,11 +193,12 @@ control_msg_log(const struct control_msg *msg) {
         }
         case CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT:
             LOG_CMSG("scroll position=%" PRIi32 ",%" PRIi32 " hscroll=%" PRIi32
-                         " vscroll=%" PRIi32,
+                         " vscroll=%" PRIi32 " buttons=%06lx",
                      msg->inject_scroll_event.position.point.x,
                      msg->inject_scroll_event.position.point.y,
                      msg->inject_scroll_event.hscroll,
-                     msg->inject_scroll_event.vscroll);
+                     msg->inject_scroll_event.vscroll,
+                     (long) msg->inject_scroll_event.buttons);
             break;
         case CONTROL_MSG_TYPE_BACK_OR_SCREEN_ON:
             LOG_CMSG("back-or-screen-on %s",

app/src/control_msg.h

@@ -70,6 +70,7 @@ struct control_msg {
             struct sc_position position;
             int32_t hscroll;
             int32_t vscroll;
+            enum android_motionevent_buttons buttons;
         } inject_scroll_event;
         struct {
             enum android_keyevent_action action; // action for the BACK key

app/src/hid_mouse.c

@@ -43,24 +43,24 @@ static const unsigned char mouse_report_desc[]  = {
 
     // Usage Minimum (1)
     0x19, 0x01,
-     // Usage Maximum (3)
-    0x29, 0x03,
+     // Usage Maximum (5)
+    0x29, 0x05,
     // Logical Minimum (0)
     0x15, 0x00,
     // Logical Maximum (1)
     0x25, 0x01,
-    // Report Count (3)
-    0x95, 0x03,
+    // Report Count (5)
+    0x95, 0x05,
     // Report Size (1)
     0x75, 0x01,
-    // Input (Data, Variable, Absolute): 3 buttons bits
+    // Input (Data, Variable, Absolute): 5 buttons bits
     0x81, 0x02,
 
     // Report Count (1)
     0x95, 0x01,
-    // Report Size (5)
-    0x75, 0x05,
-    // Input (Constant): 5 bits padding
+    // Report Size (3)
+    0x75, 0x03,
+    // Input (Constant): 3 bits padding
     0x81, 0x01,
 
     // Usage Page (Generic Desktop)
@@ -98,12 +98,14 @@ static const unsigned char mouse_report_desc[]  = {
  *
  *                   7 6 5 4 3 2 1 0
  *                  +---------------+
- *         byte 0:  |0 0 0 0 0 . . .| buttons state
+ *         byte 0:  |0 0 0 . . . . .| buttons state
  *                  +---------------+
- *                             ^ ^ ^
- *                             | | `- left button
- *                             | `--- right button
- *                             `----- middle button
+ *                         ^ ^ ^ ^ ^
+ *                         | | | | `- left button
+ *                         | | | `--- right button
+ *                         | | `----- middle button
+ *                         | `------- button 4
+ *                         `--------- button 5
  *
  *                  +---------------+
  *         byte 1:  |. . . . . . . .| relative x motion
@@ -152,7 +154,12 @@ buttons_state_to_hid_buttons(uint8_t buttons_state) {
     if (buttons_state & SC_MOUSE_BUTTON_MIDDLE) {
         c |= 1 << 2;
     }
-    // TODO buttons 4 and 5?
+    if (buttons_state & SC_MOUSE_BUTTON_X1) {
+        c |= 1 << 3;
+    }
+    if (buttons_state & SC_MOUSE_BUTTON_X2) {
+        c |= 1 << 4;
+    }
     return c;
 }
 
@@ -246,6 +253,8 @@ sc_hid_mouse_init(struct sc_hid_mouse *mouse, struct sc_aoa *aoa) {
 
     mouse->mouse_processor.ops = &ops;
 
+    mouse->mouse_processor.relative_mode = true;
+
     return true;
 }
 

app/src/input_events.h

@@ -361,6 +361,7 @@ struct sc_mouse_scroll_event {
     struct sc_position position;
     int32_t hscroll;
     int32_t vscroll;
+    uint8_t buttons_state; // bitwise-OR of sc_mouse_button values
 };
 
 struct sc_mouse_motion_event {

app/src/input_manager.c

@@ -4,6 +4,7 @@
 #include <SDL2/SDL_keycode.h>
 
 #include "input_events.h"
+#include "screen.h"
 #include "util/log.h"
 
 static inline uint16_t
@@ -121,24 +122,22 @@ is_shortcut_mod(struct input_manager *im, uint16_t sdl_mod) {
 }
 
 void
-input_manager_init(struct input_manager *im, struct controller *controller,
-                   struct screen *screen, struct sc_key_processor *kp,
-                   struct sc_mouse_processor *mp,
-                   const struct scrcpy_options *options) {
-    assert(!options->control || (kp && kp->ops));
-    assert(!options->control || (mp && mp->ops));
+input_manager_init(struct input_manager *im,
+                   const struct input_manager_params *params) {
+    assert(!params->control || (params->kp && params->kp->ops));
+    assert(!params->control || (params->mp && params->mp->ops));
 
-    im->controller = controller;
-    im->screen = screen;
-    im->kp = kp;
-    im->mp = mp;
+    im->controller = params->controller;
+    im->screen = params->screen;
+    im->kp = params->kp;
+    im->mp = params->mp;
 
-    im->control = options->control;
-    im->forward_all_clicks = options->forward_all_clicks;
-    im->legacy_paste = options->legacy_paste;
-    im->clipboard_autosync = options->clipboard_autosync;
+    im->control = params->control;
+    im->forward_all_clicks = params->forward_all_clicks;
+    im->legacy_paste = params->legacy_paste;
+    im->clipboard_autosync = params->clipboard_autosync;
 
-    const struct sc_shortcut_mods *shortcut_mods = &options->shortcut_mods;
+    const struct sc_shortcut_mods *shortcut_mods = params->shortcut_mods;
     assert(shortcut_mods->count);
     assert(shortcut_mods->count < SC_MAX_SHORTCUT_MODS);
     for (unsigned i = 0; i < shortcut_mods->count; ++i) {
@@ -777,6 +776,12 @@ input_manager_process_mouse_button(struct input_manager *im,
     assert(im->mp->ops->process_mouse_click);
     im->mp->ops->process_mouse_click(im->mp, &evt);
 
+    if (im->mp->relative_mode) {
+        assert(!im->vfinger_down); // vfinger must not be used in relative mode
+        // No pinch-to-zoom simulation
+        return;
+    }
+
     // Pinch-to-zoom simulation.
     //
     // If Ctrl is hold when the left-click button is pressed, then
@@ -814,7 +819,7 @@ input_manager_process_mouse_wheel(struct input_manager *im,
     // mouse_x and mouse_y are expressed in pixels relative to the window
     int mouse_x;
     int mouse_y;
-    SDL_GetMouseState(&mouse_x, &mouse_y);
+    uint32_t buttons = SDL_GetMouseState(&mouse_x, &mouse_y);
 
     struct sc_mouse_scroll_event evt = {
         .position = {
@@ -824,6 +829,8 @@ input_manager_process_mouse_wheel(struct input_manager *im,
         },
         .hscroll = event->x,
         .vscroll = event->y,
+        .buttons_state =
+            sc_mouse_buttons_state_from_sdl(buttons, im->forward_all_clicks),
     };
 
     im->mp->ops->process_mouse_scroll(im->mp, &evt);

app/src/input_manager.h

@@ -10,10 +10,11 @@
 #include "controller.h"
 #include "fps_counter.h"
 #include "options.h"
-#include "screen.h"
 #include "trait/key_processor.h"
 #include "trait/mouse_processor.h"
 
+struct screen; // forward declaration to avoid cycle
+
 struct input_manager {
     struct controller *controller;
     struct screen *screen;
@@ -43,11 +44,22 @@ struct input_manager {
     uint64_t next_sequence; // used for request acknowledgements
 };
 
+struct input_manager_params {
+    struct controller *controller;
+    struct screen *screen;
+    struct sc_key_processor *kp;
+    struct sc_mouse_processor *mp;
+
+    bool control;
+    bool forward_all_clicks;
+    bool legacy_paste;
+    bool clipboard_autosync;
+    const struct sc_shortcut_mods *shortcut_mods;
+};
+
 void
-input_manager_init(struct input_manager *im, struct controller *controller,
-                   struct screen *screen, struct sc_key_processor *kp,
-                   struct sc_mouse_processor *mp,
-                   const struct scrcpy_options *options);
+input_manager_init(struct input_manager *im,
+                   const struct input_manager_params *params);
 
 bool
 input_manager_handle_event(struct input_manager *im, SDL_Event *event);

app/src/mouse_inject.c

@@ -108,6 +108,7 @@ sc_mouse_processor_process_mouse_scroll(struct sc_mouse_processor *mp,
             .position = event->position,
             .hscroll = event->hscroll,
             .vscroll = event->vscroll,
+            .buttons = convert_mouse_buttons(event->buttons_state),
         },
     };
 
@@ -150,4 +151,6 @@ sc_mouse_inject_init(struct sc_mouse_inject *mi,
     };
 
     mi->mouse_processor.ops = &ops;
+
+    mi->mouse_processor.relative_mode = false;
 }

app/src/options.h

@@ -38,6 +38,11 @@ enum sc_keyboard_input_mode {
     SC_KEYBOARD_INPUT_MODE_HID,
 };
 
+enum sc_mouse_input_mode {
+    SC_MOUSE_INPUT_MODE_INJECT,
+    SC_MOUSE_INPUT_MODE_HID,
+};
+
 enum sc_key_inject_mode {
     // Inject special keys, letters and space as key events.
     // Inject numbers and punctuation as text events.
@@ -90,6 +95,7 @@ struct scrcpy_options {
     enum sc_log_level log_level;
     enum sc_record_format record_format;
     enum sc_keyboard_input_mode keyboard_input_mode;
+    enum sc_mouse_input_mode mouse_input_mode;
     struct sc_port_range port_range;
     uint32_t tunnel_host;
     uint16_t tunnel_port;

app/src/scrcpy.c

@@ -17,7 +17,6 @@
 #include "decoder.h"
 #include "events.h"
 #include "file_handler.h"
-#include "input_manager.h"
 #ifdef HAVE_AOA_HID
 # include "hid_keyboard.h"
 # include "hid_mouse.h"
@@ -63,7 +62,6 @@ struct scrcpy {
         struct sc_hid_mouse mouse_hid;
 #endif
     };
-    struct input_manager input_manager;
 };
 
 static inline void
@@ -195,11 +193,6 @@ handle_event(struct scrcpy *s, const struct scrcpy_options *options,
     }
 
     bool consumed = screen_handle_event(&s->screen, event);
-    if (consumed) {
-        goto end;
-    }
-
-    consumed = input_manager_handle_event(&s->input_manager, event);
     (void) consumed;
 
 end:
@@ -342,6 +335,8 @@ scrcpy(struct scrcpy_options *options) {
     bool stream_started = false;
 #ifdef HAVE_AOA_HID
     bool aoa_hid_initialized = false;
+    bool hid_keyboard_initialized = false;
+    bool hid_mouse_initialized = false;
 #endif
     bool controller_initialized = false;
     bool controller_started = false;
@@ -456,17 +451,100 @@ scrcpy(struct scrcpy_options *options) {
         stream_add_sink(&s->stream, &rec->packet_sink);
     }
 
+    struct sc_key_processor *kp = NULL;
+    struct sc_mouse_processor *mp = NULL;
+
     if (options->control) {
 #ifdef HAVE_AOA_HID
-        if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID) {
+        bool use_hid_keyboard =
+            options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID;
+        bool use_hid_mouse =
+            options->mouse_input_mode == SC_MOUSE_INPUT_MODE_HID;
+        if (use_hid_keyboard || use_hid_mouse) {
             bool ok = sc_acksync_init(&s->acksync);
             if (!ok) {
                 goto end;
             }
 
+            ok = sc_aoa_init(&s->aoa, serial, &s->acksync);
+            if (!ok) {
+                LOGE("Failed to enable HID over AOA");
+                sc_acksync_destroy(&s->acksync);
+                goto aoa_hid_end;
+            }
+
+            if (use_hid_keyboard) {
+                if (sc_hid_keyboard_init(&s->keyboard_hid, &s->aoa)) {
+                    hid_keyboard_initialized = true;
+                    kp = &s->keyboard_hid.key_processor;
+                } else {
+                    LOGE("Could not initialize HID keyboard");
+                }
+            }
+
+            if (use_hid_mouse) {
+                if (sc_hid_mouse_init(&s->mouse_hid, &s->aoa)) {
+                    hid_mouse_initialized = true;
+                    mp = &s->mouse_hid.mouse_processor;
+                } else {
+                    LOGE("Could not initialized HID mouse");
+                }
+            }
+
+            bool need_aoa = hid_keyboard_initialized || hid_mouse_initialized;
+
+            if (!need_aoa || !sc_aoa_start(&s->aoa)) {
+                sc_acksync_destroy(&s->acksync);
+                sc_aoa_destroy(&s->aoa);
+                goto aoa_hid_end;
+            }
+
             acksync = &s->acksync;
+
+            aoa_hid_initialized = true;
+
+aoa_hid_end:
+            if (!aoa_hid_initialized) {
+                if (hid_keyboard_initialized) {
+                    sc_hid_keyboard_destroy(&s->keyboard_hid);
+                    hid_keyboard_initialized = false;
+                }
+                if (hid_mouse_initialized) {
+                    sc_hid_mouse_destroy(&s->mouse_hid);
+                    hid_mouse_initialized = false;
+                }
+            }
+
+            if (use_hid_keyboard && !hid_keyboard_initialized) {
+                LOGE("Fallback to default keyboard injection method "
+                     "(-K/--hid-keyboard ignored)");
+                options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT;
+            }
+
+            if (use_hid_mouse && !hid_mouse_initialized) {
+                LOGE("Fallback to default mouse injection method "
+                     "(-M/--hid-mouse ignored)");
+                options->mouse_input_mode = SC_MOUSE_INPUT_MODE_INJECT;
+            }
         }
+#else
+        assert(options->keyboard_input_mode != SC_KEYBOARD_INPUT_MODE_HID);
+        assert(options->mouse_input_mode != SC_MOUSE_INPUT_MODE_HID);
 #endif
+
+        // keyboard_input_mode may have been reset if HID mode failed
+        if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_INJECT) {
+            sc_keyboard_inject_init(&s->keyboard_inject, &s->controller,
+                                    options);
+            kp = &s->keyboard_inject.key_processor;
+        }
+
+        // mouse_input_mode may have been reset if HID mode failed
+        if (options->mouse_input_mode == SC_MOUSE_INPUT_MODE_INJECT) {
+            sc_mouse_inject_init(&s->mouse_inject, &s->controller);
+            mp = &s->mouse_inject.mouse_processor;
+        }
+
         if (!controller_init(&s->controller, s->server.control_socket,
                              acksync)) {
             goto end;
@@ -487,6 +565,7 @@ scrcpy(struct scrcpy_options *options) {
                 LOGW("Could not request 'set screen power mode'");
             }
         }
+
     }
 
     if (options->display) {
@@ -494,6 +573,14 @@ scrcpy(struct scrcpy_options *options) {
             options->window_title ? options->window_title : info->device_name;
 
         struct screen_params screen_params = {
+            .controller = &s->controller,
+            .kp = kp,
+            .mp = mp,
+            .control = options->control,
+            .forward_all_clicks = options->forward_all_clicks,
+            .legacy_paste = options->legacy_paste,
+            .clipboard_autosync = options->clipboard_autosync,
+            .shortcut_mods = &options->shortcut_mods,
             .window_title = window_title,
             .frame_size = info->frame_size,
             .always_on_top = options->always_on_top,
@@ -536,65 +623,6 @@ scrcpy(struct scrcpy_options *options) {
     }
     stream_started = true;
 
-    struct sc_key_processor *kp = NULL;
-    struct sc_mouse_processor *mp = NULL;
-
-    if (options->control) {
-        if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID) {
-#ifdef HAVE_AOA_HID
-            bool aoa_hid_ok = false;
-
-            bool ok = sc_aoa_init(&s->aoa, serial, acksync);
-            if (!ok) {
-                goto aoa_hid_end;
-            }
-
-            if (!sc_hid_keyboard_init(&s->keyboard_hid, &s->aoa)) {
-                sc_aoa_destroy(&s->aoa);
-                goto aoa_hid_end;
-            }
-
-            if (!sc_aoa_start(&s->aoa)) {
-                sc_hid_keyboard_destroy(&s->keyboard_hid);
-                sc_aoa_destroy(&s->aoa);
-                goto aoa_hid_end;
-            }
-
-            aoa_hid_ok = true;
-            kp = &s->keyboard_hid.key_processor;
-
-            aoa_hid_initialized = true;
-
-aoa_hid_end:
-            if (!aoa_hid_ok) {
-                LOGE("Failed to enable HID over AOA, "
-                     "fallback to default keyboard injection method "
-                     "(-K/--hid-keyboard ignored)");
-                options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT;
-            }
-#else
-            LOGE("HID over AOA is not supported on this platform, "
-                 "fallback to default keyboard injection method "
-                 "(-K/--hid-keyboard ignored)");
-            options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT;
-#endif
-        }
-
-        // keyboard_input_mode may have been reset if HID mode failed
-        if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_INJECT) {
-            sc_keyboard_inject_init(&s->keyboard_inject, &s->controller,
-                                    options);
-            kp = &s->keyboard_inject.key_processor;
-        }
-
-        //sc_mouse_inject_init(&s->mouse_inject, &s->controller);
-        sc_hid_mouse_init(&s->mouse_hid, &s->aoa);
-        mp = &s->mouse_hid.mouse_processor;
-    }
-
-    input_manager_init(&s->input_manager, &s->controller, &s->screen, kp, mp,
-                       options);
-
     ret = event_loop(s, options);
     LOGD("quit...");
 
@@ -607,7 +635,9 @@ end:
     // end-of-stream
 #ifdef HAVE_AOA_HID
     if (aoa_hid_initialized) {
-        sc_hid_keyboard_destroy(&s->keyboard_hid);
+        if (hid_keyboard_initialized) {
+            sc_hid_keyboard_destroy(&s->keyboard_hid);
+        }
         sc_aoa_stop(&s->aoa);
     }
     if (acksync) {

app/src/screen.c

@@ -156,6 +156,17 @@ get_initial_optimal_size(struct sc_size content_size, uint16_t req_width,
     return window_size;
 }
 
+static inline void
+screen_capture_mouse(struct screen *screen, bool capture) {
+    if (SDL_SetRelativeMouseMode(capture)) {
+        LOGE("Could not set relative mouse mode to %s: %s",
+             capture ? "true" : "false", SDL_GetError());
+        return;
+    }
+
+    screen->mouse_captured = capture;
+}
+
 static void
 screen_update_content_rect(struct screen *screen) {
     int dw;
@@ -354,6 +365,8 @@ screen_init(struct screen *screen, const struct screen_params *params) {
     screen->fullscreen = false;
     screen->maximized = false;
     screen->event_failed = false;
+    screen->mouse_captured = false;
+    screen->mouse_capture_key_pressed = 0;
 
     static const struct sc_video_buffer_callbacks cbs = {
         .on_new_frame = sc_video_buffer_on_new_frame,
@@ -470,6 +483,20 @@ screen_init(struct screen *screen, const struct screen_params *params) {
         goto error_destroy_texture;
     }
 
+    struct input_manager_params im_params = {
+        .controller = params->controller,
+        .screen = screen,
+        .kp = params->kp,
+        .mp = params->mp,
+        .control = params->control,
+        .forward_all_clicks = params->forward_all_clicks,
+        .legacy_paste = params->legacy_paste,
+        .clipboard_autosync = params->clipboard_autosync,
+        .shortcut_mods = params->shortcut_mods,
+    };
+
+    input_manager_init(&screen->im, &im_params);
+
     // Reset the window size to trigger a SIZE_CHANGED event, to workaround
     // HiDPI issues with some SDL renderers when several displays having
     // different HiDPI scaling are connected
@@ -485,10 +512,6 @@ screen_init(struct screen *screen, const struct screen_params *params) {
     SDL_AddEventWatch(event_watcher, screen);
 #endif
 
-    if (SDL_SetRelativeMouseMode(true)) {
-        LOGE("Could not set relative mouse mode: %s", SDL_GetError());
-    }
-
     static const struct sc_frame_sink_ops ops = {
         .open = screen_frame_sink_open,
         .close = screen_frame_sink_close,
@@ -731,6 +754,11 @@ screen_resize_to_pixel_perfect(struct screen *screen) {
                                             content_size.height);
 }
 
+static inline bool
+screen_is_mouse_capture_key(SDL_Keycode key) {
+    return key == SDLK_LALT || key == SDLK_LGUI || key == SDLK_RGUI;
+}
+
 bool
 screen_handle_event(struct screen *screen, SDL_Event *event) {
     switch (event->type) {
@@ -773,11 +801,71 @@ screen_handle_event(struct screen *screen, SDL_Event *event) {
                     apply_pending_resize(screen);
                     screen_render(screen, true);
                     break;
+                case SDL_WINDOWEVENT_FOCUS_LOST:
+                    if (screen->im.mp->relative_mode) {
+                        screen_capture_mouse(screen, false);
+                    }
+                    break;
             }
             return true;
+        case SDL_KEYDOWN:
+            if (screen->im.mp->relative_mode) {
+                SDL_Keycode key = event->key.keysym.sym;
+                if (screen_is_mouse_capture_key(key)) {
+                    if (!screen->mouse_capture_key_pressed) {
+                        screen->mouse_capture_key_pressed = key;
+                        return true;
+                    } else {
+                        // Another mouse capture key has been pressed, cancel
+                        // mouse (un)capture
+                        screen->mouse_capture_key_pressed = 0;
+                        // Do not return, the event must be forwarded to the
+                        // input manager
+                    }
+                }
+            }
+            break;
+        case SDL_KEYUP:
+            if (screen->im.mp->relative_mode) {
+                SDL_Keycode key = event->key.keysym.sym;
+                SDL_Keycode cap = screen->mouse_capture_key_pressed;
+                screen->mouse_capture_key_pressed = 0;
+                if (key == cap) {
+                    // A mouse capture key has been pressed then released:
+                    // toggle the capture mouse mode
+                    screen_capture_mouse(screen, !screen->mouse_captured);
+                    return true;
+                }
+                // Do not return, the event must be forwarded to the input
+                // manager
+            }
+            break;
+        case SDL_MOUSEWHEEL:
+        case SDL_MOUSEMOTION:
+        case SDL_MOUSEBUTTONDOWN:
+            if (screen->im.mp->relative_mode && !screen->mouse_captured) {
+                // Do not forward to input manager, the mouse will be captured
+                // on SDL_MOUSEBUTTONUP
+                return true;
+            }
+            break;
+        case SDL_FINGERMOTION:
+        case SDL_FINGERDOWN:
+        case SDL_FINGERUP:
+            if (screen->im.mp->relative_mode) {
+                // Touch events are not compatible with relative mode
+                // (coordinates are not relative)
+                return true;
+            }
+            break;
+        case SDL_MOUSEBUTTONUP:
+            if (screen->im.mp->relative_mode && !screen->mouse_captured) {
+                screen_capture_mouse(screen, true);
+                return true;
+            }
     }
 
-    return false;
+    return input_manager_handle_event(&screen->im, event);
 }
 
 struct sc_point

app/src/screen.h

@@ -7,10 +7,14 @@
 #include <SDL2/SDL.h>
 #include <libavformat/avformat.h>
 
+#include "controller.h"
 #include "coords.h"
 #include "fps_counter.h"
+#include "input_manager.h"
 #include "opengl.h"
+#include "trait/key_processor.h"
 #include "trait/frame_sink.h"
+#include "trait/mouse_processor.h"
 #include "video_buffer.h"
 
 struct screen {
@@ -20,6 +24,7 @@ struct screen {
     bool open; // track the open/close state to assert correct behavior
 #endif
 
+    struct input_manager im;
     struct sc_video_buffer vb;
     struct fps_counter fps_counter;
 
@@ -46,10 +51,25 @@ struct screen {
 
     bool event_failed; // in case SDL_PushEvent() returned an error
 
+    bool mouse_captured; // only relevant in relative mouse mode
+    // To enable/disable mouse capture, a mouse capture key (LALT, LGUI or
+    // RGUI) must be pressed. This variable tracks the pressed capture key.
+    SDL_Keycode mouse_capture_key_pressed;
+
     AVFrame *frame;
 };
 
 struct screen_params {
+    struct controller *controller;
+    struct sc_key_processor *kp;
+    struct sc_mouse_processor *mp;
+
+    bool control;
+    bool forward_all_clicks;
+    bool legacy_paste;
+    bool clipboard_autosync;
+    const struct sc_shortcut_mods *shortcut_mods;
+
     const char *window_title;
     struct sc_size frame_size;
     bool always_on_top;

app/src/trait/mouse_processor.h

@@ -16,6 +16,13 @@
  */
 struct sc_mouse_processor {
     const struct sc_mouse_processor_ops *ops;
+
+    /**
+     * If set, the mouse processor works in relative mode (the absolute
+     * position is irrelevant). In particular, it indicates that the mouse
+     * pointer must be "captured" by the UI.
+     */
+    bool relative_mode;
 };
 
 struct sc_mouse_processor_ops {

app/tests/test_adb_parser.c

@@ -11,6 +11,7 @@ static void test_get_ip_single_line() {
     char *ip = sc_adb_parse_device_ip_from_output(ip_route, sizeof(ip_route));
     assert(ip);
     assert(!strcmp(ip, "192.168.12.34"));
+    free(ip);
 }
 
 static void test_get_ip_single_line_without_eol() {
@@ -20,6 +21,7 @@ static void test_get_ip_single_line_without_eol() {
     char *ip = sc_adb_parse_device_ip_from_output(ip_route, sizeof(ip_route));
     assert(ip);
     assert(!strcmp(ip, "192.168.12.34"));
+    free(ip);
 }
 
 static void test_get_ip_single_line_with_trailing_space() {
@@ -29,6 +31,7 @@ static void test_get_ip_single_line_with_trailing_space() {
     char *ip = sc_adb_parse_device_ip_from_output(ip_route, sizeof(ip_route));
     assert(ip);
     assert(!strcmp(ip, "192.168.12.34"));
+    free(ip);
 }
 
 static void test_get_ip_multiline_first_ok() {
@@ -40,6 +43,7 @@ static void test_get_ip_multiline_first_ok() {
     char *ip = sc_adb_parse_device_ip_from_output(ip_route, sizeof(ip_route));
     assert(ip);
     assert(!strcmp(ip, "192.168.1.2"));
+    free(ip);
 }
 
 static void test_get_ip_multiline_second_ok() {
@@ -51,6 +55,7 @@ static void test_get_ip_multiline_second_ok() {
     char *ip = sc_adb_parse_device_ip_from_output(ip_route, sizeof(ip_route));
     assert(ip);
     assert(!strcmp(ip, "192.168.1.3"));
+    free(ip);
 }
 
 static void test_get_ip_no_wlan() {

app/tests/test_control_msg_serialize.c

@@ -126,12 +126,13 @@ static void test_serialize_inject_scroll_event(void) {
             },
             .hscroll = 1,
             .vscroll = -1,
+            .buttons = 1,
         },
     };
 
     unsigned char buf[CONTROL_MSG_MAX_SIZE];
     size_t size = control_msg_serialize(&msg, buf);
-    assert(size == 21);
+    assert(size == 25);
 
     const unsigned char expected[] = {
         CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT,
@@ -139,6 +140,7 @@ static void test_serialize_inject_scroll_event(void) {
         0x04, 0x38, 0x07, 0x80, // 1080 1920
         0x00, 0x00, 0x00, 0x01, // 1
         0xFF, 0xFF, 0xFF, 0xFF, // -1
+        0x00, 0x00, 0x00, 0x01, // 1
     };
     assert(!memcmp(buf, expected, sizeof(expected)));
 }

server/src/main/java/com/genymobile/scrcpy/ControlMessage.java

@@ -71,12 +71,13 @@ public final class ControlMessage {
         return msg;
     }
 
-    public static ControlMessage createInjectScrollEvent(Position position, int hScroll, int vScroll) {
+    public static ControlMessage createInjectScrollEvent(Position position, int hScroll, int vScroll, int buttons) {
         ControlMessage msg = new ControlMessage();
         msg.type = TYPE_INJECT_SCROLL_EVENT;
         msg.position = position;
         msg.hScroll = hScroll;
         msg.vScroll = vScroll;
+        msg.buttons = buttons;
         return msg;
     }
 

server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java

@@ -10,7 +10,7 @@ public class ControlMessageReader {
 
     static final int INJECT_KEYCODE_PAYLOAD_LENGTH = 13;
     static final int INJECT_TOUCH_EVENT_PAYLOAD_LENGTH = 27;
-    static final int INJECT_SCROLL_EVENT_PAYLOAD_LENGTH = 20;
+    static final int INJECT_SCROLL_EVENT_PAYLOAD_LENGTH = 24;
     static final int BACK_OR_SCREEN_ON_LENGTH = 1;
     static final int SET_SCREEN_POWER_MODE_PAYLOAD_LENGTH = 1;
     static final int GET_CLIPBOARD_LENGTH = 1;
@@ -154,7 +154,8 @@ public class ControlMessageReader {
         Position position = readPosition(buffer);
         int hScroll = buffer.getInt();
         int vScroll = buffer.getInt();
-        return ControlMessage.createInjectScrollEvent(position, hScroll, vScroll);
+        int buttons = buffer.getInt();
+        return ControlMessage.createInjectScrollEvent(position, hScroll, vScroll, buttons);
     }
 
     private ControlMessage parseBackOrScreenOnEvent() {

server/src/main/java/com/genymobile/scrcpy/Controller.java

@@ -98,7 +98,7 @@ public class Controller {
                 break;
             case ControlMessage.TYPE_INJECT_SCROLL_EVENT:
                 if (device.supportsInputEvents()) {
-                    injectScroll(msg.getPosition(), msg.getHScroll(), msg.getVScroll());
+                    injectScroll(msg.getPosition(), msg.getHScroll(), msg.getVScroll(), msg.getButtons());
                 }
                 break;
             case ControlMessage.TYPE_BACK_OR_SCREEN_ON:
@@ -221,7 +221,7 @@ public class Controller {
         return device.injectEvent(event, Device.INJECT_MODE_ASYNC);
     }
 
-    private boolean injectScroll(Position position, int hScroll, int vScroll) {
+    private boolean injectScroll(Position position, int hScroll, int vScroll, int buttons) {
         long now = SystemClock.uptimeMillis();
         Point point = device.getPhysicalPoint(position);
         if (point == null) {
@@ -239,7 +239,7 @@ public class Controller {
         coords.setAxisValue(MotionEvent.AXIS_VSCROLL, vScroll);
 
         MotionEvent event = MotionEvent
-                .obtain(lastTouchDown, now, MotionEvent.ACTION_SCROLL, 1, pointerProperties, pointerCoords, 0, 0, 1f, 1f, DEFAULT_DEVICE_ID, 0,
+                .obtain(lastTouchDown, now, MotionEvent.ACTION_SCROLL, 1, pointerProperties, pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0,
                         InputDevice.SOURCE_MOUSE, 0);
         return device.injectEvent(event, Device.INJECT_MODE_ASYNC);
     }

server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java

@@ -128,6 +128,7 @@ public class ControlMessageReaderTest {
         dos.writeShort(1920);
         dos.writeInt(1);
         dos.writeInt(-1);
+        dos.writeInt(1);
 
         byte[] packet = bos.toByteArray();
 
@@ -144,6 +145,7 @@ public class ControlMessageReaderTest {
         Assert.assertEquals(1920, event.getPosition().getScreenSize().getHeight());
         Assert.assertEquals(1, event.getHScroll());
         Assert.assertEquals(-1, event.getVScroll());
+        Assert.assertEquals(1, event.getButtons());
     }
 
     @Test