Fix HiDPI issues on secondary screen

Reset the renderer and texture on display scaling change.

FIXME problem with texture (see FIXME in code)

Co-authored-by: Louis Kruger <louisk@gmail.com>

FAQ.md

@@ -142,13 +142,13 @@ screen, then you might get poor quality, especially visible on text (see [#40]).
 To improve downscaling quality, trilinear filtering is enabled automatically
 if the renderer is OpenGL and if it supports mipmapping.
 
-On Windows or macOS, you might need to force OpenGL:
+On Windows, you might want to force OpenGL:
 
 ```
 scrcpy --render-driver=opengl
 ```
 
-On Windows, you may also need to configure the [scaling behavior].
+You may also need to configure the [scaling behavior]:
 
 > `scrcpy.exe` > Properties > Compatibility > Change high DPI settings >
 > Override high DPI scaling behavior > Scaling performed by: _Application_.

README.md

@@ -363,7 +363,7 @@ Note that _scrcpy_ manages 3 different rotations:
    current running app may refuse, if it does support the requested
    orientation).
  - `--lock-video-orientation` changes the mirroring orientation (the orientation
-   of the video sent from the device to the computer. This affects the
+   of the video sent from the device to the computer). This affects the
    recording.
  - `--rotation` (or `Ctrl`+`←`/`Ctrl`+`→`) rotates only the window content. This
    affects only the display, not the recording.

app/meson.build

@@ -116,9 +116,6 @@ conf.set('DEFAULT_LOCK_VIDEO_ORIENTATION', '-1')  # -1: unlocked
 # overridden by option --bit-rate
 conf.set('DEFAULT_BIT_RATE', '8000000')  # 8Mbps
 
-# enable High DPI support
-conf.set('HIDPI_SUPPORT', get_option('hidpi_support'))
-
 # disable console on Windows
 conf.set('WINDOWS_NOCONSOLE', get_option('windows_noconsole'))
 

app/src/control_msg.c

@@ -45,8 +45,9 @@ control_msg_serialize(const struct control_msg *msg, unsigned char *buf) {
             buffer_write32be(&buf[6], msg->inject_keycode.metastate);
             return 10;
         case CONTROL_MSG_TYPE_INJECT_TEXT: {
-            size_t len = write_string(msg->inject_text.text,
-                                      CONTROL_MSG_TEXT_MAX_LENGTH, &buf[1]);
+            size_t len =
+                write_string(msg->inject_text.text,
+                             CONTROL_MSG_INJECT_TEXT_MAX_LENGTH, &buf[1]);
             return 1 + len;
         }
         case CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT:

app/src/control_msg.h

@@ -10,7 +10,7 @@
 #include "android/keycodes.h"
 #include "common.h"
 
-#define CONTROL_MSG_TEXT_MAX_LENGTH 300
+#define CONTROL_MSG_INJECT_TEXT_MAX_LENGTH 300
 #define CONTROL_MSG_CLIPBOARD_TEXT_MAX_LENGTH 4093
 #define CONTROL_MSG_SERIALIZED_MAX_SIZE \
     (3 + CONTROL_MSG_CLIPBOARD_TEXT_MAX_LENGTH)

app/src/input_manager.c

@@ -442,36 +442,6 @@ input_manager_process_key(struct input_manager *im,
     }
 }
 
-static struct point
-rotate_position(struct screen *screen, int32_t x, int32_t y) {
-    unsigned rotation = screen->rotation;
-    assert(rotation < 4);
-
-    int32_t w = screen->content_size.width;
-    int32_t h = screen->content_size.height;
-    struct point result;
-    switch (rotation) {
-        case 0:
-            result.x = x;
-            result.y = y;
-            break;
-        case 1:
-            result.x = h - y;
-            result.y = x;
-            break;
-        case 2:
-            result.x = w - x;
-            result.y = h - y;
-            break;
-        default:
-            assert(rotation == 3);
-            result.x = y;
-            result.y = w - x;
-            break;
-    }
-    return result;
-}
-
 static bool
 convert_mouse_motion(const SDL_MouseMotionEvent *from, struct screen *screen,
                      struct control_msg *to) {
@@ -480,7 +450,7 @@ convert_mouse_motion(const SDL_MouseMotionEvent *from, struct screen *screen,
     to->inject_touch_event.pointer_id = POINTER_ID_MOUSE;
     to->inject_touch_event.position.screen_size = screen->frame_size;
     to->inject_touch_event.position.point =
-        rotate_position(screen, from->x, from->y);
+        screen_convert_to_frame_coords(screen, from->x, from->y);
     to->inject_touch_event.pressure = 1.f;
     to->inject_touch_event.buttons = convert_mouse_buttons(from->state);
 
@@ -515,14 +485,13 @@ convert_touch(const SDL_TouchFingerEvent *from, struct screen *screen,
         return false;
     }
 
-    struct size frame_size = screen->frame_size;
-
     to->inject_touch_event.pointer_id = from->fingerId;
-    to->inject_touch_event.position.screen_size = frame_size;
+    to->inject_touch_event.position.screen_size = screen->frame_size;
     // SDL touch event coordinates are normalized in the range [0; 1]
-    float x = from->x * frame_size.width;
-    float y = from->y * frame_size.height;
-    to->inject_touch_event.position.point = rotate_position(screen, x, y);
+    float x = from->x * screen->content_size.width;
+    float y = from->y * screen->content_size.height;
+    to->inject_touch_event.position.point =
+        screen_convert_to_frame_coords(screen, x, y);
     to->inject_touch_event.pressure = from->pressure;
     to->inject_touch_event.buttons = 0;
     return true;
@@ -558,7 +527,7 @@ convert_mouse_button(const SDL_MouseButtonEvent *from, struct screen *screen,
     to->inject_touch_event.pointer_id = POINTER_ID_MOUSE;
     to->inject_touch_event.position.screen_size = screen->frame_size;
     to->inject_touch_event.position.point =
-        rotate_position(screen, from->x, from->y);
+        screen_convert_to_frame_coords(screen, from->x, from->y);
     to->inject_touch_event.pressure = 1.f;
     to->inject_touch_event.buttons =
         convert_mouse_buttons(SDL_BUTTON(from->button));

app/src/scrcpy.c

@@ -109,9 +109,10 @@ static int
 event_watcher(void *data, SDL_Event *event) {
     (void) data;
     if (event->type == SDL_WINDOWEVENT
-            && event->window.event == SDL_WINDOWEVENT_RESIZED) {
-        // called from another thread, not very safe, but it's a workaround!
-        screen_render(&screen);
+            && event->window.event == SDL_WINDOWEVENT_SIZE_CHANGED) {
+        // In practice, it seems to always be called from the same thread in
+        // that specific case. Anyway, it's just a workaround.
+        screen_handle_window_event(&screen, &event->window);
     }
     return 0;
 }
@@ -127,6 +128,7 @@ enum event_result {
     EVENT_RESULT_CONTINUE,
     EVENT_RESULT_STOPPED_BY_USER,
     EVENT_RESULT_STOPPED_BY_EOS,
+    EVENT_RESULT_STOPPED_BY_ERROR,
 };
 
 static enum event_result
@@ -149,7 +151,9 @@ handle_event(SDL_Event *event, bool control) {
             }
             break;
         case SDL_WINDOWEVENT:
-            screen_handle_window_event(&screen, &event->window);
+            if (!screen_handle_window_event(&screen, &event->window)) {
+                return EVENT_RESULT_STOPPED_BY_ERROR;
+            }
             break;
         case SDL_TEXTINPUT:
             if (!control) {
@@ -221,6 +225,9 @@ event_loop(bool display, bool control) {
             case EVENT_RESULT_STOPPED_BY_EOS:
                 LOGW("Device disconnected");
                 return false;
+            case EVENT_RESULT_STOPPED_BY_ERROR:
+                LOGC("Stopping due to unrecoverable error");
+                return false;
             case EVENT_RESULT_CONTINUE:
                 break;
         }

app/src/screen.c

@@ -113,6 +113,13 @@ get_optimal_size(struct size current_size, struct size content_size) {
         h = MIN(current_size.height, display_size.height);
     }
 
+    if (h == w * content_size.height / content_size.width
+     || w == h * content_size.width / content_size.height) {
+        // The size is already optimal, if we ignore rounding errors due to
+        // integer window dimensions
+        return (struct size) {w, h};
+    }
+
     bool keep_width = content_size.width * h > content_size.height * w;
     if (keep_width) {
         // remove black borders on top and bottom
@@ -210,10 +217,9 @@ screen_init_rendering(struct screen *screen, const char *window_title,
 
     struct size window_size =
         get_initial_optimal_size(content_size, window_width, window_height);
-    uint32_t window_flags = SDL_WINDOW_HIDDEN | SDL_WINDOW_RESIZABLE;
-#ifdef HIDPI_SUPPORT
-    window_flags |= SDL_WINDOW_ALLOW_HIGHDPI;
-#endif
+    uint32_t window_flags = SDL_WINDOW_HIDDEN
+                          | SDL_WINDOW_RESIZABLE
+                          | SDL_WINDOW_ALLOW_HIGHDPI;
     if (always_on_top) {
 #ifdef SCRCPY_SDL_HAS_WINDOW_ALWAYS_ON_TOP
         window_flags |= SDL_WINDOW_ALWAYS_ON_TOP;
@@ -258,7 +264,7 @@ screen_init_rendering(struct screen *screen, const char *window_title,
         return false;
     }
 
-    // stats with "opengl"
+    // starts with "opengl"
     screen->use_opengl = renderer_name && !strncmp(renderer_name, "opengl", 6);
     if (screen->use_opengl) {
         struct sc_opengl *gl = &screen->gl;
@@ -303,6 +309,11 @@ screen_init_rendering(struct screen *screen, const char *window_title,
 
     screen->windowed_window_size = window_size;
 
+    struct scale_ratio *rw = &screen->scale.w;
+    struct scale_ratio *rh = &screen->scale.h;
+    SDL_GetWindowSize(screen->window, &rw->window, &rh->window);
+    SDL_GL_GetDrawableSize(screen->window, &rw->drawable, &rh->drawable);
+
     return true;
 }
 
@@ -491,7 +502,8 @@ screen_resize_to_fit(struct screen *screen) {
     struct size optimal_size =
         get_optimal_window_size(screen, screen->content_size);
     SDL_SetWindowSize(screen->window, optimal_size.width, optimal_size.height);
-    LOGD("Resized to optimal size");
+    LOGD("Resized to optimal size: %ux%u", optimal_size.width,
+                                           optimal_size.height);
 }
 
 void
@@ -507,14 +519,78 @@ screen_resize_to_pixel_perfect(struct screen *screen) {
 
     struct size content_size = screen->content_size;
     SDL_SetWindowSize(screen->window, content_size.width, content_size.height);
-    LOGD("Resized to pixel-perfect");
+    LOGD("Resized to pixel-perfect: %ux%u", content_size.width,
+                                            content_size.height);
 }
 
-void
+bool
+screen_fix_hidpi(struct screen *screen) {
+    // If the scale ratio has changed, recreate the renderer to fix HiDPI issues
+    // <https://github.com/Genymobile/scrcpy/issues/15>
+
+    int ww, wh, dw, dh;
+    SDL_GetWindowSize(screen->window, &ww, &wh);
+    SDL_GL_GetDrawableSize(screen->window, &dw, &dh);
+
+    struct scale_ratio *rw = &screen->scale.w;
+    struct scale_ratio *rh = &screen->scale.h;
+
+    if (ww * rw->drawable == dw * rw->window &&
+        wh * rh->drawable == dh * rh->window) {
+        // same ratio, both horizontally and vertically, nothing to do
+        return true;
+    }
+
+    // update the current scale
+    rw->window = ww;
+    rh->window = wh;
+    rw->drawable = dw;
+    rh->drawable = dh;
+
+    if (screen->texture) {
+        SDL_DestroyTexture(screen->texture);
+    }
+    if (screen->renderer) {
+        SDL_DestroyRenderer(screen->renderer);
+    }
+
+    screen->renderer = SDL_CreateRenderer(screen->window, -1,
+                                          SDL_RENDERER_ACCELERATED);
+    if (!screen->renderer) {
+        LOGC("Could not create renderer: %s", SDL_GetError());
+        return false;
+    }
+
+    struct size content_size = screen->content_size;
+    if (SDL_RenderSetLogicalSize(screen->renderer, content_size.width,
+                                 content_size.height)) {
+        LOGE("Could not set renderer logical size: %s", SDL_GetError());
+        SDL_DestroyRenderer(screen->renderer);
+        return false;
+    }
+
+    // FIXME this is wrong, we must update the last frame to the new texture,
+    // but we don't have it anymore!
+    screen->texture = create_texture(screen);
+    if (!screen->texture) {
+        LOGC("Could not create texture: %s", SDL_GetError());
+        SDL_DestroyRenderer(screen->renderer);
+        return false;
+    }
+
+    LOGD("Renderer and texture reset");
+    return true;
+}
+
+bool
 screen_handle_window_event(struct screen *screen,
                            const SDL_WindowEvent *event) {
     switch (event->event) {
         case SDL_WINDOWEVENT_EXPOSED:
+            if (!screen_fix_hidpi(screen)) {
+                // unrecoverable
+                return false;
+            }
             screen_render(screen);
             break;
         case SDL_WINDOWEVENT_SIZE_CHANGED:
@@ -550,4 +626,36 @@ screen_handle_window_event(struct screen *screen,
             apply_windowed_size(screen);
             break;
     }
+
+    return true;
+}
+
+struct point
+screen_convert_to_frame_coords(struct screen *screen, int32_t x, int32_t y) {
+    unsigned rotation = screen->rotation;
+    assert(rotation < 4);
+
+    int32_t w = screen->content_size.width;
+    int32_t h = screen->content_size.height;
+    struct point result;
+    switch (rotation) {
+        case 0:
+            result.x = x;
+            result.y = y;
+            break;
+        case 1:
+            result.x = h - y;
+            result.y = x;
+            break;
+        case 2:
+            result.x = w - x;
+            result.y = h - y;
+            break;
+        default:
+            assert(rotation == 3);
+            result.x = y;
+            result.y = w - x;
+            break;
+    }
+    return result;
 }

app/src/screen.h

@@ -13,6 +13,11 @@
 
 struct video_buffer;
 
+struct scale_ratio {
+    int window;
+    int drawable;
+};
+
 struct screen {
     SDL_Window *window;
     SDL_Renderer *renderer;
@@ -33,6 +38,11 @@ struct screen {
     bool maximized;
     bool no_window;
     bool mipmaps;
+
+    struct {
+        struct scale_ratio w;
+        struct scale_ratio h;
+    } scale;
 };
 
 #define SCREEN_INITIALIZER { \
@@ -63,6 +73,16 @@ struct screen {
     .maximized = false, \
     .no_window = false, \
     .mipmaps = false, \
+    .scale = { \
+        .w = { \
+            .window = 0, \
+            .drawable = 0, \
+        }, \
+        .h = { \
+            .window = 0, \
+            .drawable = 0, \
+        }, \
+    } \
 }
 
 // initialize default values
@@ -111,7 +131,13 @@ void
 screen_set_rotation(struct screen *screen, unsigned rotation);
 
 // react to window events
-void
+// return true on success, false on unrecoverable error
+bool
 screen_handle_window_event(struct screen *screen, const SDL_WindowEvent *event);
 
+// convert point from window coordinates to frame coordinates
+// x and y are expressed in pixels
+struct point
+screen_convert_to_frame_coords(struct screen *screen, int32_t x, int32_t y);
+
 #endif

app/tests/test_control_msg_serialize.c

@@ -49,20 +49,20 @@ static void test_serialize_inject_text(void) {
 static void test_serialize_inject_text_long(void) {
     struct control_msg msg;
     msg.type = CONTROL_MSG_TYPE_INJECT_TEXT;
-    char text[CONTROL_MSG_TEXT_MAX_LENGTH + 1];
+    char text[CONTROL_MSG_INJECT_TEXT_MAX_LENGTH + 1];
     memset(text, 'a', sizeof(text));
-    text[CONTROL_MSG_TEXT_MAX_LENGTH] = '\0';
+    text[CONTROL_MSG_INJECT_TEXT_MAX_LENGTH] = '\0';
     msg.inject_text.text = text;
 
     unsigned char buf[CONTROL_MSG_SERIALIZED_MAX_SIZE];
     int size = control_msg_serialize(&msg, buf);
-    assert(size == 3 + CONTROL_MSG_TEXT_MAX_LENGTH);
+    assert(size == 3 + CONTROL_MSG_INJECT_TEXT_MAX_LENGTH);
 
-    unsigned char expected[3 + CONTROL_MSG_TEXT_MAX_LENGTH];
+    unsigned char expected[3 + CONTROL_MSG_INJECT_TEXT_MAX_LENGTH];
     expected[0] = CONTROL_MSG_TYPE_INJECT_TEXT;
     expected[1] = 0x01;
     expected[2] = 0x2c; // text length (16 bits)
-    memset(&expected[3], 'a', CONTROL_MSG_TEXT_MAX_LENGTH);
+    memset(&expected[3], 'a', CONTROL_MSG_INJECT_TEXT_MAX_LENGTH);
 
     assert(!memcmp(buf, expected, sizeof(expected)));
 }

meson.build

@@ -1,6 +1,6 @@
 project('scrcpy', 'c',
         version: '1.12.1',
-        meson_version: '>= 0.37',
+        meson_version: '>= 0.48',
         default_options: [
             'c_std=c11',
             'warning_level=2',

meson_options.txt

@@ -4,6 +4,5 @@ option('crossbuild_windows', type: 'boolean', value: false, description: 'Build
 option('windows_noconsole', type: 'boolean', value: false, description: 'Disable console on Windows (pass -mwindows flag)')
 option('prebuilt_server', type: 'string', description: 'Path of the prebuilt server')
 option('portable', type: 'boolean', value: false, description: 'Use scrcpy-server from the same directory as the scrcpy executable')
-option('hidpi_support', type: 'boolean', value: true, description: 'Enable High DPI support')
 option('server_debugger', type: 'boolean', value: false, description: 'Run a server debugger and wait for a client to be attached')
 option('server_debugger_method', type: 'combo', choices: ['old', 'new'], value: 'new', description: 'Select the debugger method (Android < 9: "old", Android >= 9: "new")')

server/meson.build

@@ -3,7 +3,9 @@
 prebuilt_server = get_option('prebuilt_server')
 if prebuilt_server == ''
     custom_target('scrcpy-server',
-                  build_always: true,  # gradle is responsible for tracking source changes
+                  # gradle is responsible for tracking source changes
+                  build_by_default: true,
+                  build_always_stale: true,
                   output: 'scrcpy-server',
                   command: [find_program('./scripts/build-wrapper.sh'), meson.current_source_dir(), '@OUTPUT@', get_option('buildtype')],
                   console: true,

server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java

@@ -13,8 +13,8 @@ public class ControlMessageReader {
     static final int INJECT_SCROLL_EVENT_PAYLOAD_LENGTH = 20;
     static final int SET_SCREEN_POWER_MODE_PAYLOAD_LENGTH = 1;
 
-    public static final int TEXT_MAX_LENGTH = 300;
     public static final int CLIPBOARD_TEXT_MAX_LENGTH = 4093;
+    public static final int INJECT_TEXT_MAX_LENGTH = 300;
     private static final int RAW_BUFFER_SIZE = 1024;
 
     private final byte[] rawBuffer = new byte[RAW_BUFFER_SIZE];

server/src/main/java/com/genymobile/scrcpy/Device.java

@@ -36,10 +36,7 @@ public final class Device {
      */
     private final int layerStack;
 
-    /**
-     * The FLAG_PRESENTATION from the DisplayInfo
-     */
-    private final boolean isPresentationDisplay;
+    private final boolean supportsInputEvents;
 
     public Device(Options options) {
         displayId = options.getDisplayId();
@@ -53,7 +50,6 @@ public final class Device {
 
         screenInfo = ScreenInfo.computeScreenInfo(displayInfo, options.getCrop(), options.getMaxSize(), options.getLockedVideoOrientation());
         layerStack = displayInfo.getLayerStack();
-        isPresentationDisplay = (displayInfoFlags & DisplayInfo.FLAG_PRESENTATION) != 0;
 
         registerRotationWatcher(new IRotationWatcher.Stub() {
             @Override
@@ -73,8 +69,10 @@ public final class Device {
             Ln.w("Display doesn't have FLAG_SUPPORTS_PROTECTED_BUFFERS flag, mirroring can be restricted");
         }
 
-        if (!supportsInputEvents()) {
-            Ln.w("Input events are not supported for displays with FLAG_PRESENTATION enabled for devices with API lower than 29");
+        // main display or any display on Android >= Q
+        supportsInputEvents = displayId == 0 || Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q;
+        if (!supportsInputEvents) {
+            Ln.w("Input events are not supported for secondary displays before Android 10");
         }
     }
 
@@ -116,10 +114,7 @@ public final class Device {
     }
 
     public boolean supportsInputEvents() {
-        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
-            return true;
-        }
-        return !isPresentationDisplay;
+        return supportsInputEvents;
     }
 
     public boolean injectInputEvent(InputEvent inputEvent, int mode) {

server/src/main/java/com/genymobile/scrcpy/DisplayInfo.java

@@ -8,7 +8,6 @@ public final class DisplayInfo {
     private final int flags;
 
     public static final int FLAG_SUPPORTS_PROTECTED_BUFFERS = 0x00000001;
-    public static final int FLAG_PRESENTATION = 0x00000008;
 
     public DisplayInfo(int displayId, Size size, int rotation, int layerStack, int flags) {
         this.displayId = displayId;

server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java

@@ -136,12 +136,12 @@ public class ScreenEncoder implements Device.RotationListener {
     }
 
     private static MediaCodec createCodec() throws IOException {
-        return MediaCodec.createEncoderByType("video/avc");
+        return MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC);
     }
 
     private static MediaFormat createFormat(int bitRate, int maxFps, int iFrameInterval) {
         MediaFormat format = new MediaFormat();
-        format.setString(MediaFormat.KEY_MIME, "video/avc");
+        format.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_VIDEO_AVC);
         format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);
         // must be present to configure the encoder, but does not impact the actual frame rate, which is variable
         format.setInteger(MediaFormat.KEY_FRAME_RATE, 60);

server/src/main/java/com/genymobile/scrcpy/wrappers/InputManager.java

@@ -53,8 +53,7 @@ public final class InputManager {
             method.invoke(inputEvent, displayId);
             return true;
         } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) {
-            // just a warning, it might happen on old devices
-            Ln.w("Cannot associate a display id to the input event");
+            Ln.e("Cannot associate a display id to the input event", e);
             return false;
         }
     }

server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java

@@ -66,7 +66,7 @@ public class ControlMessageReaderTest {
         ByteArrayOutputStream bos = new ByteArrayOutputStream();
         DataOutputStream dos = new DataOutputStream(bos);
         dos.writeByte(ControlMessage.TYPE_INJECT_TEXT);
-        byte[] text = new byte[ControlMessageReader.TEXT_MAX_LENGTH];
+        byte[] text = new byte[ControlMessageReader.INJECT_TEXT_MAX_LENGTH];
         Arrays.fill(text, (byte) 'a');
         dos.writeShort(text.length);
         dos.write(text);