aljaz/scrcpy
1
0
Fork 0
mirror of https://github.com/Genymobile/scrcpy.git synced 2025-12-18 22:14:20 +01:00
Code Issues Packages Projects Releases Wiki Activity

Implement keyboard/mouse control

Browse Source 
To control the device from the computer:
 - retrieve mouse and keyboard SDL events;
 - convert them to Android events;
 - serialize them;
 - send them on the same socket used by the video stream (but in the
 opposite direction);
 - deserialize the events on the Android side;
 - inject them using the InputManager.
This commit is contained in:
Romain Vimont
2017-12-14 11:38:44 +01:00
parent 6605ab8e23
commit cabb102a04
23 changed files with 2999 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

286
app/src/screen.c
View File

@@ -9,6 +9,8 @@
#include <SDL2/SDL_net.h>
#include "command.h"
#include "control.h"
#include "convert.h"
#include "decoder.h"
#include "events.h"
#include "frames.h"
@@ -21,20 +23,40 @@
#define MIN(X,Y) (X) < (Y) ? (X) : (Y)
#define MAX(X,Y) (X) > (Y) ? (X) : (Y)
static struct frames frames;
static struct decoder decoder;
struct size {
Uint16 width;
Uint16 height;
};
static struct frames frames;
static struct decoder decoder;
static struct controller controller;
static SDL_Window *window;
static SDL_Renderer *renderer;
static SDL_Texture *texture;
static struct size frame_size;
static SDL_bool texture_empty = SDL_TRUE;
static SDL_bool fullscreen = SDL_FALSE;
static long timestamp_ms(void) {
struct timeval tv;
gettimeofday(&tv, NULL);
return tv.tv_sec * 1000 + tv.tv_usec / 1000;
}
static void count_frame(void) {
static long ts = 0;
static int nbframes = 0;
long now = timestamp_ms();
++nbframes;
if (now - ts > 1000) {
SDL_LogDebug(SDL_LOG_CATEGORY_APPLICATION, "%d fps", nbframes);
ts = now;
nbframes = 0;
}
}
static TCPsocket listen_on_port(Uint16 port) {
IPaddress addr = {
.host = INADDR_ANY,
@@ -214,6 +236,155 @@ static int wait_for_success(process_t proc, const char *name) {
return 0;
}
static SDL_bool handle_new_frame(void) {
mutex_lock(frames.mutex);
AVFrame *frame = frames.rendering_frame;
frames.rendering_frame_consumed = SDL_TRUE;
if (!decoder.skip_frames) {
cond_signal(frames.rendering_frame_consumed_cond);
}
struct size current_frame_size = {frame->width, frame->height};
if (!prepare_for_frame(window, renderer, &texture, frame_size, current_frame_size)) {
return SDL_FALSE;
}
frame_size = current_frame_size;
update_texture(frame, texture);
mutex_unlock(frames.mutex);
render(renderer, texture);
return SDL_TRUE;
}
static void handle_text_input(SDL_TextInputEvent *event) {
struct control_event control_event;
control_event.type = CONTROL_EVENT_TYPE_TEXT;
strncpy(control_event.text_event.text, event->text, TEXT_MAX_LENGTH);
control_event.text_event.text[TEXT_MAX_LENGTH] = '\0';
if (!controller_push_event(&controller, &control_event)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Cannot send text event");
}
}
static void handle_key(SDL_KeyboardEvent *event) {
SDL_Keycode keycode = event->keysym.sym;
SDL_bool ctrl = event->keysym.mod & (KMOD_LCTRL | KMOD_RCTRL);
SDL_bool shift = event->keysym.mod & (KMOD_LSHIFT | KMOD_RSHIFT);
SDL_bool repeat = event->repeat;
// Capture Ctrl+x: optimal size
if (keycode == SDLK_x && !repeat && ctrl && !shift) {
if (event->type == SDL_KEYDOWN) {
struct size optimal_size = get_optimal_window_size(window, frame_size);
SDL_SetWindowSize(window, optimal_size.width, optimal_size.height);
}
return;
}
// Capture Ctrl+f: switch fullscreen
if (keycode == SDLK_f && !repeat && ctrl && !shift) {
if (event->type == SDL_KEYDOWN) {
Uint32 new_mode = fullscreen ? 0 : SDL_WINDOW_FULLSCREEN_DESKTOP;
if (!SDL_SetWindowFullscreen(window, new_mode)) {
fullscreen = !fullscreen;
render(renderer, texture_empty ? NULL : texture);
} else {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Could not switch fullscreen mode: %s", SDL_GetError());
}
}
return;
}
struct control_event control_event;
if (input_key_from_sdl_to_android(event, &control_event)) {
if (!controller_push_event(&controller, &control_event)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Cannot send control event");
}
}
}
static void handle_mouse_motion(SDL_MouseMotionEvent *event) {
struct control_event control_event;
if (mouse_motion_from_sdl_to_android(event, &control_event)) {
if (!controller_push_event(&controller, &control_event)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Cannot send mouse motion event");
}
}
}
static void handle_mouse_button(SDL_MouseButtonEvent *event) {
struct control_event control_event;
if (mouse_button_from_sdl_to_android(event, &control_event)) {
if (!controller_push_event(&controller, &control_event)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Cannot send mouse button event");
}
}
}
static void handle_mouse_wheel(struct complete_mouse_wheel_event *event) {
struct control_event control_event;
if (mouse_wheel_from_sdl_to_android(event, &control_event)) {
if (!controller_push_event(&controller, &control_event)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Cannot send wheel button event");
}
}
}
void event_loop(void) {
SDL_Event event;
while (SDL_WaitEvent(&event)) {
switch (event.type) {
case EVENT_DECODER_STOPPED:
SDL_LogDebug(SDL_LOG_CATEGORY_APPLICATION, "Video decoder stopped");
case SDL_QUIT:
return;
case EVENT_NEW_FRAME:
if (!handle_new_frame()) {
return;
}
texture_empty = SDL_FALSE;
count_frame(); // display fps for debug
break;
case SDL_WINDOWEVENT:
switch (event.window.event) {
case SDL_WINDOWEVENT_EXPOSED:
case SDL_WINDOWEVENT_SIZE_CHANGED:
render(renderer, texture_empty ? NULL : texture);
break;
}
break;
case SDL_TEXTINPUT: {
handle_text_input(&event.text);
break;
}
case SDL_KEYDOWN:
case SDL_KEYUP:
handle_key(&event.key);
break;
case SDL_MOUSEMOTION:
handle_mouse_motion(&event.motion);
break;
case SDL_MOUSEWHEEL: {
struct complete_mouse_wheel_event complete_event;
complete_event.mouse_wheel_event = &event.wheel;
int x;
int y;
SDL_GetMouseState(&x, &y);
complete_event.x = (Sint32) x;
complete_event.y = (Sint32) y;
handle_mouse_wheel(&complete_event);
break;
}
case SDL_MOUSEBUTTONDOWN:
case SDL_MOUSEBUTTONUP:
handle_mouse_button(&event.button);
break;
}
}
}
SDL_bool show_screen(const char *serial, Uint16 local_port) {
SDL_bool ret = 0;
@@ -258,7 +429,6 @@ SDL_bool show_screen(const char *serial, Uint16 local_port) {
goto screen_finally_adb_reverse_remove;
}
struct size frame_size;
char device_name[DEVICE_NAME_FIELD_LENGTH];
// screenrecord does not send frames when the screen content does not change
@@ -292,10 +462,24 @@ SDL_bool show_screen(const char *serial, Uint16 local_port) {
goto screen_finally_destroy_frames;
}
if (!controller_init(&controller, device_socket)) {
ret = SDL_FALSE;
SDLNet_TCP_Close(device_socket);
stop_server(server);
goto screen_finally_stop_decoder;
}
if (!controller_start(&controller)) {
ret = SDL_FALSE;
SDLNet_TCP_Close(device_socket);
stop_server(server);
goto screen_finally_destroy_controller;
}
if (SDL_Init(SDL_INIT_VIDEO)) {
SDL_LogCritical(SDL_LOG_CATEGORY_APPLICATION, "Could not initialize SDL: %s", SDL_GetError());
ret = SDL_FALSE;
goto screen_finally_stop_decoder;
goto screen_finally_stop_and_join_controller;
}
// FIXME it may crash in SDL_Quit in i965_dri.so
// As a workaround, do not call SDL_Quit() (we are exiting anyway).
@@ -307,7 +491,7 @@ SDL_bool show_screen(const char *serial, Uint16 local_port) {
}
struct size window_size = get_initial_optimal_size(frame_size);
SDL_Window *window = SDL_CreateWindow(device_name, SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED,
window = SDL_CreateWindow(device_name, SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED,
window_size.width, window_size.height, SDL_WINDOW_SHOWN | SDL_WINDOW_RESIZABLE);
if (!window) {
SDL_LogCritical(SDL_LOG_CATEGORY_SYSTEM, "Could not create window: %s", SDL_GetError());
@@ -315,7 +499,7 @@ SDL_bool show_screen(const char *serial, Uint16 local_port) {
goto screen_finally_stop_decoder;
}
SDL_Renderer *renderer = SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED);
renderer = SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED);
if (!renderer) {
SDL_LogCritical(SDL_LOG_CATEGORY_RENDER, "Could not create renderer: %s", SDL_GetError());
ret = SDL_FALSE;
@@ -329,7 +513,7 @@ SDL_bool show_screen(const char *serial, Uint16 local_port) {
}
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Initial texture: %" PRIu16 "x%" PRIu16, frame_size.width, frame_size.height);
SDL_Texture *texture = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_YV12, SDL_TEXTUREACCESS_STREAMING, frame_size.width, frame_size.height);
texture = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_YV12, SDL_TEXTUREACCESS_STREAMING, frame_size.width, frame_size.height);
if (!texture) {
SDL_LogCritical(SDL_LOG_CATEGORY_RENDER, "Could not create texture: %s", SDL_GetError());
ret = SDL_FALSE;
@@ -339,87 +523,8 @@ SDL_bool show_screen(const char *serial, Uint16 local_port) {
SDL_RenderClear(renderer);
SDL_RenderPresent(renderer);
long ts = timestamp_ms();
int nbframes = 0;
event_loop();
SDL_bool texture_empty = SDL_TRUE;
SDL_bool fullscreen = SDL_FALSE;
SDL_Event event;
while (SDL_WaitEvent(&event)) {
switch (event.type) {
case EVENT_DECODER_STOPPED:
SDL_LogDebug(SDL_LOG_CATEGORY_APPLICATION, "Video decoder stopped");
case SDL_QUIT:
goto screen_quit;
case EVENT_NEW_FRAME:
mutex_lock(frames.mutex);
AVFrame *frame = frames.rendering_frame;
frames.rendering_frame_consumed = SDL_TRUE;
if (!decoder.skip_frames) {
cond_signal(frames.rendering_frame_consumed_cond);
}
struct size current_frame_size = {frame->width, frame->height};
if (!prepare_for_frame(window, renderer, &texture, frame_size, current_frame_size)) {
goto screen_quit;
}
frame_size = current_frame_size;
update_texture(frame, texture);
mutex_unlock(frames.mutex);
texture_empty = SDL_FALSE;
long now = timestamp_ms();
++nbframes;
if (now - ts > 1000) {
SDL_LogDebug(SDL_LOG_CATEGORY_APPLICATION, "%d fps", nbframes);
ts = now;
nbframes = 0;
}
render(renderer, texture);
break;
case SDL_WINDOWEVENT:
switch (event.window.event) {
case SDL_WINDOWEVENT_EXPOSED:
case SDL_WINDOWEVENT_SIZE_CHANGED:
render(renderer, texture_empty ? NULL : texture);
break;
}
break;
case SDL_KEYDOWN: {
SDL_bool ctrl = SDL_GetModState() & (KMOD_LCTRL | KMOD_RCTRL);
SDL_bool shift = SDL_GetModState() & (KMOD_LSHIFT | KMOD_RSHIFT);
SDL_bool repeat = event.key.repeat;
switch (event.key.keysym.sym) {
case SDLK_x:
if (!repeat && ctrl && !shift) {
// Ctrl+x
struct size optimal_size = get_optimal_window_size(window, frame_size);
SDL_SetWindowSize(window, optimal_size.width, optimal_size.height);
}
break;
case SDLK_f:
if (!repeat && ctrl && !shift) {
// Ctrl+f
Uint32 new_mode = fullscreen ? 0 : SDL_WINDOW_FULLSCREEN_DESKTOP;
if (!SDL_SetWindowFullscreen(window, new_mode)) {
fullscreen = !fullscreen;
render(renderer, texture_empty ? NULL : texture);
} else {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Could not switch fullscreen mode: %s", SDL_GetError());
}
}
break;
}
break;
}
}
}
screen_quit:
SDL_LogDebug(SDL_LOG_CATEGORY_APPLICATION, "quit...");
SDL_DestroyTexture(texture);
screen_finally_destroy_renderer:
@@ -429,6 +534,11 @@ screen_finally_destroy_renderer:
//SDL_DestroyRenderer(renderer);
screen_finally_destroy_window:
//SDL_DestroyWindow(window);
screen_finally_stop_and_join_controller:
controller_stop(&controller);
controller_join(&controller);
screen_finally_destroy_controller:
controller_destroy(&controller);
screen_finally_stop_decoder:
SDLNet_TCP_Close(device_socket);
// kill the server before decoder_join() to wake up the decoder