From bd4b51c85c791101e9a5a783f64f789171b85ac6 Mon Sep 17 00:00:00 2001 From: Kenta420 Date: Tue, 23 Jan 2024 14:39:19 +0700 Subject: [PATCH] now screen touch --- client-electron/src/hooks/adbStore.ts | 26 +++++ client-electron/src/pages/Android.tsx | 131 ++++++++++++++++++++++++-- 2 files changed, 151 insertions(+), 6 deletions(-) create mode 100644 client-electron/src/hooks/adbStore.ts diff --git a/client-electron/src/hooks/adbStore.ts b/client-electron/src/hooks/adbStore.ts new file mode 100644 index 0000000..b516708 --- /dev/null +++ b/client-electron/src/hooks/adbStore.ts @@ -0,0 +1,26 @@ +import { create } from 'zustand' +import { createJSONStorage, persist } from 'zustand/middleware' + +interface AdbStore { + deviceSerial: string + isAlreadyConnected: boolean + setDeviceSerial: (deviceSerial: string) => void + setIsAlreadyConnected: (isAlreadyConnected: boolean) => void +} + +const useAdbStore = create( + persist( + set => ({ + deviceSerial: '', + isAlreadyConnected: false, + setDeviceSerial: deviceSerial => set({ deviceSerial }), + setIsAlreadyConnected: isAlreadyConnected => set({ isAlreadyConnected }) + }), + { + name: 'adb', + storage: createJSONStorage(() => sessionStorage) + } + ) +) + +export default useAdbStore diff --git a/client-electron/src/pages/Android.tsx b/client-electron/src/pages/Android.tsx index c641641..095608a 100644 --- a/client-electron/src/pages/Android.tsx +++ b/client-electron/src/pages/Android.tsx @@ -7,11 +7,13 @@ import type { AdbScrcpyVideoStream } from '@yume-chan/adb-scrcpy' import { AdbScrcpyClient, AdbScrcpyOptions1_22 } from '@yume-chan/adb-scrcpy' import { WebCodecsDecoder } from '@yume-chan/scrcpy-decoder-webcodecs' import { + AndroidMotionEventAction, ScrcpyLogLevel1_18, ScrcpyOptions1_25, + ScrcpyPointerId, ScrcpyVideoCodecId } from '@yume-chan/scrcpy' -import { useCallback, useRef, useState } from 'react' +import { useCallback, useEffect, useRef, useState } from 'react' import { ReadableStream, WritableStream, @@ -19,6 +21,7 @@ import { DecodeUtf8Stream } from '@yume-chan/stream-extra' import { useBeforeUnload } from 'react-router-dom' +import useAdbStore from '../hooks/adbStore' const AndroidPage: React.FC = () => { const { adb, manager, device, setAdb, setDevice } = useAdb( @@ -31,6 +34,60 @@ const AndroidPage: React.FC = () => { })) ) + const { + isAlreadyConnected, + deviceSerial, + setIsAlreadyConnected, + setDeviceSerial + } = useAdbStore( + useShallow(state => ({ + isAlreadyConnected: state.isAlreadyConnected, + setIsAlreadyConnected: state.setIsAlreadyConnected, + deviceSerial: state.deviceSerial, + setDeviceSerial: state.setDeviceSerial + })) + ) + + useEffect(() => { + if (isAlreadyConnected) { + const reconnectAdb = async () => { + const device = await manager + ?.getDevices([ + { + ...ADB_DEFAULT_DEVICE_FILTER, + serialNumber: deviceSerial + } + ]) + .then(devices => devices[0]) + + if (!device) { + // clear state + setIsAlreadyConnected(false) + setDeviceSerial('') + return + } + + const connection = await device.connect() + + const credentialStore: AdbWebCredentialStore = + new AdbWebCredentialStore() + + const transport = await AdbDaemonTransport.authenticate({ + serial: deviceSerial, + connection: connection, + credentialStore: credentialStore + }) + + const adb: Adb = new Adb(transport) + + setAdb(adb) + setDevice(device) + } + + reconnectAdb() + } + }, []) + async function createNewConnection() { device?.raw.forget() setDevice(undefined) @@ -61,6 +118,9 @@ const AndroidPage: React.FC = () => { const adb: Adb = new Adb(transport) setAdb(adb) + + setDeviceSerial(selectedDevice.serial) + setIsAlreadyConnected(true) } const [client, setClient] = useState() @@ -118,13 +178,69 @@ const AndroidPage: React.FC = () => { const videoStream: AdbScrcpyVideoStream | undefined = await _client?.videoStream - const _decoder = new WebCodecsDecoder(ScrcpyVideoCodecId.H264) - _decoder.renderer.style.maxHeight = 'inherit' + if (videoStream) { + const _decoder = new WebCodecsDecoder(ScrcpyVideoCodecId.H264) + + _decoder.renderer.style.maxHeight = 'inherit' + screenRef.current?.appendChild(_decoder.renderer) + videoStream?.stream.pipeTo(_decoder.writable) + setDecoder(_decoder) + + if (_client.controlMessageWriter) { + _decoder.renderer.addEventListener('mousedown', e => { + const react = _decoder.renderer.getBoundingClientRect() + const x = e.clientX - react.left + const y = e.clientY - react.top + _client.controlMessageWriter?.injectTouch({ + action: AndroidMotionEventAction.Down, + pointerId: ScrcpyPointerId.Mouse, + pointerX: x, + pointerY: y, + pressure: 1, + screenWidth: _decoder.renderer.clientWidth, + screenHeight: _decoder.renderer.clientHeight, + buttons: 0, + actionButton: 0 + }) + }) + + _decoder.renderer.addEventListener('mousemove', e => { + const react = _decoder.renderer.getBoundingClientRect() + const x = e.clientX - react.left + const y = e.clientY - react.top + _client.controlMessageWriter?.injectTouch({ + action: AndroidMotionEventAction.Move, + pointerId: ScrcpyPointerId.Mouse, + pointerX: x, + pointerY: y, + pressure: 1, + screenWidth: _decoder.renderer.clientWidth, + screenHeight: _decoder.renderer.clientHeight, + buttons: 0, + actionButton: 0 + }) + }) + + _decoder.renderer.addEventListener('mouseup', e => { + const react = _decoder.renderer.getBoundingClientRect() + const x = e.clientX - react.left + const y = e.clientY - react.top + _client.controlMessageWriter?.injectTouch({ + action: AndroidMotionEventAction.Up, + pointerId: ScrcpyPointerId.Mouse, + pointerX: x, + pointerY: y, + pressure: 1, + screenWidth: _decoder.renderer.clientWidth, + screenHeight: _decoder.renderer.clientHeight, + buttons: 0, + actionButton: 0 + }) + }) + } + } - screenRef.current?.appendChild(_decoder.renderer) - videoStream?.stream.pipeTo(_decoder.writable) - setDecoder(_decoder) setClient(_client) } @@ -133,6 +249,9 @@ const AndroidPage: React.FC = () => { adb?.close() setClient(undefined) setAdb(undefined) + setIsAlreadyConnected(false) + setDeviceSerial('') + device?.raw.forget() } function scrcpyDisconnect() {