From ac0f5bbeea2b09d73be6ee89d7f4145f73ee299f Mon Sep 17 00:00:00 2001 From: Kenta420 Date: Mon, 5 Feb 2024 17:10:34 +0700 Subject: [PATCH] re-implement scrcpy --- client-electron/src/pages/android/android.tsx | 200 --------------- .../pages/android/components/scrcpy-tab.tsx | 227 +++++++++++++++++- 2 files changed, 218 insertions(+), 209 deletions(-) diff --git a/client-electron/src/pages/android/android.tsx b/client-electron/src/pages/android/android.tsx index 912ec39..45ef8bd 100644 --- a/client-electron/src/pages/android/android.tsx +++ b/client-electron/src/pages/android/android.tsx @@ -1,9 +1,3 @@ -import { useShallow } from 'zustand/react/shallow' -import useAdb from '../../hooks/useAdb' -import { type AdbScrcpyClient } from '@yume-chan/adb-scrcpy' -import { type WebCodecsDecoder } from '@yume-chan/scrcpy-decoder-webcodecs' -import { useCallback, useState } from 'react' -import { useBeforeUnload } from 'react-router-dom' import { ToolBar } from './components/tool-bar' import { ScrcpyTab } from './components/scrcpy-tab' import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs' @@ -11,200 +5,6 @@ import { Toaster } from '@/components/ui/toaster' import { ShellTab } from './components/shell-tab' const AndroidPage: React.FC = () => { - const { adb, setAdb } = useAdb( - useShallow(state => ({ - adb: state.adb, - manager: state.manager, - device: state.device, - setAdb: state.setAdb, - setDevice: state.setDevice - })) - ) - - const [client, setClient] = useState() - const [decoder, setDecoder] = useState() - - // when user close or refresh the page, close the adb connection - useBeforeUnload( - useCallback(() => { - decoder?.dispose() - client?.close() - adb?.close() - - setDecoder(undefined) - setClient(undefined) - setAdb(undefined) - }, []) - ) - - // async function scrcpyConnect() { - // const server: ArrayBuffer = await fetch(new URL('../scrcpy/scrcpy_server_v1.25', import.meta.url)).then(res => - // res.arrayBuffer() - // ) - - // await AdbScrcpyClient.pushServer( - // adb!, - // new ReadableStream({ - // start(controller) { - // controller.enqueue(new Consumable(new Uint8Array(server))) - // controller.close() - // } - // }) - // ) - - // const res = await adb!.subprocess.spawn( - // 'CLASSPATH=/data/local/tmp/scrcpy-server.jar app_process / com.genymobile.scrcpy.Server 1.25' - // ) - - // res.stdout.pipeThrough(new DecodeUtf8Stream()).pipeTo( - // new WritableStream({ - // write(chunk) { - // console.log(chunk) - // } - // }) - // ) - - // const scrcpyOption = new ScrcpyOptions1_25({ - // maxFps: 60, - // bitRate: 4000000, - // stayAwake: true, - // control: true, - // logLevel: ScrcpyLogLevel1_18.Debug - // }) - // const _client = await AdbScrcpyClient.start( - // adb!, - // '/data/local/tmp/scrcpy-server.jar', - // '1.25', - // new AdbScrcpyOptions1_22(scrcpyOption) - // ) - - // const videoStream: AdbScrcpyVideoStream | undefined = await _client?.videoStream - - // if (videoStream) { - // const _decoder = new WebCodecsDecoder(ScrcpyVideoCodecId.H264) - - // _decoder.renderer.style.width = '100%' - // _decoder.renderer.style.height = '100%' - - // if (screenRef.current && screenRef.current.firstChild) { - // screenRef.current.removeChild(screenRef.current.firstChild) - // } - // screenRef.current?.appendChild(_decoder.renderer) - // videoStream?.stream.pipeTo(_decoder.writable) - // setDecoder(_decoder) - - // if (_client.controlMessageWriter) { - // _decoder.renderer.addEventListener('mousedown', e => { - // // client width and height 700 x 400 - // const react = _decoder.renderer.getBoundingClientRect() - - // // normalize to _decoder.renderer.width and height 1080 x 1920 - // const x = ((e.clientX - react.left) * _decoder.renderer.width) / react.width - // const y = ((e.clientY - react.top) * _decoder.renderer.height) / react.height - - // //console.log('mouse down at ' + x + ' ' + y) - // _client.controlMessageWriter?.injectTouch({ - // action: AndroidMotionEventAction.Down, - // pointerId: ScrcpyPointerId.Mouse | ScrcpyPointerId.Finger, - // pointerX: x, - // pointerY: y, - // pressure: 1, - // screenWidth: _decoder.renderer.width, - // screenHeight: _decoder.renderer.height, - // buttons: 0, - // actionButton: 0 - // }) - // }) - - // _decoder.renderer.addEventListener('mouseup', e => { - // // client width and height 700 x 400 - // const react = _decoder.renderer.getBoundingClientRect() - - // // normalize to _decoder.renderer.width and height 1080 x 1920 - // const x = ((e.clientX - react.left) * _decoder.renderer.width) / react.width - // const y = ((e.clientY - react.top) * _decoder.renderer.height) / react.height - - // //console.log('mouse up at ' + x + ' ' + y) - // _client.controlMessageWriter?.injectTouch({ - // action: AndroidMotionEventAction.Up, - // pointerId: ScrcpyPointerId.Mouse, - // pointerX: x, - // pointerY: y, - // pressure: 1, - // screenWidth: _decoder.renderer.width, - // screenHeight: _decoder.renderer.height, - // buttons: 0, - // actionButton: 0 - // }) - // }) - - // _decoder.renderer.addEventListener('mousemove', e => { - // // client width and height 700 x 400 - // const react = _decoder.renderer.getBoundingClientRect() - - // // normalize to _decoder.renderer.width and height 1080 x 1920 - // const x = ((e.clientX - react.left) * _decoder.renderer.width) / react.width - // const y = ((e.clientY - react.top) * _decoder.renderer.height) / react.height - - // //console.log('mouse move at ' + x + ' ' + y) - // _client.controlMessageWriter?.injectTouch({ - // action: AndroidMotionEventAction.Move, - // pointerId: ScrcpyPointerId.Mouse, - // pointerX: x, - // pointerY: y, - // pressure: 1, - // screenWidth: _decoder.renderer.width, - // screenHeight: _decoder.renderer.height, - // buttons: 0, - // actionButton: 0 - // }) - // }) - // } - // } - - // setClient(_client) - // } - - // function disconnectAdb() { - // client?.close() - // adb?.close() - // setClient(undefined) - // setAdb(undefined) - // } - - // function scrcpyDisconnect() { - // decoder?.dispose() - // client?.close() - // if (decoder && screenRef.current) { - // screenRef.current.removeChild(decoder.renderer) - // } - // setClient(undefined) - // setDecoder(undefined) - // } - - // async function rebootDevice() { - // const res = await adb?.power.reboot() - // console.log('[rebootDevice] res: ', res) - // } - - // function goHome() { - // client?.controlMessageWriter?.injectKeyCode({ - // action: AndroidKeyEventAction.Up, - // keyCode: AndroidKeyCode.AndroidHome, - // metaState: 0, - // repeat: 0 - // }) - // } - - // function goBack() { - // client?.controlMessageWriter?.injectKeyCode({ - // action: AndroidKeyEventAction.Up, - // keyCode: AndroidKeyCode.AndroidBack, - // metaState: 0, - // repeat: 0 - // }) - // } - return (
diff --git a/client-electron/src/pages/android/components/scrcpy-tab.tsx b/client-electron/src/pages/android/components/scrcpy-tab.tsx index 0a71cd8..84ebcfe 100644 --- a/client-electron/src/pages/android/components/scrcpy-tab.tsx +++ b/client-electron/src/pages/android/components/scrcpy-tab.tsx @@ -1,9 +1,21 @@ import { Button } from '@/components/ui/button' import { Card, CardHeader, CardTitle, CardDescription, CardContent, CardFooter } from '@/components/ui/card' +import { toast } from '@/components/ui/use-toast' import useAdb from '@/hooks/useAdb' import { type AdbSubprocessProtocol } from '@yume-chan/adb' -import { WritableStream } from '@yume-chan/stream-extra' -import { useEffect, useRef, useState } from 'react' +import { AdbScrcpyClient, AdbScrcpyOptions1_22 } from '@yume-chan/adb-scrcpy' +import { + AndroidKeyCode, + AndroidKeyEventAction, + AndroidMotionEventAction, + ScrcpyLogLevel1_18, + ScrcpyOptions1_25, + ScrcpyPointerId, + ScrcpyVideoCodecId +} from '@yume-chan/scrcpy' +import { WebCodecsDecoder } from '@yume-chan/scrcpy-decoder-webcodecs' +import { Consumable, WritableStream, ReadableStream, DecodeUtf8Stream } from '@yume-chan/stream-extra' +import { useCallback, useEffect, useRef, useState } from 'react' import { Terminal } from 'xterm' import { FitAddon } from 'xterm-addon-fit' @@ -13,8 +25,11 @@ export const ScrcpyTab: React.FC = () => { const adb = useAdb(state => state.adb) const logcatRef = useRef(null) + const scrcpyScreenRef = useRef(null) const [process, setProcess] = useState() + const [client, setClient] = useState() + const [decoder, setDecoder] = useState() useEffect(() => { const startTerminal = async () => { @@ -49,6 +64,7 @@ export const ScrcpyTab: React.FC = () => { setProcess(process) } } + startTerminal() return () => { @@ -59,6 +75,194 @@ export const ScrcpyTab: React.FC = () => { } }, [logcatRef, adb]) + const connectScrcpy = useCallback(async () => { + if (!adb) { + toast({ + title: 'No ADB connection', + description: 'Please connect to a device first', + duration: 3000, + variant: 'destructive' + }) + return + } + + // clean up the scrcpy screen + if (scrcpyScreenRef.current && scrcpyScreenRef.current.children.length > 0) { + while (scrcpyScreenRef.current.firstChild) { + scrcpyScreenRef.current.removeChild(scrcpyScreenRef.current.firstChild) + } + } + + // fetch the scrcpy server binary + // TODO: should load from real server instead of local file. Fix this later + const server: ArrayBuffer = await fetch(new URL('../../../scrcpy/scrcpy_server_v1.25', import.meta.url)).then(res => + res.arrayBuffer() + ) + + // push the server binary to the device + const sync = await adb.sync() + try { + await sync.write({ + filename: '/data/local/tmp/scrcpy-server.jar', + file: new ReadableStream({ + start(controller) { + controller.enqueue(new Consumable(new Uint8Array(server))) + controller.close() + } + }) + }) + } finally { + await sync.dispose() + } + + // start the scrcpy server + const res = await adb.subprocess.spawn( + 'CLASSPATH=/data/local/tmp/scrcpy-server.jar app_process / com.genymobile.scrcpy.Server 1.25' + ) + + // pipe the server output to the console + res.stdout.pipeThrough(new DecodeUtf8Stream()).pipeTo( + new WritableStream({ + write(chunk) { + console.log(chunk) + } + }) + ) + + const scrcpyOption = new ScrcpyOptions1_25({ + maxFps: 60, + bitRate: 4000000, + stayAwake: true, + control: true, + logLevel: ScrcpyLogLevel1_18.Debug + }) + + // start the scrcpy client + const _client = await AdbScrcpyClient.start( + adb, + '/data/local/tmp/scrcpy-server.jar', + '1.25', + new AdbScrcpyOptions1_22(scrcpyOption) + ) + + // get the video stream + const videoStream = await _client?.videoStream + + // create a decoder + const _decoder = new WebCodecsDecoder(ScrcpyVideoCodecId.H264) + scrcpyScreenRef.current?.appendChild(_decoder.renderer) + _decoder.renderer.style.width = '100%' + _decoder.renderer.style.height = '100%' + + // pipe the video stream to the decoder + videoStream?.stream.pipeTo(_decoder.writable) + + // if client has controlMessageWriter, Inject mouse and button events + if (_client.controlMessageWriter) { + _decoder.renderer.addEventListener('mousedown', e => { + // client width and height 450 x 800 + const react = _decoder.renderer.getBoundingClientRect() + + // normalize to _decoder.renderer.width and height 1080 x 1920 + const x = ((e.clientX - react.left) * _decoder.renderer.width) / react.width + const y = ((e.clientY - react.top) * _decoder.renderer.height) / react.height + + //console.log('mouse down at ' + x + ' ' + y) + _client.controlMessageWriter?.injectTouch({ + action: AndroidMotionEventAction.Down, + pointerId: ScrcpyPointerId.Mouse | ScrcpyPointerId.Finger, + pointerX: x, + pointerY: y, + pressure: 1, + screenWidth: _decoder.renderer.width, + screenHeight: _decoder.renderer.height, + buttons: 0, + actionButton: 0 + }) + }) + + _decoder.renderer.addEventListener('mouseup', e => { + // client width and height 450 x 800 + const react = _decoder.renderer.getBoundingClientRect() + + // normalize to _decoder.renderer.width and height 1080 x 1920 + const x = ((e.clientX - react.left) * _decoder.renderer.width) / react.width + const y = ((e.clientY - react.top) * _decoder.renderer.height) / react.height + + //console.log('mouse up at ' + x + ' ' + y) + _client.controlMessageWriter?.injectTouch({ + action: AndroidMotionEventAction.Up, + pointerId: ScrcpyPointerId.Mouse, + pointerX: x, + pointerY: y, + pressure: 1, + screenWidth: _decoder.renderer.width, + screenHeight: _decoder.renderer.height, + buttons: 0, + actionButton: 0 + }) + }) + + _decoder.renderer.addEventListener('mousemove', e => { + // client width and height 450 x 800 + const react = _decoder.renderer.getBoundingClientRect() + + // normalize to _decoder.renderer.width and height 1080 x 1920 + const x = ((e.clientX - react.left) * _decoder.renderer.width) / react.width + const y = ((e.clientY - react.top) * _decoder.renderer.height) / react.height + + //console.log('mouse move at ' + x + ' ' + y) + _client.controlMessageWriter?.injectTouch({ + action: AndroidMotionEventAction.Move, + pointerId: ScrcpyPointerId.Mouse, + pointerX: x, + pointerY: y, + pressure: 1, + screenWidth: _decoder.renderer.width, + screenHeight: _decoder.renderer.height, + buttons: 0, + actionButton: 0 + }) + }) + } + + setDecoder(_decoder) + setClient(_client) + }, [adb, scrcpyScreenRef]) + + function onHomeClickHandler() { + client?.controlMessageWriter?.injectKeyCode({ + action: AndroidKeyEventAction.Up, + keyCode: AndroidKeyCode.AndroidHome, + metaState: 0, + repeat: 0 + }) + } + + function onBackClickHandler() { + client?.controlMessageWriter?.injectKeyCode({ + action: AndroidKeyEventAction.Up, + keyCode: AndroidKeyCode.AndroidBack, + metaState: 0, + repeat: 0 + }) + } + + function disconnectScrcpy() { + // clean ref + if (scrcpyScreenRef.current && scrcpyScreenRef.current.children.length > 0) { + while (scrcpyScreenRef.current.firstChild) { + scrcpyScreenRef.current.removeChild(scrcpyScreenRef.current.firstChild) + } + } + + decoder?.dispose() + client?.close() + + setClient(undefined) + setDecoder(undefined) + } + return ( @@ -67,12 +271,12 @@ export const ScrcpyTab: React.FC = () => {
-
+
- -
@@ -84,10 +288,15 @@ export const ScrcpyTab: React.FC = () => {
- - - - + {client ? ( + + ) : ( + + )}