re-implement scrcpy

client-electron/src/pages/android/android.tsx

@@ -1,9 +1,3 @@
-import { useShallow } from 'zustand/react/shallow'
-import useAdb from '../../hooks/useAdb'
-import { type AdbScrcpyClient } from '@yume-chan/adb-scrcpy'
-import { type WebCodecsDecoder } from '@yume-chan/scrcpy-decoder-webcodecs'
-import { useCallback, useState } from 'react'
-import { useBeforeUnload } from 'react-router-dom'
 import { ToolBar } from './components/tool-bar'
 import { ScrcpyTab } from './components/scrcpy-tab'
 import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'
@@ -11,200 +5,6 @@ import { Toaster } from '@/components/ui/toaster'
 import { ShellTab } from './components/shell-tab'
 
 const AndroidPage: React.FC = () => {
-  const { adb, setAdb } = useAdb(
-    useShallow(state => ({
-      adb: state.adb,
-      manager: state.manager,
-      device: state.device,
-      setAdb: state.setAdb,
-      setDevice: state.setDevice
-    }))
-  )
-
-  const [client, setClient] = useState<AdbScrcpyClient | undefined>()
-  const [decoder, setDecoder] = useState<WebCodecsDecoder | undefined>()
-
-  // when user close or refresh the page, close the adb connection
-  useBeforeUnload(
-    useCallback(() => {
-      decoder?.dispose()
-      client?.close()
-      adb?.close()
-
-      setDecoder(undefined)
-      setClient(undefined)
-      setAdb(undefined)
-    }, [])
-  )
-
-  // async function scrcpyConnect() {
-  //   const server: ArrayBuffer = await fetch(new URL('../scrcpy/scrcpy_server_v1.25', import.meta.url)).then(res =>
-  //     res.arrayBuffer()
-  //   )
-
-  //   await AdbScrcpyClient.pushServer(
-  //     adb!,
-  //     new ReadableStream({
-  //       start(controller) {
-  //         controller.enqueue(new Consumable(new Uint8Array(server)))
-  //         controller.close()
-  //       }
-  //     })
-  //   )
-
-  //   const res = await adb!.subprocess.spawn(
-  //     'CLASSPATH=/data/local/tmp/scrcpy-server.jar app_process / com.genymobile.scrcpy.Server 1.25'
-  //   )
-
-  //   res.stdout.pipeThrough(new DecodeUtf8Stream()).pipeTo(
-  //     new WritableStream({
-  //       write(chunk) {
-  //         console.log(chunk)
-  //       }
-  //     })
-  //   )
-
-  //   const scrcpyOption = new ScrcpyOptions1_25({
-  //     maxFps: 60,
-  //     bitRate: 4000000,
-  //     stayAwake: true,
-  //     control: true,
-  //     logLevel: ScrcpyLogLevel1_18.Debug
-  //   })
-  //   const _client = await AdbScrcpyClient.start(
-  //     adb!,
-  //     '/data/local/tmp/scrcpy-server.jar',
-  //     '1.25',
-  //     new AdbScrcpyOptions1_22(scrcpyOption)
-  //   )
-
-  //   const videoStream: AdbScrcpyVideoStream | undefined = await _client?.videoStream
-
-  //   if (videoStream) {
-  //     const _decoder = new WebCodecsDecoder(ScrcpyVideoCodecId.H264)
-
-  //     _decoder.renderer.style.width = '100%'
-  //     _decoder.renderer.style.height = '100%'
-
-  //     if (screenRef.current && screenRef.current.firstChild) {
-  //       screenRef.current.removeChild(screenRef.current.firstChild)
-  //     }
-  //     screenRef.current?.appendChild(_decoder.renderer)
-  //     videoStream?.stream.pipeTo(_decoder.writable)
-  //     setDecoder(_decoder)
-
-  //     if (_client.controlMessageWriter) {
-  //       _decoder.renderer.addEventListener('mousedown', e => {
-  //         // client width and height 700 x 400
-  //         const react = _decoder.renderer.getBoundingClientRect()
-
-  //         // normalize to _decoder.renderer.width and height 1080 x 1920
-  //         const x = ((e.clientX - react.left) * _decoder.renderer.width) / react.width
-  //         const y = ((e.clientY - react.top) * _decoder.renderer.height) / react.height
-
-  //         //console.log('mouse down at ' + x + ' ' + y)
-  //         _client.controlMessageWriter?.injectTouch({
-  //           action: AndroidMotionEventAction.Down,
-  //           pointerId: ScrcpyPointerId.Mouse | ScrcpyPointerId.Finger,
-  //           pointerX: x,
-  //           pointerY: y,
-  //           pressure: 1,
-  //           screenWidth: _decoder.renderer.width,
-  //           screenHeight: _decoder.renderer.height,
-  //           buttons: 0,
-  //           actionButton: 0
-  //         })
-  //       })
-
-  //       _decoder.renderer.addEventListener('mouseup', e => {
-  //         // client width and height 700 x 400
-  //         const react = _decoder.renderer.getBoundingClientRect()
-
-  //         // normalize to _decoder.renderer.width and height 1080 x 1920
-  //         const x = ((e.clientX - react.left) * _decoder.renderer.width) / react.width
-  //         const y = ((e.clientY - react.top) * _decoder.renderer.height) / react.height
-
-  //         //console.log('mouse up at ' + x + ' ' + y)
-  //         _client.controlMessageWriter?.injectTouch({
-  //           action: AndroidMotionEventAction.Up,
-  //           pointerId: ScrcpyPointerId.Mouse,
-  //           pointerX: x,
-  //           pointerY: y,
-  //           pressure: 1,
-  //           screenWidth: _decoder.renderer.width,
-  //           screenHeight: _decoder.renderer.height,
-  //           buttons: 0,
-  //           actionButton: 0
-  //         })
-  //       })
-
-  //       _decoder.renderer.addEventListener('mousemove', e => {
-  //         // client width and height 700 x 400
-  //         const react = _decoder.renderer.getBoundingClientRect()
-
-  //         // normalize to _decoder.renderer.width and height 1080 x 1920
-  //         const x = ((e.clientX - react.left) * _decoder.renderer.width) / react.width
-  //         const y = ((e.clientY - react.top) * _decoder.renderer.height) / react.height
-
-  //         //console.log('mouse move at ' + x + ' ' + y)
-  //         _client.controlMessageWriter?.injectTouch({
-  //           action: AndroidMotionEventAction.Move,
-  //           pointerId: ScrcpyPointerId.Mouse,
-  //           pointerX: x,
-  //           pointerY: y,
-  //           pressure: 1,
-  //           screenWidth: _decoder.renderer.width,
-  //           screenHeight: _decoder.renderer.height,
-  //           buttons: 0,
-  //           actionButton: 0
-  //         })
-  //       })
-  //     }
-  //   }
-
-  //   setClient(_client)
-  // }
-
-  // function disconnectAdb() {
-  //   client?.close()
-  //   adb?.close()
-  //   setClient(undefined)
-  //   setAdb(undefined)
-  // }
-
-  // function scrcpyDisconnect() {
-  //   decoder?.dispose()
-  //   client?.close()
-  //   if (decoder && screenRef.current) {
-  //     screenRef.current.removeChild(decoder.renderer)
-  //   }
-  //   setClient(undefined)
-  //   setDecoder(undefined)
-  // }
-
-  // async function rebootDevice() {
-  //   const res = await adb?.power.reboot()
-  //   console.log('[rebootDevice] res: ', res)
-  // }
-
-  // function goHome() {
-  //   client?.controlMessageWriter?.injectKeyCode({
-  //     action: AndroidKeyEventAction.Up,
-  //     keyCode: AndroidKeyCode.AndroidHome,
-  //     metaState: 0,
-  //     repeat: 0
-  //   })
-  // }
-
-  // function goBack() {
-  //   client?.controlMessageWriter?.injectKeyCode({
-  //     action: AndroidKeyEventAction.Up,
-  //     keyCode: AndroidKeyCode.AndroidBack,
-  //     metaState: 0,
-  //     repeat: 0
-  //   })
-  // }
-
   return (
     <div className="flex flex-col w-full">
       <div className="flex w-full p-5">

client-electron/src/pages/android/components/scrcpy-tab.tsx

@@ -1,9 +1,21 @@
 import { Button } from '@/components/ui/button'
 import { Card, CardHeader, CardTitle, CardDescription, CardContent, CardFooter } from '@/components/ui/card'
+import { toast } from '@/components/ui/use-toast'
 import useAdb from '@/hooks/useAdb'
 import { type AdbSubprocessProtocol } from '@yume-chan/adb'
-import { WritableStream } from '@yume-chan/stream-extra'
+import { AdbScrcpyClient, AdbScrcpyOptions1_22 } from '@yume-chan/adb-scrcpy'
-import { useEffect, useRef, useState } from 'react'
+import {
+  AndroidKeyCode,
+  AndroidKeyEventAction,
+  AndroidMotionEventAction,
+  ScrcpyLogLevel1_18,
+  ScrcpyOptions1_25,
+  ScrcpyPointerId,
+  ScrcpyVideoCodecId
+} from '@yume-chan/scrcpy'
+import { WebCodecsDecoder } from '@yume-chan/scrcpy-decoder-webcodecs'
+import { Consumable, WritableStream, ReadableStream, DecodeUtf8Stream } from '@yume-chan/stream-extra'
+import { useCallback, useEffect, useRef, useState } from 'react'
 import { Terminal } from 'xterm'
 import { FitAddon } from 'xterm-addon-fit'
 
@@ -13,8 +25,11 @@ export const ScrcpyTab: React.FC = () => {
   const adb = useAdb(state => state.adb)
 
   const logcatRef = useRef<HTMLDivElement>(null)
+  const scrcpyScreenRef = useRef<HTMLDivElement>(null)
 
   const [process, setProcess] = useState<AdbSubprocessProtocol | undefined>()
+  const [client, setClient] = useState<AdbScrcpyClient | undefined>()
+  const [decoder, setDecoder] = useState<WebCodecsDecoder | undefined>()
 
   useEffect(() => {
     const startTerminal = async () => {
@@ -49,6 +64,7 @@ export const ScrcpyTab: React.FC = () => {
         setProcess(process)
       }
     }
+
     startTerminal()
 
     return () => {
@@ -59,6 +75,194 @@ export const ScrcpyTab: React.FC = () => {
     }
   }, [logcatRef, adb])
 
+  const connectScrcpy = useCallback(async () => {
+    if (!adb) {
+      toast({
+        title: 'No ADB connection',
+        description: 'Please connect to a device first',
+        duration: 3000,
+        variant: 'destructive'
+      })
+      return
+    }
+
+    // clean up the scrcpy screen
+    if (scrcpyScreenRef.current && scrcpyScreenRef.current.children.length > 0) {
+      while (scrcpyScreenRef.current.firstChild) {
+        scrcpyScreenRef.current.removeChild(scrcpyScreenRef.current.firstChild)
+      }
+    }
+
+    // fetch the scrcpy server binary
+    // TODO: should load from real server instead of local file. Fix this later
+    const server: ArrayBuffer = await fetch(new URL('../../../scrcpy/scrcpy_server_v1.25', import.meta.url)).then(res =>
+      res.arrayBuffer()
+    )
+
+    // push the server binary to the device
+    const sync = await adb.sync()
+    try {
+      await sync.write({
+        filename: '/data/local/tmp/scrcpy-server.jar',
+        file: new ReadableStream({
+          start(controller) {
+            controller.enqueue(new Consumable(new Uint8Array(server)))
+            controller.close()
+          }
+        })
+      })
+    } finally {
+      await sync.dispose()
+    }
+
+    // start the scrcpy server
+    const res = await adb.subprocess.spawn(
+      'CLASSPATH=/data/local/tmp/scrcpy-server.jar app_process / com.genymobile.scrcpy.Server 1.25'
+    )
+
+    // pipe the server output to the console
+    res.stdout.pipeThrough(new DecodeUtf8Stream()).pipeTo(
+      new WritableStream({
+        write(chunk) {
+          console.log(chunk)
+        }
+      })
+    )
+
+    const scrcpyOption = new ScrcpyOptions1_25({
+      maxFps: 60,
+      bitRate: 4000000,
+      stayAwake: true,
+      control: true,
+      logLevel: ScrcpyLogLevel1_18.Debug
+    })
+
+    // start the scrcpy client
+    const _client = await AdbScrcpyClient.start(
+      adb,
+      '/data/local/tmp/scrcpy-server.jar',
+      '1.25',
+      new AdbScrcpyOptions1_22(scrcpyOption)
+    )
+
+    // get the video stream
+    const videoStream = await _client?.videoStream
+
+    // create a decoder
+    const _decoder = new WebCodecsDecoder(ScrcpyVideoCodecId.H264)
+    scrcpyScreenRef.current?.appendChild(_decoder.renderer)
+    _decoder.renderer.style.width = '100%'
+    _decoder.renderer.style.height = '100%'
+
+    // pipe the video stream to the decoder
+    videoStream?.stream.pipeTo(_decoder.writable)
+
+    // if client has controlMessageWriter, Inject mouse and button events
+    if (_client.controlMessageWriter) {
+      _decoder.renderer.addEventListener('mousedown', e => {
+        // client width and height 450 x 800
+        const react = _decoder.renderer.getBoundingClientRect()
+
+        // normalize to _decoder.renderer.width and height 1080 x 1920
+        const x = ((e.clientX - react.left) * _decoder.renderer.width) / react.width
+        const y = ((e.clientY - react.top) * _decoder.renderer.height) / react.height
+
+        //console.log('mouse down at ' + x + ' ' + y)
+        _client.controlMessageWriter?.injectTouch({
+          action: AndroidMotionEventAction.Down,
+          pointerId: ScrcpyPointerId.Mouse | ScrcpyPointerId.Finger,
+          pointerX: x,
+          pointerY: y,
+          pressure: 1,
+          screenWidth: _decoder.renderer.width,
+          screenHeight: _decoder.renderer.height,
+          buttons: 0,
+          actionButton: 0
+        })
+      })
+
+      _decoder.renderer.addEventListener('mouseup', e => {
+        // client width and height 450 x 800
+        const react = _decoder.renderer.getBoundingClientRect()
+
+        // normalize to _decoder.renderer.width and height 1080 x 1920
+        const x = ((e.clientX - react.left) * _decoder.renderer.width) / react.width
+        const y = ((e.clientY - react.top) * _decoder.renderer.height) / react.height
+
+        //console.log('mouse up at ' + x + ' ' + y)
+        _client.controlMessageWriter?.injectTouch({
+          action: AndroidMotionEventAction.Up,
+          pointerId: ScrcpyPointerId.Mouse,
+          pointerX: x,
+          pointerY: y,
+          pressure: 1,
+          screenWidth: _decoder.renderer.width,
+          screenHeight: _decoder.renderer.height,
+          buttons: 0,
+          actionButton: 0
+        })
+      })
+
+      _decoder.renderer.addEventListener('mousemove', e => {
+        // client width and height 450 x 800
+        const react = _decoder.renderer.getBoundingClientRect()
+
+        // normalize to _decoder.renderer.width and height 1080 x 1920
+        const x = ((e.clientX - react.left) * _decoder.renderer.width) / react.width
+        const y = ((e.clientY - react.top) * _decoder.renderer.height) / react.height
+
+        //console.log('mouse move at ' + x + ' ' + y)
+        _client.controlMessageWriter?.injectTouch({
+          action: AndroidMotionEventAction.Move,
+          pointerId: ScrcpyPointerId.Mouse,
+          pointerX: x,
+          pointerY: y,
+          pressure: 1,
+          screenWidth: _decoder.renderer.width,
+          screenHeight: _decoder.renderer.height,
+          buttons: 0,
+          actionButton: 0
+        })
+      })
+    }
+
+    setDecoder(_decoder)
+    setClient(_client)
+  }, [adb, scrcpyScreenRef])
+
+  function onHomeClickHandler() {
+    client?.controlMessageWriter?.injectKeyCode({
+      action: AndroidKeyEventAction.Up,
+      keyCode: AndroidKeyCode.AndroidHome,
+      metaState: 0,
+      repeat: 0
+    })
+  }
+
+  function onBackClickHandler() {
+    client?.controlMessageWriter?.injectKeyCode({
+      action: AndroidKeyEventAction.Up,
+      keyCode: AndroidKeyCode.AndroidBack,
+      metaState: 0,
+      repeat: 0
+    })
+  }
+
+  function disconnectScrcpy() {
+    // clean ref
+    if (scrcpyScreenRef.current && scrcpyScreenRef.current.children.length > 0) {
+      while (scrcpyScreenRef.current.firstChild) {
+        scrcpyScreenRef.current.removeChild(scrcpyScreenRef.current.firstChild)
+      }
+    }
+
+    decoder?.dispose()
+    client?.close()
+
+    setClient(undefined)
+    setDecoder(undefined)
+  }
+
   return (
     <Card>
       <CardHeader>
@@ -67,12 +271,12 @@ export const ScrcpyTab: React.FC = () => {
       </CardHeader>
       <CardContent className="flex w-full justify-around items-start">
         <div>
-          <div className="w-[450px] h-[800px] bg-slate-700" />
+          <div className="w-[450px] max-w-[450px] h-[800px] max-h-[800px] bg-slate-700" ref={scrcpyScreenRef} />
           <div className="flex pt-3 justify-center items-center space-x-4 w-[450px]">
-            <Button variant={'outline'} className="flex-1">
+            <Button onClick={onHomeClickHandler} variant={'outline'} className="flex-1">
               Home
             </Button>
-            <Button variant={'outline'} className="flex-1">
+            <Button onClick={onBackClickHandler} variant={'outline'} className="flex-1">
               Back
             </Button>
           </div>
@@ -84,10 +288,15 @@ export const ScrcpyTab: React.FC = () => {
             </CardHeader>
             <CardContent>
               <div className="flex space-x-4 items-center">
-                <Button>Connect</Button>
+                {client ? (
-                <Button>Power</Button>
+                  <Button onClick={disconnectScrcpy} variant="destructive">
-                <Button>Volume Up</Button>
+                    Disconnect
-                <Button>Volume Down</Button>
+                  </Button>
+                ) : (
+                  <Button onClick={connectScrcpy} variant="default">
+                    Connect
+                  </Button>
+                )}
               </div>
             </CardContent>
           </Card>