| electron桌面流和远程控制 | 您所在的位置：网站首页 › react-desktop › electron桌面流和远程控制 |
electron桌面流和远程控制
|
本文相关文档
desktop-capturer(桌面窗口捕获视频) robotjs 本门起步项目github.com/Licht-club/… 分支主要更新 1. webpack多入口多页面,多个渲染线程可以用同一个webpack配置完成 2. webpack一些基础优化 复制代码 desktop-capturer的使用/render-process/control/getDesSource.ts 这段是获取桌面流的核心逻辑,基本都是上面的desktop-capturer(桌面窗口捕获视频) 文档复制的 修改了两个地方: if (source.name === 'Electron')这个判断先注释了 video:{mandatory:{}} 这边的配置的es.dom的签名不匹配,需要使用@ts-ignore的魔法注释 import {desktopCapturer,} from 'electron' export type handleStream = (stream: MediaStream) => void export type handleError = (err: Error) => void export function getDesSource(handleStream: handleStream, handleError ?: handleError) { desktopCapturer.getSources({types: ['window', 'screen']}).then(async sources => { for (const source of sources) { // if (source.name === 'Electron') { } try { const stream = await navigator.mediaDevices.getUserMedia({ audio: false, video: { // @ts-ignore mandatory: { chromeMediaSource: 'desktop', chromeMediaSourceId: source.id, minWidth: 1280, maxWidth: 1280, minHeight: 720, maxHeight: 720 } } }) handleStream(stream) } catch (e) { handleError && handleError(e) } return } }) } render-process/control/DesktopCapturerVideo.tsx 封装一个video组件,调用上面的getDesSource方法 import {desktopCapturer} from 'electron' import React, {useEffect, useMemo, useRef} from "react"; import {getDesSource} from "./getDesSource"; const DesktopCapturerVideo = () => { const videoRef = useRef(null); const handleStream = (stream: MediaStream) => { console.log(videoRef.current) if (videoRef.current) { const video = videoRef.current video.srcObject = stream video.onloadedmetadata = (e) => video.play() } } const handleError = (error:Error) => { console.log(error,'获取桌面流出错') } useEffect(() => { getDesSource(handleStream, handleError) }, []) return } export default DesktopCapturerVideo 复制代码render-process/control/index.tsx 渲染线程(控制页面)调用DesktopCapturerVideo组件 import ReactDom from "react-dom"; import React, {useEffect} from "react"; import DesktopCapturerVideo from "./DesktopCapturerVideo"; function App(){ return 模拟远程控制台 } ReactDom.render(, document.getElementById('root')) 复制代码启动进程 Mac系统会弹出权限提示,需要配置下 配置后,看到成功捕获桌面流 可以看到,我们目前的项目启动比较麻烦,需要起三个服务,一个react/webpack的渲染服务,一个tsc的实时编译服务,一个electron的窗口服务 这种情况,可以使用这种情况,可以使用concurrently 解决 安装 $ npm install -g concurrently or $ npm install concurrently --save 复制代码我这里就不全局安装了 配置启动命令 "start:dev": "concurrently -k -p \\"[{name}]\\" -n \\"React,TypeScript,Electron\\" -c \\"yellow.bold,cyan.bold,green.bold\\" \\"npm run start:react\\" \\"npm run dev:main\\" \\"npm run start:electron\\"" 复制代码执行start:dev,查看命令行,可以看到React,TypeScript,Electron三个线程都启动了,但是electron窗口是空白的, 这是因为在Electron线程启动成功的时候,React线程还没完成启动,可以在electron窗口刷新解决,或者使用 wait-on wait-on wait-on安装 npm install wait-on # local version OR npm install -g wait-on # global version 复制代码 修改我们的start:dev命令 "start:dev": "concurrently -k -p \"[{name}]\" -n \"React,TypeScript,Electron\" -c \"yellow.bold,cyan.bold,green.bold\" \"npm run start:react\" \"npm run dev:main\" \" wait-on http://localhost:8080/main.html && npm run start:electron\"" 复制代码 启动可以看到,我们的Electron线程成功卡在React后面才启动 远程控制robotjs实现了桌面自动化,可以控制鼠标和屏幕 robotjs的安装 npm install robotjs // 安装需要一段时间 复制代码安装成功后,重启start:dev,在electron窗口中输入require('robotjs'), 会发现出现报错 报错图 报这个错误的原因 :robotjs是基于c++编写的，在不同的平台，不同的node版本环境需要重新编译。 robotjs的编译(build)请确保在对应的平台已经安装对应的依赖 Windows windows-build-tools npm package (npm install --global --production windows-build-tools from an elevated PowerShell or CMD.exe) Mac Xcode Command Line Tools. Linux Python (v2.7 recommended, v3.x.x is not supported). make. A C/C++ compiler like GCC. libxtst-dev and libpng++-dev (sudo apt-get install libxtst-dev libpng++-dev). 复制代码开始编译 $ npm install electron-rebuild --save-dev $ npx electron-rebuild 复制代码编译成功如图! 需要关闭app.allowRendererProcessReuse import {app, BrowserWindow} from 'electron' import {create} from './mainWindow' import {ipc} from "./ipc"; app.allowRendererProcessReuse = false // 默认为true,防止在渲染器进程中加载非上下文感知的本机模块。 app.on('ready', () => { process.env['ELECTRON_DISABLE_SECURITY_WARNINGS'] = 'true'; // 关闭web安全警告 ipc() create() }) 复制代码重启start:dev,再次尝试reqire('rebotjs'),如果可以成功,就说明安装成功了~ robotjs的使用main-process/onRobot.ts robotjs只能在主进程中运行，所以robotjs代码在主进程通过ipc的方法，让渲染进程调用主进程去做软件控制(键盘和鼠标) import {ipcMain} from 'electron'; import robot from 'robotjs' export interface RobotData { keyCode: number; shift: boolean; meta: boolean; alt: boolean; screen?: { width: number; height: number }, video?: { width:number; height:number; } } export type RobotType = 'mouse' | 'key' // 鼠标 键盘 const robotHandle = (function () { function mouseHandle(data: RobotData) { console.log(data, '暂时逻辑--mouseHandle') } function keyHandle(data: RobotData) { console.log(data, '暂时逻辑--keyHandle') } return { mouseHandle, keyHandle } })() export default function onRobot() { ipcMain.on('robot', (e, type: RobotType, data: RobotData) => { if (type === 'mouse') { robotHandle.mouseHandle(data) } else if (type === 'key') { robotHandle.keyHandle(data) } }) } 复制代码main-process/index.ts 主线程入口添加上面封装的逻辑 import {app, BrowserWindow} from 'electron' import {create} from './mainWindow' import {ipc} from "./ipc"; import onRobot from "./onRobot"; app.allowRendererProcessReuse = false app.on('ready', () => { process.env['ELECTRON_DISABLE_SECURITY_WARNINGS'] = 'true'; // 关闭web安全警告 ipc() create() onRobot() }) 复制代码render-process/control/events.ts 使用nodejs内置的events模块(发布订阅模式) import Events from 'events' import {RobotData, RobotType} from "../../main-process/onRobot"; import {ipcRenderer} from "electron"; const peer = new Events() peer.on('robot', (type: RobotType, data: RobotData) => { if (type === 'mouse') { data.screen = { width: window.screen.width, height: window.screen.height } } // ipcRenderer.send 通信主线程 ipcRenderer.send('robot', type, data) }) export default peer; 复制代码render-process/control/setRobot.ts 渲染线程监听鼠标和键盘事件 import {ipcRenderer} from 'electron' import peer from "./events"; const geneRobotKeyData = (e: KeyboardEvent) => { return { keyCode: e.keyCode, shift: e.shiftKey, meta: e.metaKey, alt: e.altKey } } const geneRobotMouseData = (e: MouseEvent) => { return { clientX: e.clientX, clientY: e.clientY, video: { // width: } } } export function setRobot(videoDom: HTMLVideoElement) { window.addEventListener('keydown', (e) => { const data = geneRobotKeyData(e) peer.emit('robot', 'key', data) }) window.addEventListener('mouseup', (e) => { const data = geneRobotMouseData(e) data.video = { width: videoDom.getBoundingClientRect().width, height: videoDom.getBoundingClientRect().height } peer.emit('robot', 'mouse', data) }) } 复制代码render-process/control/DesktopCapturerVideo.tsx 控制台(video)组件引入上面的监听逻辑, useEffect(() => { getDesSource(handleStream, handleError) setRobot(videoRef.current!) }, []) 复制代码重启项目,点击鼠标或者键盘,查看命令行输出 9.打印成功 梳理下这套流程 渲染线程: 页面监听mouseup->event.emit 渲染线程: event.on->ipcRender.send 主线程: ipcMain.on 复制代码 完善主线程的handle逻辑/utils/vkey.ts 键盘映射,出处: github.com/chrisdickin… 又水了100多行QaQ 'use strict' var ua = typeof window !== 'undefined' ? window.navigator.userAgent : '' , isOSX = /OS X/.test(ua) , isOpera = /Opera/.test(ua) , maybeFirefox = !/like Gecko/.test(ua) && !isOpera let i=0; let output :Record = { 0: isOSX ? '' : '' , 1: '' , 2: '' , 3: '' , 4: '' , 5: '' , 6: '' , 8: '' , 9: '' , 12: '' , 13: '' , 16: '' , 17: '' , 18: '' , 19: '' , 20: '' , 21: '' , 23: '' , 24: '' , 25: '' , 27: '' , 28: '' , 29: '' , 30: '' , 31: '' , 32: '' , 33: '' , 34: '' , 35: '' , 36: '' , 37: '' , 38: '' , 39: '' , 40: '' , 41: '' , 42: '' , 43: '' , 44: '' , 45: '' , 46: '' , 47: '' , 91: '' // meta-left -- no one handles left and right properly, so we coerce into one. , 92: '' // meta-right , 93: isOSX ? '' : '' // chrome,opera,safari all report this for meta-right (osx mbp). , 95: '' , 106: '' , 107: '' , 108: '' , 109: '' , 110: '' , 111: '' , 144: '' , 145: '' , 160: '' , 161: '' , 162: '' , 163: '' , 164: '' , 165: '' , 166: '' , 167: '' , 168: '' , 169: '' , 170: '' , 171: '' , 172: '' // ff/osx reports '' for '-' , 173: isOSX && maybeFirefox ? '-' : '' , 174: '' , 175: '' , 176: '' , 177: '' , 178: '' , 179: '' , 180: '' , 181: '' , 182: '' , 183: '' , 186: ';' , 187: '=' , 188: ',' , 189: '-' , 190: '.' , 191: '/' , 192: '\`' , 219: '[' , 220: '\\\\' , 221: ']' , 222: "'" , 223: '' , 224: '' // firefox reports meta here. , 226: '' , 229: '' , 231: isOpera ? '\`' : '' , 246: '' , 247: '' , 248: '' , 249: '' , 250: '' , 251: '' , 252: '' , 253: '' , 254: '' } for(i = 58; i < 65; ++i) { output[i] = String.fromCharCode(i) } // 0-9 for(i = 48; i < 58; ++i) { output[i] = (i - 48)+'' } // A-Z for(i = 65; i < 91; ++i) { output[i] = String.fromCharCode(i) } // num0-9 for(i = 96; i < 106; ++i) { output[i] = '' } // F1-F24 for(i = 112; i < 136; ++i) { output[i] = 'F'+(i-111) } export default output 复制代码 发现一个bug主线程打包出来的文件目录dist/main-process/index.js,但是package.json里面的main字段错误了 正确的应该是"main": "dist/main-process/index.js" 继续main-process/onRobot.ts import {ipcMain} from 'electron'; import robot from 'robotjs' import vkey from "../utils/vkey"; export interface ScreenVideoInfo { screen: { width: number; height: number }, video: { width: number; height: number } } export type RobotKeyData = { keyCode: number; shift: boolean; meta: boolean; alt: boolean; } export type RobotMouseData = { clientX: number; clientY: number; } & ScreenVideoInfo export type RobotType = 'mouse' | 'key' // 鼠标 键盘 const robotHandle = (function () { function mouseHandle(data: RobotMouseData) { const {clientX, clientY, video, screen} = data let x = clientX * screen.width / video.width let y = clientY * screen.height / video.height // robot.moveMouse(x, y) // robot.mouseClick() console.log(\`robot点击了\${x},\${y}\`) } function keyHandle(data: RobotKeyData) { const modifiers = [] // 修饰键 if (data.meta) { modifiers.push('meta') } if (data.shift) { modifiers.push('shift') } if (data.alt) { modifiers.push('alt') } let key = vkey[data.keyCode].toLowerCase() if (key[0] !== ' |
【本文地址】