From a2d36a770a40b5bb5df50277cd003e81e5655260 Mon Sep 17 00:00:00 2001 From: Hailey Jang Date: Thu, 11 Mar 2021 12:39:27 +0900 Subject: [PATCH 01/42] Push starter files --- .py | 1 + renderer/components/video-feed/VideoFeed.tsx | 5 +++++ renderer/components/video-feed/index.tsx | 8 ++++++++ 3 files changed, 14 insertions(+) create mode 100644 .py create mode 100644 renderer/components/video-feed/VideoFeed.tsx create mode 100644 renderer/components/video-feed/index.tsx diff --git a/.py b/.py new file mode 100644 index 00000000..16aecdaf --- /dev/null +++ b/.py @@ -0,0 +1 @@ +sdfjknjasdlfkjnwaeljkndkjflnaklsjdnflawenjfksdf \ No newline at end of file diff --git a/renderer/components/video-feed/VideoFeed.tsx b/renderer/components/video-feed/VideoFeed.tsx new file mode 100644 index 00000000..5ccbf15c --- /dev/null +++ b/renderer/components/video-feed/VideoFeed.tsx @@ -0,0 +1,5 @@ +export const VideoFeed = () => { + return ( + + ) +} \ No newline at end of file diff --git a/renderer/components/video-feed/index.tsx b/renderer/components/video-feed/index.tsx new file mode 100644 index 00000000..7a5c38f3 --- /dev/null +++ b/renderer/components/video-feed/index.tsx @@ -0,0 +1,8 @@ +import React from 'react'; +import ReactDOM from 'react-dom'; +import { VideoFeed } from './VideoFeed'; + +ReactDOM.render( + , + document.getElementById("Videofeed-react") +); \ No newline at end of file From 07f8b4fe24037bf4ad8b57e0fd040483f1291ee4 Mon Sep 17 00:00:00 2001 From: uthmanmomen13 Date: Wed, 10 Mar 2021 19:48:27 -0800 Subject: [PATCH 02/42] updated keyboard connect/disconnect --- renderer/utils/sagas.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/renderer/utils/sagas.ts b/renderer/utils/sagas.ts index b54bed4d..be9bfa60 100644 --- a/renderer/utils/sagas.ts +++ b/renderer/utils/sagas.ts @@ -258,6 +258,21 @@ function formatGamepads(newGamepads: (Gamepad | null)[]): Input[] { return formattedGamepads; } +/* + Send a packet to runtime telling them when keyboard is connected/disconnected +*/ +function* sendKeyboardConnectionStatus() { + const currEditorState = yield select(editorState); + + const keyboardConnected = new Input({ + connected: currEditorState.isKeyboardModeToggled, + axes: [], + buttons: 0, + source: Source.KEYBOARD + }) + ipcRenderer.send('stateUpdate', [keyboardConnected], Source.KEYBOARD); +} + function* sendKeyboardInputs() { const currEditorState = yield select(editorState); @@ -615,6 +630,7 @@ export default function* rootSaga() { takeEvery('TOGGLE_FIELD_CONTROL', handleFieldControl), takeEvery('TIMESTAMP_CHECK', timestampBounceback), takeEvery('UPDATE_KEYBOARD_BITMAP', sendKeyboardInputs), + takeEvery('UPDATE_IS_KEYBOARD_MODE_TOGGLED', sendKeyboardConnectionStatus), fork(runtimeHeartbeat), fork(runtimeGamepads), fork(runtimeSaga), From 78eebf25abb8fe34e3ee66f01a1acd89dba8154b Mon Sep 17 00:00:00 2001 From: Hailey Jang Date: Sun, 14 Mar 2021 06:39:51 +0900 Subject: [PATCH 03/42] Videofeed Window In process --- main/MenuTemplate/DebugMenu.ts | 15 ++- main/MenuTemplate/HelpMenu.ts | 4 +- main/RendererBridge.ts | 36 ++++++- main/main-process.ts | 27 ++++- renderer/components/DNav.tsx | 10 ++ renderer/components/video-feed/VideoFeed.tsx | 108 ++++++++++++++++++- renderer/components/video-feed/index.tsx | 2 +- static/video-feed/video.html | 12 +++ webpack.config.babel.js | 12 +++ 9 files changed, 210 insertions(+), 16 deletions(-) create mode 100644 static/video-feed/video.html diff --git a/main/MenuTemplate/DebugMenu.ts b/main/MenuTemplate/DebugMenu.ts index 1dc43060..975de247 100644 --- a/main/MenuTemplate/DebugMenu.ts +++ b/main/MenuTemplate/DebugMenu.ts @@ -19,8 +19,8 @@ const DebugMenu: MenuItemConstructorOptions = { { label: 'Toggle DevTools', click() { - if (RendererBridge.registeredWindow) { - RendererBridge.registeredWindow.webContents.toggleDevTools(); + if (RendererBridge.registeredWindows) { + RendererBridge.registeredWindows['main']?.webContents.toggleDevTools(); } }, accelerator: 'CommandOrControl+alt+I', @@ -57,8 +57,8 @@ const DebugMenu: MenuItemConstructorOptions = { label: 'Reload', accelerator: 'CommandOrControl+R', click() { - if (RendererBridge.registeredWindow) { - RendererBridge.registeredWindow.reload(); + if (RendererBridge.registeredWindows['main']) { + RendererBridge.registeredWindows['main']?.reload(); } }, }, @@ -71,6 +71,13 @@ const DebugMenu: MenuItemConstructorOptions = { }); }, }, + + { + label: 'Toggle Videofeed DevTools', + click() { + RendererBridge.toggleWindowDevtools('Videofeed-react'); + }, + }, ], }; diff --git a/main/MenuTemplate/HelpMenu.ts b/main/MenuTemplate/HelpMenu.ts index 7cdc04a0..6be11346 100644 --- a/main/MenuTemplate/HelpMenu.ts +++ b/main/MenuTemplate/HelpMenu.ts @@ -11,8 +11,8 @@ const HelpMenu: MenuItemConstructorOptions = { { label: 'Interactive Tutorial', click() { - if (RendererBridge.registeredWindow) { - RendererBridge.registeredWindow.webContents.send('start-interactive-tour'); + if (RendererBridge.registeredWindows['main']) { + RendererBridge.registeredWindows['main']?.webContents.send('start-interactive-tour'); } }, accelerator: 'CommandOrControl+T', diff --git a/main/RendererBridge.ts b/main/RendererBridge.ts index 7d8e708a..199853e6 100644 --- a/main/RendererBridge.ts +++ b/main/RendererBridge.ts @@ -8,17 +8,43 @@ import _ from 'lodash'; import { BrowserWindow } from "electron"; class RendererBridge { - registeredWindow: BrowserWindow | null = null; + registeredWindows: Record = {}; - registerWindow = (electronWindow: BrowserWindow) => { - this.registeredWindow = electronWindow; + registerWindow = (key: string, electronWindow: BrowserWindow) => { + console.log('registering window', key) + this.registeredWindows[key] = electronWindow; }; + unregisterWindow = (key: string) => { + console.log('unregistering window', key); + delete this.registeredWindows[key]; + } + + reloadWindow = (key: string) => { + console.log('reloading window', key); + const registeredWindow = this.registeredWindows[key] + registeredWindow?.reload(); + } + + toggleWindowDevtools = (key: string) => { + console.log('toggling devtools for window', key); + const registeredWindow = this.registeredWindows[key]; + if (registeredWindow) { + registeredWindow.webContents.toggleDevTools(); + // console.log("successfully opened devtools window", key); + } else { + // console.log("failed to open devtools -- window", key, "does not exist"); + } + + } + reduxDispatch = (action: any) => { - if (this.registeredWindow) { - this.registeredWindow.webContents.send('dispatch', action); + for (const key of Object.keys(this.registeredWindows)) { + const registeredWindow = this.registeredWindows[key]; + registeredWindow?.webContents.send('dispatch', action); } }; + }; export default new RendererBridge(); diff --git a/main/main-process.ts b/main/main-process.ts index 24bb8011..50dd14ea 100644 --- a/main/main-process.ts +++ b/main/main-process.ts @@ -61,6 +61,8 @@ export default function showAPI() { } app.on('ready', () => { + let videoWindow: BrowserWindow | null = null; + Runtime.setup(); ipcMain.on('FC_CONFIG_CHANGE', FCObject.changeFCInfo); ipcMain.on('FC_INITIALIZE', initializeFC); @@ -73,8 +75,31 @@ app.on('ready', () => { } }); + ipcMain.on('SHOW_VIDEOFEED', () => { + if (!videoWindow) { + videoWindow = new BrowserWindow({ + webPreferences: { + nodeIntegration: true, + enableRemoteModule: true, + }, + width: 1000, + height: 700, + }); + RendererBridge.registerWindow('videofeed', videoWindow); + videoWindow.on('closed', () => { + videoWindow = null; + }); + videoWindow.loadURL(`file://${__dirname}/../static/video-feed/video.html`); + videoWindow.once('ready-to-show', () => { + if (videoWindow) { + videoWindow.show(); + } + }); + } + }) + // Binding for the main process to inject into Redux workflow - RendererBridge.registerWindow(mainWindow); + RendererBridge.registerWindow('main', mainWindow); mainWindow.maximize(); mainWindow.loadURL(`file://${__dirname}/../static/index.html`); diff --git a/renderer/components/DNav.tsx b/renderer/components/DNav.tsx index 348f9edc..4da6b35e 100644 --- a/renderer/components/DNav.tsx +++ b/renderer/components/DNav.tsx @@ -7,6 +7,7 @@ import { StatusLabel } from './StatusLabel'; import { TooltipButton } from './TooltipButton'; import { VERSION } from '../consts'; import { robotState } from '../utils/utils'; +import { ipcRenderer } from 'electron'; interface StateProps { runtimeVersion: string; @@ -127,6 +128,15 @@ const DNavComponent = (props: Props) => { id="update-software-button" glyph="cloud-upload" /> + ipcRenderer.send('SHOW_VIDEOFEED')} + id="show-video-feed-button" + glyph="play" + disabled={false} + /> diff --git a/renderer/components/video-feed/VideoFeed.tsx b/renderer/components/video-feed/VideoFeed.tsx index 5ccbf15c..2276efdb 100644 --- a/renderer/components/video-feed/VideoFeed.tsx +++ b/renderer/components/video-feed/VideoFeed.tsx @@ -1,5 +1,107 @@ -export const VideoFeed = () => { +import React from 'react'; +import { keyboardButtons } from '../../consts/keyboard-buttons'; +import { TooltipButton } from '../TooltipButton'; +import { updateKeyboardBitmap } from '../../actions/EditorActions'; +import { Dispatch } from 'redux'; +import { connect } from 'react-redux'; +import { Panel } from 'react-bootstrap'; + +interface OwnProps { + onUpdateKeyboardBitmap: (keyboardBitmap: number) => void; +} + +type Props = OwnProps; + +interface State { + isKeyboardModeToggled: boolean; + keyboardBitmap: number; +} + +export class VideoFeedComponent extends React.Component{ + + constructor(props: Props) { + super(props); + this.state = { + isKeyboardModeToggled: false, + keyboardBitmap: 0 + }; + } + + bitShiftLeft = (value: number, numPositions: number) => { + return value * Math.pow(2, numPositions); + } + + + // toggle keyboard control and add/remove listening for key presses to control robot + toggleKeyboardControl = () => { + this.setState({ isKeyboardModeToggled: !this.state.isKeyboardModeToggled }); + + if (!this.state.isKeyboardModeToggled) { + // We need passive true so that we are able to remove the event listener when we are not in Keyboard Control mode + window.addEventListener('keydown', this.turnCharacterOn, { passive: true }); + window.addEventListener('keyup', this.turnCharacterOff, { passive: true }); + } else { + window.removeEventListener('keydown', this.turnCharacterOn); + window.removeEventListener('keyup', this.turnCharacterOff); + this.setState({ keyboardBitmap: 0 }); + this.props.onUpdateKeyboardBitmap(this.state.keyboardBitmap); + } + }; + + updateKeyboardBitmap = (currentCharacter: string, isKeyPressed: boolean) => { + const keyboardNum = keyboardButtons[currentCharacter]; + let newKeyboardBitmap: number = this.state.keyboardBitmap; + + const shift = this.bitShiftLeft(1, keyboardNum); + const MAX_INT32_BITS = 2147483648; // 2^31 + + const shiftHighBits = shift / MAX_INT32_BITS; + const shiftLowBits = shift % MAX_INT32_BITS; + const mapHighBits = newKeyboardBitmap / MAX_INT32_BITS; + const mapLowBits = newKeyboardBitmap % MAX_INT32_BITS; + + if (!isKeyPressed) { + newKeyboardBitmap = (~shiftHighBits & mapHighBits) * MAX_INT32_BITS + (~shiftLowBits & mapLowBits); + } else if (isKeyPressed) { + newKeyboardBitmap = (shiftHighBits | mapHighBits) * MAX_INT32_BITS + (shiftLowBits | mapLowBits); + } + + this.setState({ keyboardBitmap: newKeyboardBitmap }); + this.props.onUpdateKeyboardBitmap(this.state.keyboardBitmap); + }; + + turnCharacterOff = (e: KeyboardEvent) => { + // NOT THE ACTION updateKeyboardBitmap. THIS IS A LOCAL FUNCTION + this.updateKeyboardBitmap(e.key, false); + } + turnCharacterOn = (e: KeyboardEvent) => { + this.updateKeyboardBitmap(e.key, true) + } + + render(){ return ( - + + + ) -} \ No newline at end of file + } +} + +const mapStateToProps = (state: ApplicationState) => ({ + keyboardBitmap: state.editor.keyboardBitmap +}); + +const mapDispatchToProps = (dispatch: Dispatch) => ({ + onUpdateKeyboardBitmap: (keyboardBitmap: number) => { + dispatch(updateKeyboardBitmap(keyboardBitmap)); + } +}); + +export const VideoFeed = connect(mapStateToProps, mapDispatchToProps)(VideoFeedComponent); \ No newline at end of file diff --git a/renderer/components/video-feed/index.tsx b/renderer/components/video-feed/index.tsx index 7a5c38f3..b47c2744 100644 --- a/renderer/components/video-feed/index.tsx +++ b/renderer/components/video-feed/index.tsx @@ -3,6 +3,6 @@ import ReactDOM from 'react-dom'; import { VideoFeed } from './VideoFeed'; ReactDOM.render( - , + , document.getElementById("Videofeed-react") ); \ No newline at end of file diff --git a/static/video-feed/video.html b/static/video-feed/video.html new file mode 100644 index 00000000..27b579fb --- /dev/null +++ b/static/video-feed/video.html @@ -0,0 +1,12 @@ + + + + + + VideoFeed + + +
+ + + \ No newline at end of file diff --git a/webpack.config.babel.js b/webpack.config.babel.js index 3eb90a3d..4d86671e 100644 --- a/webpack.config.babel.js +++ b/webpack.config.babel.js @@ -46,6 +46,18 @@ export default [ target: 'electron-renderer', module: modules, }, + { + entry: './renderer/components/video-feed/index.tsx', + resolve: { + extensions: ['.tsx', '.ts', '.js'], + }, + output: { + path: path.join(__dirname, 'build'), + filename: "videofeed.js", + }, + target: 'electron-renderer', + module: modules, + }, { entry: './main/main-process.ts', resolve: { From cfced80a89efe3e24df1e96c169744fe789cf7c3 Mon Sep 17 00:00:00 2001 From: uthmanmomen13 Date: Sat, 13 Mar 2021 14:03:40 -0800 Subject: [PATCH 04/42] removed unnecessary bitmap change in editor.tsx --- main/networking/Runtime.ts | 1 + renderer/components/Editor.tsx | 11 +++++++---- renderer/components/EditorContainer.ts | 6 +++++- renderer/utils/sagas.ts | 5 +++-- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/main/networking/Runtime.ts b/main/networking/Runtime.ts index df6c577a..10d911de 100644 --- a/main/networking/Runtime.ts +++ b/main/networking/Runtime.ts @@ -450,6 +450,7 @@ class UDPConn { * or when 100 ms has passed (with 50 ms cooldown) */ sendInputs = (_event: IpcMainEvent, data: protos.Input[], source: protos.Source) => { + if (data.length === 0) { data.push( protos.Input.create({ diff --git a/renderer/components/Editor.tsx b/renderer/components/Editor.tsx index 27f7110b..0a605f3d 100644 --- a/renderer/components/Editor.tsx +++ b/renderer/components/Editor.tsx @@ -70,6 +70,7 @@ interface OwnProps { onDownloadCode: () => void; onUploadCode: () => void; onUpdateKeyboardBitmap: (keyboardBitmap: number) => void; + onUpdateKeyboardModeToggle: (isKeyboardToggled: boolean) => void; } type Props = StateProps & OwnProps; @@ -277,17 +278,19 @@ export class Editor extends React.Component { // toggle keyboard control and add/remove listening for key presses to control robot toggleKeyboardControl = () => { - this.setState({ isKeyboardModeToggled: !this.state.isKeyboardModeToggled }); + const { isKeyboardModeToggled } = this.state; + this.setState({ isKeyboardModeToggled: !isKeyboardModeToggled }); + this.props.onUpdateKeyboardModeToggle(!isKeyboardModeToggled); - if (!this.state.isKeyboardModeToggled) { + if (!isKeyboardModeToggled) { // We need passive true so that we are able to remove the event listener when we are not in Keyboard Control mode window.addEventListener('keydown', this.turnCharacterOn, { passive: true }); window.addEventListener('keyup', this.turnCharacterOff, { passive: true }); } else { window.removeEventListener('keydown', this.turnCharacterOn); window.removeEventListener('keyup', this.turnCharacterOff); - this.setState({ keyboardBitmap: 0 }); - this.props.onUpdateKeyboardBitmap(this.state.keyboardBitmap); + // this.setState({ keyboardBitmap: 0 }); + // this.props.onUpdateKeyboardBitmap(this.state.keyboardBitmap); } }; diff --git a/renderer/components/EditorContainer.ts b/renderer/components/EditorContainer.ts index b037e5b8..b35020bf 100644 --- a/renderer/components/EditorContainer.ts +++ b/renderer/components/EditorContainer.ts @@ -8,7 +8,8 @@ import { createNewFile, downloadCode, uploadCode, - updateKeyboardBitmap + updateKeyboardBitmap, + updateIsKeyboardModeToggled } from '../actions/EditorActions'; import { changeTheme, changeFontSize } from '../actions/SettingsActions'; import { toggleConsole, clearConsole } from '../actions/ConsoleActions'; @@ -78,6 +79,9 @@ const mapDispatchToProps = (dispatch: Dispatch) => ({ }, onUpdateKeyboardBitmap: (keyboardBitmap: number) => { dispatch(updateKeyboardBitmap(keyboardBitmap)); + }, + onUpdateKeyboardModeToggle: (isKeyboardToggled: boolean) => { + dispatch(updateIsKeyboardModeToggled(isKeyboardToggled)); } }); diff --git a/renderer/utils/sagas.ts b/renderer/utils/sagas.ts index be9bfa60..641d14a2 100644 --- a/renderer/utils/sagas.ts +++ b/renderer/utils/sagas.ts @@ -267,15 +267,16 @@ function* sendKeyboardConnectionStatus() { const keyboardConnected = new Input({ connected: currEditorState.isKeyboardModeToggled, axes: [], - buttons: 0, + buttons: 1, source: Source.KEYBOARD }) + ipcRenderer.send('stateUpdate', [keyboardConnected], Source.KEYBOARD); } function* sendKeyboardInputs() { const currEditorState = yield select(editorState); - + const keyboard = new Input({ connected: true, axes: [], From af275cdd2a8b56d6289174d1dc6f3babb3cda4b7 Mon Sep 17 00:00:00 2001 From: ewc340 Date: Sat, 13 Mar 2021 14:09:48 -0800 Subject: [PATCH 05/42] Change buttons back to 0 --- renderer/utils/sagas.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/renderer/utils/sagas.ts b/renderer/utils/sagas.ts index 641d14a2..483f4dd5 100644 --- a/renderer/utils/sagas.ts +++ b/renderer/utils/sagas.ts @@ -267,10 +267,10 @@ function* sendKeyboardConnectionStatus() { const keyboardConnected = new Input({ connected: currEditorState.isKeyboardModeToggled, axes: [], - buttons: 1, + buttons: 0, source: Source.KEYBOARD - }) - + }); + ipcRenderer.send('stateUpdate', [keyboardConnected], Source.KEYBOARD); } From 7a844559a2a668db2897ebe2d42168991d49b4ae Mon Sep 17 00:00:00 2001 From: Hailey Jang Date: Sun, 14 Mar 2021 07:55:51 +0900 Subject: [PATCH 06/42] Now the button is funky --- .py | 1 - main/MenuTemplate/DebugMenu.ts | 2 +- renderer/components/video-feed/VideoFeed.tsx | 44 ++++++++------------ static/video-feed/video.html | 2 +- 4 files changed, 19 insertions(+), 30 deletions(-) delete mode 100644 .py diff --git a/.py b/.py deleted file mode 100644 index 16aecdaf..00000000 --- a/.py +++ /dev/null @@ -1 +0,0 @@ -sdfjknjasdlfkjnwaeljkndkjflnaklsjdnflawenjfksdf \ No newline at end of file diff --git a/main/MenuTemplate/DebugMenu.ts b/main/MenuTemplate/DebugMenu.ts index 975de247..94add31f 100644 --- a/main/MenuTemplate/DebugMenu.ts +++ b/main/MenuTemplate/DebugMenu.ts @@ -75,7 +75,7 @@ const DebugMenu: MenuItemConstructorOptions = { { label: 'Toggle Videofeed DevTools', click() { - RendererBridge.toggleWindowDevtools('Videofeed-react'); + RendererBridge.toggleWindowDevtools('videofeed'); }, }, ], diff --git a/renderer/components/video-feed/VideoFeed.tsx b/renderer/components/video-feed/VideoFeed.tsx index 2276efdb..cb69bd32 100644 --- a/renderer/components/video-feed/VideoFeed.tsx +++ b/renderer/components/video-feed/VideoFeed.tsx @@ -1,25 +1,17 @@ import React from 'react'; import { keyboardButtons } from '../../consts/keyboard-buttons'; import { TooltipButton } from '../TooltipButton'; -import { updateKeyboardBitmap } from '../../actions/EditorActions'; -import { Dispatch } from 'redux'; -import { connect } from 'react-redux'; -import { Panel } from 'react-bootstrap'; +import { Input, Source } from '../../../protos/protos'; +import { ipcRenderer } from 'electron'; -interface OwnProps { - onUpdateKeyboardBitmap: (keyboardBitmap: number) => void; -} - -type Props = OwnProps; - interface State { isKeyboardModeToggled: boolean; keyboardBitmap: number; } -export class VideoFeedComponent extends React.Component{ +export class VideoFeed extends React.Component<{},State>{ - constructor(props: Props) { + constructor(props = {}) { super(props); this.state = { isKeyboardModeToggled: false, @@ -27,6 +19,18 @@ export class VideoFeedComponent extends React.Component{ }; } + sendToRuntime =() => { + + const keyboard = new Input({ + connected: true, + axes: [], + buttons: this.state.keyboardBitmap, + source: Source.KEYBOARD + }); + + ipcRenderer.send('stateUpdate', [keyboard], Source.KEYBOARD); + } + bitShiftLeft = (value: number, numPositions: number) => { return value * Math.pow(2, numPositions); } @@ -44,7 +48,6 @@ export class VideoFeedComponent extends React.Component{ window.removeEventListener('keydown', this.turnCharacterOn); window.removeEventListener('keyup', this.turnCharacterOff); this.setState({ keyboardBitmap: 0 }); - this.props.onUpdateKeyboardBitmap(this.state.keyboardBitmap); } }; @@ -67,7 +70,7 @@ export class VideoFeedComponent extends React.Component{ } this.setState({ keyboardBitmap: newKeyboardBitmap }); - this.props.onUpdateKeyboardBitmap(this.state.keyboardBitmap); + this.sendToRuntime(); }; turnCharacterOff = (e: KeyboardEvent) => { @@ -80,7 +83,6 @@ export class VideoFeedComponent extends React.Component{ render(){ return ( - { disabled={false} bsStyle={this.state.isKeyboardModeToggled ? 'info' : 'default'} /> - ) } } -const mapStateToProps = (state: ApplicationState) => ({ - keyboardBitmap: state.editor.keyboardBitmap -}); - -const mapDispatchToProps = (dispatch: Dispatch) => ({ - onUpdateKeyboardBitmap: (keyboardBitmap: number) => { - dispatch(updateKeyboardBitmap(keyboardBitmap)); - } -}); - -export const VideoFeed = connect(mapStateToProps, mapDispatchToProps)(VideoFeedComponent); \ No newline at end of file diff --git a/static/video-feed/video.html b/static/video-feed/video.html index 27b579fb..636691b0 100644 --- a/static/video-feed/video.html +++ b/static/video-feed/video.html @@ -7,6 +7,6 @@
- + \ No newline at end of file From 6cc06a853f05ddcf012f26aead3e4fe92be839d8 Mon Sep 17 00:00:00 2001 From: ewc340 Date: Sat, 20 Mar 2021 13:41:02 -0700 Subject: [PATCH 07/42] Remove duplicate sendKeyboardInputs function --- renderer/utils/sagas.ts | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/renderer/utils/sagas.ts b/renderer/utils/sagas.ts index c25745aa..91fd18ea 100644 --- a/renderer/utils/sagas.ts +++ b/renderer/utils/sagas.ts @@ -274,19 +274,6 @@ function* sendKeyboardConnectionStatus() { ipcRenderer.send('stateUpdate', [keyboardConnectionStatus], Source.KEYBOARD); } -function* sendKeyboardInputs() { - const currEditorState = yield select(editorState); - - const keyboardConnected = new Input({ - connected: currEditorState.isKeyboardModeToggled, - axes: [], - buttons: 0, - source: Source.KEYBOARD - }); - - ipcRenderer.send('stateUpdate', [keyboardConnected], Source.KEYBOARD); -} - function* sendKeyboardInputs() { const currEditorState = yield select(editorState); From 345ce20067098c7f655cbda4a40fe56f695914b4 Mon Sep 17 00:00:00 2001 From: ewc340 Date: Sat, 20 Mar 2021 14:12:10 -0700 Subject: [PATCH 08/42] Change VideoFeed component to functional --- renderer/components/video-feed/VideoFeed.tsx | 107 ++++++++----------- 1 file changed, 46 insertions(+), 61 deletions(-) diff --git a/renderer/components/video-feed/VideoFeed.tsx b/renderer/components/video-feed/VideoFeed.tsx index cb69bd32..183acf84 100644 --- a/renderer/components/video-feed/VideoFeed.tsx +++ b/renderer/components/video-feed/VideoFeed.tsx @@ -1,61 +1,48 @@ -import React from 'react'; +import React, { useState } from 'react'; import { keyboardButtons } from '../../consts/keyboard-buttons'; import { TooltipButton } from '../TooltipButton'; import { Input, Source } from '../../../protos/protos'; import { ipcRenderer } from 'electron'; -interface State { - isKeyboardModeToggled: boolean; - keyboardBitmap: number; -} +export const VideoFeed = () => { + const [isKeyboardModeToggled, setIsKeyboardModeToggled] = useState(false); + const [keyboardBitmap, setKeyboardBitmap] = useState(0); -export class VideoFeed extends React.Component<{},State>{ - - constructor(props = {}) { - super(props); - this.state = { - isKeyboardModeToggled: false, - keyboardBitmap: 0 - }; - } - - sendToRuntime =() => { + const sendKeyboardInputsToRuntime = () => { + const keyboard = new Input({ + connected: true, + axes: [], + buttons: keyboardBitmap, + source: Source.KEYBOARD + }); - const keyboard = new Input({ - connected: true, - axes: [], - buttons: this.state.keyboardBitmap, - source: Source.KEYBOARD - }); - - ipcRenderer.send('stateUpdate', [keyboard], Source.KEYBOARD); - } + ipcRenderer.send('stateUpdate', [keyboard], Source.KEYBOARD); + }; - bitShiftLeft = (value: number, numPositions: number) => { + const bitShiftLeft = (value: number, numPositions: number) => { return value * Math.pow(2, numPositions); - } - + }; // toggle keyboard control and add/remove listening for key presses to control robot - toggleKeyboardControl = () => { - this.setState({ isKeyboardModeToggled: !this.state.isKeyboardModeToggled }); - - if (!this.state.isKeyboardModeToggled) { + const toggleKeyboardControl = () => { + if (!isKeyboardModeToggled) { // We need passive true so that we are able to remove the event listener when we are not in Keyboard Control mode - window.addEventListener('keydown', this.turnCharacterOn, { passive: true }); - window.addEventListener('keyup', this.turnCharacterOff, { passive: true }); + window.addEventListener('keydown', turnCharacterOn, { passive: true }); + window.addEventListener('keyup', turnCharacterOff, { passive: true }); } else { - window.removeEventListener('keydown', this.turnCharacterOn); - window.removeEventListener('keyup', this.turnCharacterOff); - this.setState({ keyboardBitmap: 0 }); + window.removeEventListener('keydown', turnCharacterOn); + window.removeEventListener('keyup', turnCharacterOff); + setKeyboardBitmap(0); } + + setIsKeyboardModeToggled(!isKeyboardModeToggled); }; - updateKeyboardBitmap = (currentCharacter: string, isKeyPressed: boolean) => { + const updateKeyboardBitmap = (currentCharacter: string, isKeyPressed: boolean) => { const keyboardNum = keyboardButtons[currentCharacter]; - let newKeyboardBitmap: number = this.state.keyboardBitmap; + let newKeyboardBitmap: number = keyboardBitmap; - const shift = this.bitShiftLeft(1, keyboardNum); + const shift = bitShiftLeft(1, keyboardNum); const MAX_INT32_BITS = 2147483648; // 2^31 const shiftHighBits = shift / MAX_INT32_BITS; @@ -69,29 +56,27 @@ export class VideoFeed extends React.Component<{},State>{ newKeyboardBitmap = (shiftHighBits | mapHighBits) * MAX_INT32_BITS + (shiftLowBits | mapLowBits); } - this.setState({ keyboardBitmap: newKeyboardBitmap }); - this.sendToRuntime(); + setKeyboardBitmap(newKeyboardBitmap); + sendKeyboardInputsToRuntime(); }; - turnCharacterOff = (e: KeyboardEvent) => { + const turnCharacterOff = (e: KeyboardEvent) => { // NOT THE ACTION updateKeyboardBitmap. THIS IS A LOCAL FUNCTION - this.updateKeyboardBitmap(e.key, false); - } - turnCharacterOn = (e: KeyboardEvent) => { - this.updateKeyboardBitmap(e.key, true) - } + updateKeyboardBitmap(e.key, false); + }; - render(){ - return ( - - ) - } -} + const turnCharacterOn = (e: KeyboardEvent) => { + updateKeyboardBitmap(e.key, true); + }; + return ( + + ); +}; From c545e026853344a90448196a9e046d13568acf36 Mon Sep 17 00:00:00 2001 From: Hailey Jang Date: Sun, 21 Mar 2021 06:58:49 +0900 Subject: [PATCH 09/42] Changed to functional component --- renderer/components/video-feed/VideoFeed.tsx | 96 +++++++++++--------- 1 file changed, 52 insertions(+), 44 deletions(-) diff --git a/renderer/components/video-feed/VideoFeed.tsx b/renderer/components/video-feed/VideoFeed.tsx index cb69bd32..c21783c4 100644 --- a/renderer/components/video-feed/VideoFeed.tsx +++ b/renderer/components/video-feed/VideoFeed.tsx @@ -1,61 +1,67 @@ -import React from 'react'; +import React, { useEffect, useState } from 'react'; import { keyboardButtons } from '../../consts/keyboard-buttons'; import { TooltipButton } from '../TooltipButton'; import { Input, Source } from '../../../protos/protos'; import { ipcRenderer } from 'electron'; +import { ButtonGroup } from 'react-bootstrap'; -interface State { - isKeyboardModeToggled: boolean; - keyboardBitmap: number; -} +const VideoFeed = () => { -export class VideoFeed extends React.Component<{},State>{ - - constructor(props = {}) { - super(props); - this.state = { - isKeyboardModeToggled: false, - keyboardBitmap: 0 - }; - } + const [isKeyboardModeToggled, changeKeyboardMode] = useState(false); + const [keyboardBitmap, changeKeyboardBitmap] = useState(0); - sendToRuntime =() => { + const sendToRuntime = () => { const keyboard = new Input({ connected: true, axes: [], - buttons: this.state.keyboardBitmap, + buttons: keyboardBitmap, source: Source.KEYBOARD }); ipcRenderer.send('stateUpdate', [keyboard], Source.KEYBOARD); } - bitShiftLeft = (value: number, numPositions: number) => { + const bitShiftLeft = (value: number, numPositions: number) => { return value * Math.pow(2, numPositions); } + useEffect(() => { + const player = OvenPlayer.create('player', { + sources: [ + { + type: 'webRTC', + file: 'ws://161.35.224.231:3333/app/stream', + label: '480p' + } + ] + }); + player.on('error', function (error) { + console.log(error); + }); + }, []); + // toggle keyboard control and add/remove listening for key presses to control robot - toggleKeyboardControl = () => { - this.setState({ isKeyboardModeToggled: !this.state.isKeyboardModeToggled }); + const toggleKeyboardControl = () => { + changeKeyboardMode(!isKeyboardModeToggled); - if (!this.state.isKeyboardModeToggled) { + if (!isKeyboardModeToggled) { // We need passive true so that we are able to remove the event listener when we are not in Keyboard Control mode - window.addEventListener('keydown', this.turnCharacterOn, { passive: true }); - window.addEventListener('keyup', this.turnCharacterOff, { passive: true }); + window.addEventListener('keydown', turnCharacterOn, { passive: true }); + window.addEventListener('keyup', turnCharacterOff, { passive: true }); } else { - window.removeEventListener('keydown', this.turnCharacterOn); - window.removeEventListener('keyup', this.turnCharacterOff); - this.setState({ keyboardBitmap: 0 }); + window.removeEventListener('keydown', turnCharacterOn); + window.removeEventListener('keyup', turnCharacterOff); + changeKeyboardBitmap(0); } }; - updateKeyboardBitmap = (currentCharacter: string, isKeyPressed: boolean) => { + const updateKeyboardBitmap = (currentCharacter: string, isKeyPressed: boolean) => { const keyboardNum = keyboardButtons[currentCharacter]; - let newKeyboardBitmap: number = this.state.keyboardBitmap; + let newKeyboardBitmap: number = keyboardBitmap; - const shift = this.bitShiftLeft(1, keyboardNum); + const shift = bitShiftLeft(1, keyboardNum); const MAX_INT32_BITS = 2147483648; // 2^31 const shiftHighBits = shift / MAX_INT32_BITS; @@ -69,29 +75,31 @@ export class VideoFeed extends React.Component<{},State>{ newKeyboardBitmap = (shiftHighBits | mapHighBits) * MAX_INT32_BITS + (shiftLowBits | mapLowBits); } - this.setState({ keyboardBitmap: newKeyboardBitmap }); - this.sendToRuntime(); + changeKeyboardBitmap(newKeyboardBitmap); + sendToRuntime(); }; - turnCharacterOff = (e: KeyboardEvent) => { + const turnCharacterOff = (e: KeyboardEvent) => { // NOT THE ACTION updateKeyboardBitmap. THIS IS A LOCAL FUNCTION - this.updateKeyboardBitmap(e.key, false); + updateKeyboardBitmap(e.key, false); } - turnCharacterOn = (e: KeyboardEvent) => { - this.updateKeyboardBitmap(e.key, true) + const turnCharacterOn = (e: KeyboardEvent) => { + updateKeyboardBitmap(e.key, true) } - render(){ return ( - +
+ + + +
) - } } From a1b5d8c1d4b3a9fec167faeff28d31e975207c18 Mon Sep 17 00:00:00 2001 From: Hailey Jang Date: Tue, 23 Mar 2021 09:59:17 +0900 Subject: [PATCH 10/42] Added Ovenplayer as module and added type declaration --- package.json | 1 + renderer/components/video-feed/VideoFeed.tsx | 19 +++++++++++++++++-- renderer/types/ovenplayer.d.ts | 1 + yarn.lock | 5 +++++ 4 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 renderer/types/ovenplayer.d.ts diff --git a/package.json b/package.json index 4761f661..f15a18d5 100644 --- a/package.json +++ b/package.json @@ -120,6 +120,7 @@ "mousetrap": "1.6.1", "numeral": "2.0.6", "object-assign": "4.1.1", + "ovenplayer": "^0.10.0-alpha.1", "patch-package": "^6.2.2", "postinstall-postinstall": "^2.1.0", "prop-types": "15.6.0", diff --git a/renderer/components/video-feed/VideoFeed.tsx b/renderer/components/video-feed/VideoFeed.tsx index fadbf31d..4d3a8748 100644 --- a/renderer/components/video-feed/VideoFeed.tsx +++ b/renderer/components/video-feed/VideoFeed.tsx @@ -1,9 +1,9 @@ -import React, { useState } from 'react'; +import React, { useEffect, useState } from 'react'; import { keyboardButtons } from '../../consts/keyboard-buttons'; import { TooltipButton } from '../TooltipButton'; import { Input, Source } from '../../../protos/protos'; import { ipcRenderer } from 'electron'; -import { ButtonGroup } from 'react-bootstrap'; +import OvenPlayer from 'OvenPlayer'; export const VideoFeed = () => { const [isKeyboardModeToggled, setIsKeyboardModeToggled] = useState(false); @@ -70,6 +70,21 @@ export const VideoFeed = () => { updateKeyboardBitmap(e.key, true); }; + useEffect(() => { + const player = OvenPlayer.create('player', { + sources: [ + { + type: 'webRTC', + file: 'ws://161.35.224.231:3333/app/stream', + label: '480p' + } + ] + }); + player.on('error', function (error: any) { + console.log(error); + }); + }, []); + return ( Date: Mon, 22 Mar 2021 20:06:43 -0700 Subject: [PATCH 11/42] Small naming change for main video player --- main/networking/Runtime.ts | 1 - renderer/components/video-feed/VideoFeed.tsx | 57 +++++++++++--------- renderer/types/modules.d.ts | 3 +- renderer/types/ovenplayer.d.ts | 1 - 4 files changed, 34 insertions(+), 28 deletions(-) delete mode 100644 renderer/types/ovenplayer.d.ts diff --git a/main/networking/Runtime.ts b/main/networking/Runtime.ts index 10d911de..df6c577a 100644 --- a/main/networking/Runtime.ts +++ b/main/networking/Runtime.ts @@ -450,7 +450,6 @@ class UDPConn { * or when 100 ms has passed (with 50 ms cooldown) */ sendInputs = (_event: IpcMainEvent, data: protos.Input[], source: protos.Source) => { - if (data.length === 0) { data.push( protos.Input.create({ diff --git a/renderer/components/video-feed/VideoFeed.tsx b/renderer/components/video-feed/VideoFeed.tsx index 4d3a8748..b3cea286 100644 --- a/renderer/components/video-feed/VideoFeed.tsx +++ b/renderer/components/video-feed/VideoFeed.tsx @@ -1,14 +1,30 @@ import React, { useEffect, useState } from 'react'; +import OvenPlayer from 'ovenplayer'; import { keyboardButtons } from '../../consts/keyboard-buttons'; import { TooltipButton } from '../TooltipButton'; import { Input, Source } from '../../../protos/protos'; import { ipcRenderer } from 'electron'; -import OvenPlayer from 'OvenPlayer'; export const VideoFeed = () => { const [isKeyboardModeToggled, setIsKeyboardModeToggled] = useState(false); const [keyboardBitmap, setKeyboardBitmap] = useState(0); + useEffect(() => { + const mainVideoFeedPlayer = OvenPlayer.create('main-video-feed', { + sources: [ + { + type: 'webRTC', + file: 'ws://161.35.224.231:3333/app/stream', + label: '480p' + } + ] + }); + + mainVideoFeedPlayer.on('error', (error: any) => { + console.log(error); + }); + }, []); + const sendKeyboardInputsToRuntime = () => { const keyboard = new Input({ connected: true, @@ -62,7 +78,6 @@ export const VideoFeed = () => { }; const turnCharacterOff = (e: KeyboardEvent) => { - // NOT THE ACTION updateKeyboardBitmap. THIS IS A LOCAL FUNCTION updateKeyboardBitmap(e.key, false); }; @@ -70,29 +85,21 @@ export const VideoFeed = () => { updateKeyboardBitmap(e.key, true); }; - useEffect(() => { - const player = OvenPlayer.create('player', { - sources: [ - { - type: 'webRTC', - file: 'ws://161.35.224.231:3333/app/stream', - label: '480p' - } - ] - }); - player.on('error', function (error: any) { - console.log(error); - }); - }, []); - return ( - + <> + + + {/* MainVideoFeedPlayer above will target the main-video-feed div id below */} +
+ ); }; diff --git a/renderer/types/modules.d.ts b/renderer/types/modules.d.ts index 426006be..99979ba2 100644 --- a/renderer/types/modules.d.ts +++ b/renderer/types/modules.d.ts @@ -1 +1,2 @@ -declare module 'smalltalk'; \ No newline at end of file +declare module 'smalltalk'; +declare module 'ovenplayer'; diff --git a/renderer/types/ovenplayer.d.ts b/renderer/types/ovenplayer.d.ts deleted file mode 100644 index 0a26fb79..00000000 --- a/renderer/types/ovenplayer.d.ts +++ /dev/null @@ -1 +0,0 @@ -declare module 'OvenPlayer'; \ No newline at end of file From cae70255664d1a2a469fcdc495ede0badedadbb4 Mon Sep 17 00:00:00 2001 From: ewc340 Date: Mon, 22 Mar 2021 20:20:45 -0700 Subject: [PATCH 12/42] Change video feed html title --- static/video-feed/video.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/video-feed/video.html b/static/video-feed/video.html index 636691b0..3befe4ae 100644 --- a/static/video-feed/video.html +++ b/static/video-feed/video.html @@ -3,7 +3,7 @@ - VideoFeed + Video Feed
From 5e7328ad5adc8c16c9b29aa17f77b8c81be15932 Mon Sep 17 00:00:00 2001 From: Hailey Jang Date: Tue, 23 Mar 2021 12:46:52 +0900 Subject: [PATCH 13/42] More changes --- renderer/components/video-feed/VideoFeed.tsx | 29 ++++++++++++++------ static/video-feed/video.html | 1 + 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/renderer/components/video-feed/VideoFeed.tsx b/renderer/components/video-feed/VideoFeed.tsx index 4d3a8748..03a95168 100644 --- a/renderer/components/video-feed/VideoFeed.tsx +++ b/renderer/components/video-feed/VideoFeed.tsx @@ -4,6 +4,7 @@ import { TooltipButton } from '../TooltipButton'; import { Input, Source } from '../../../protos/protos'; import { ipcRenderer } from 'electron'; import OvenPlayer from 'OvenPlayer'; +import { ButtonGroup, Form, FormGroup, Panel } from 'react-bootstrap'; export const VideoFeed = () => { const [isKeyboardModeToggled, setIsKeyboardModeToggled] = useState(false); @@ -56,7 +57,7 @@ export const VideoFeed = () => { } else if (isKeyPressed) { newKeyboardBitmap = (shiftHighBits | mapHighBits) * MAX_INT32_BITS + (shiftLowBits | mapLowBits); } - + console.log(newKeyboardBitmap); setKeyboardBitmap(newKeyboardBitmap); sendKeyboardInputsToRuntime(); }; @@ -86,13 +87,23 @@ export const VideoFeed = () => { }, []); return ( - + + +
+ + + + + +
+ + ); }; diff --git a/static/video-feed/video.html b/static/video-feed/video.html index 636691b0..3ab9e056 100644 --- a/static/video-feed/video.html +++ b/static/video-feed/video.html @@ -6,6 +6,7 @@ VideoFeed +
From 8add5a3189dd93c6fef64ebe508ddc02da15f5e3 Mon Sep 17 00:00:00 2001 From: ewc340 Date: Mon, 22 Mar 2021 22:09:00 -0700 Subject: [PATCH 14/42] Fix network error --- static/video-feed/video.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/static/video-feed/video.html b/static/video-feed/video.html index 3befe4ae..91ca2b48 100644 --- a/static/video-feed/video.html +++ b/static/video-feed/video.html @@ -3,6 +3,9 @@ + Video Feed From b93beac5684c3035461a870ec80c5c86270dde2d Mon Sep 17 00:00:00 2001 From: ewc340 Date: Tue, 23 Mar 2021 01:25:11 -0700 Subject: [PATCH 15/42] Add try/catch to RendererBridge reduxDispatch --- main/RendererBridge.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/main/RendererBridge.ts b/main/RendererBridge.ts index 199853e6..fa9c77f8 100644 --- a/main/RendererBridge.ts +++ b/main/RendererBridge.ts @@ -39,12 +39,15 @@ class RendererBridge { } reduxDispatch = (action: any) => { - for (const key of Object.keys(this.registeredWindows)) { - const registeredWindow = this.registeredWindows[key]; - registeredWindow?.webContents.send('dispatch', action); + try { + for (const key of Object.keys(this.registeredWindows)) { + const registeredWindow = this.registeredWindows[key]; + registeredWindow?.webContents.send('dispatch', action); + } + } catch (e) { + console.log(e); } }; - }; export default new RendererBridge(); From 2b1b33a5fd837d892d17f56af6d494695c23a0f9 Mon Sep 17 00:00:00 2001 From: ewc340 Date: Tue, 23 Mar 2021 01:25:37 -0700 Subject: [PATCH 16/42] Add maximize to video feed window --- main/main-process.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/main/main-process.ts b/main/main-process.ts index 50dd14ea..e09472e0 100644 --- a/main/main-process.ts +++ b/main/main-process.ts @@ -89,6 +89,7 @@ app.on('ready', () => { videoWindow.on('closed', () => { videoWindow = null; }); + videoWindow.maximize(); videoWindow.loadURL(`file://${__dirname}/../static/video-feed/video.html`); videoWindow.once('ready-to-show', () => { if (videoWindow) { From e55247994e48926db0951d0be68cf10f7521c67a Mon Sep 17 00:00:00 2001 From: ewc340 Date: Tue, 23 Mar 2021 01:29:23 -0700 Subject: [PATCH 17/42] Add overhead video feed and styling --- renderer/components/video-feed/VideoFeed.tsx | 37 ++++++++++++++++---- static/video-feed/video-feed.css | 30 ++++++++++++++++ 2 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 static/video-feed/video-feed.css diff --git a/renderer/components/video-feed/VideoFeed.tsx b/renderer/components/video-feed/VideoFeed.tsx index b3cea286..b23dac4e 100644 --- a/renderer/components/video-feed/VideoFeed.tsx +++ b/renderer/components/video-feed/VideoFeed.tsx @@ -4,23 +4,42 @@ import { keyboardButtons } from '../../consts/keyboard-buttons'; import { TooltipButton } from '../TooltipButton'; import { Input, Source } from '../../../protos/protos'; import { ipcRenderer } from 'electron'; +import '../../../static/video-feed/video-feed.css'; export const VideoFeed = () => { const [isKeyboardModeToggled, setIsKeyboardModeToggled] = useState(false); const [keyboardBitmap, setKeyboardBitmap] = useState(0); useEffect(() => { - const mainVideoFeedPlayer = OvenPlayer.create('main-video-feed', { + const driverVideoFeed = OvenPlayer.create('driver-video-feed', { sources: [ { type: 'webRTC', - file: 'ws://161.35.224.231:3333/app/stream', - label: '480p' + file: 'ws://64.227.109.107:3333/driver/stream', + label: '720p' } - ] + ], + autoStart: true, + controls: false }); - mainVideoFeedPlayer.on('error', (error: any) => { + const overheadVideoFeed = OvenPlayer.create('overhead-video-feed', { + sources: [ + { + type: 'webRTC', + file: 'ws://64.227.109.107:3333/overhead/stream', + label: '720p' + } + ], + autoStart: true, + controls: false + }); + + overheadVideoFeed.on('error', (error: any) => { + console.log(error); + }); + + driverVideoFeed.on('error', (error: any) => { console.log(error); }); }, []); @@ -98,8 +117,12 @@ export const VideoFeed = () => { /> - {/* MainVideoFeedPlayer above will target the main-video-feed div id below */} -
+
+
+
+
+
+
); }; diff --git a/static/video-feed/video-feed.css b/static/video-feed/video-feed.css new file mode 100644 index 00000000..8aff52c2 --- /dev/null +++ b/static/video-feed/video-feed.css @@ -0,0 +1,30 @@ +.container { + position: relative; + height: 95vh; + overflow: hidden; +} + +.content { + position: absolute; + top: 0; + color:#f1f1f1; + background-color: rgba(0, 0, 0, 0.5); + width: 100%; + height: 100%; + display: flex; + justify-content: center; + align-items: center; + padding: 0; +} + +#overhead-video-feed { + width: 30%; + height: 30%; + position: absolute; + top: 0; + left: 0; + border-right: white; + border-bottom: white; + border-width: 1px; +} + From 8bdc0bf69368c103f2c0c9abbfbef6aeb0ea3ca9 Mon Sep 17 00:00:00 2001 From: ewc340 Date: Tue, 23 Mar 2021 01:30:17 -0700 Subject: [PATCH 18/42] Fix variable spacing --- static/video-feed/video.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/video-feed/video.html b/static/video-feed/video.html index 91ca2b48..464ae7cb 100644 --- a/static/video-feed/video.html +++ b/static/video-feed/video.html @@ -4,7 +4,7 @@ Video Feed From 2e4aeb59ccef11a02a0764a567f70b76f2fff02d Mon Sep 17 00:00:00 2001 From: Hailey Jang Date: Sat, 27 Mar 2021 12:09:03 -0700 Subject: [PATCH 19/42] Fixed button issue --- static/video-feed/video.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/static/video-feed/video.html b/static/video-feed/video.html index f39f68f2..99727e12 100644 --- a/static/video-feed/video.html +++ b/static/video-feed/video.html @@ -6,6 +6,8 @@ + + Video Feed From d6b79b6c68966414cf2c618edc69ace0fa0b9c63 Mon Sep 17 00:00:00 2001 From: ewc340 Date: Sat, 27 Mar 2021 16:01:53 -0700 Subject: [PATCH 20/42] Add Shepherd overlay --- main/RendererBridge.ts | 9 + main/main-process.ts | 4 +- main/networking/FieldControl.ts | 3 +- package.json | 5 +- renderer/components/Editor.tsx | 7 +- .../components/video-feed/ShepherdOverlay.tsx | 12 + renderer/components/video-feed/VideoFeed.tsx | 37 +- renderer/components/video-feed/scoreboard.js | 314 +++ static/video-feed/sandstorm.png | Bin 0 -> 633342 bytes static/video-feed/scoreboard.css | 102 + static/video-feed/scoreboard.html | 53 + static/video-feed/video-feed.css | 25 +- static/video-feed/video.html | 2 + webpack.config.babel.js | 22 +- yarn.lock | 2160 +++++++++++------ 15 files changed, 1939 insertions(+), 816 deletions(-) create mode 100644 renderer/components/video-feed/ShepherdOverlay.tsx create mode 100644 renderer/components/video-feed/scoreboard.js create mode 100644 static/video-feed/sandstorm.png create mode 100644 static/video-feed/scoreboard.css create mode 100644 static/video-feed/scoreboard.html diff --git a/main/RendererBridge.ts b/main/RendererBridge.ts index fa9c77f8..0e6f263e 100644 --- a/main/RendererBridge.ts +++ b/main/RendererBridge.ts @@ -48,6 +48,15 @@ class RendererBridge { console.log(e); } }; + + dispatchToWindow = (windowKey: string, channel: string, data: any) => { + try { + if (windowKey in this.registeredWindows) { + const registeredWindow = this.registeredWindows[windowKey]; + registeredWindow?.webContents.send(channel, data); + } + } catch (e) {} + } }; export default new RendererBridge(); diff --git a/main/main-process.ts b/main/main-process.ts index e09472e0..9c0344a2 100644 --- a/main/main-process.ts +++ b/main/main-process.ts @@ -91,7 +91,9 @@ app.on('ready', () => { }); videoWindow.maximize(); videoWindow.loadURL(`file://${__dirname}/../static/video-feed/video.html`); - videoWindow.once('ready-to-show', () => { + videoWindow.webContents.on('dom-ready', () => RendererBridge.dispatchToWindow('videofeed', 'shepherdScoreboardServerIpAddress', FCObject.bridgeAddress)) + // setTimeout(() => RendererBridge.windowDispatch('videofeed', 'shepherdScoreboardServerIpAddress', FCObject.bridgeAddress), 3000) + videoWindow.on('ready-to-show', () => { if (videoWindow) { videoWindow.show(); } diff --git a/main/networking/FieldControl.ts b/main/networking/FieldControl.ts index 035389d3..63c24b9b 100644 --- a/main/networking/FieldControl.ts +++ b/main/networking/FieldControl.ts @@ -20,7 +20,7 @@ class FCInternals { } init() { - this.socket = io(`http://${this.bridgeAddress}:7000`); + this.socket = io.connect(this.bridgeAddress); this.socket.on('connect', () => { this.logger.log('Connected to Field Control Socket'); this.socket!.on('robot_state', (data: any) => { @@ -76,6 +76,7 @@ export const FCObject = { if (arg.bridgeAddress !== null) { FCObject.bridgeAddress = arg.bridgeAddress; + RendererBridge.dispatchToWindow('videofeed', 'shepherdScoreboardServerIpAddress', arg.bridgeAddress); } }, }; diff --git a/package.json b/package.json index f15a18d5..634c01a4 100644 --- a/package.json +++ b/package.json @@ -113,7 +113,10 @@ "bufferutil": "4.0.1", "create-react-class": "15.6.2", "electron-json-storage": "4.0.2", + "html-loader": "^0.5.5", + "html-react-parser": "^1.2.4", "immutable": "3.8.2", + "jquery": "^3.6.0", "json-loader": "0.5.7", "keymirror": "0.1.1", "lodash": ">=4.17.19", @@ -136,7 +139,7 @@ "redux-saga": "^1.1.3", "seedrandom": "2.4.3", "smalltalk": "^4.0.6", - "socket.io-client": "^2.3.0", + "socket.io-client": "^4.0.0", "ssh2": "^0.8.9", "superagent": "3.8.2", "ts-node": "^9.0.0", diff --git a/renderer/components/Editor.tsx b/renderer/components/Editor.tsx index ab16f170..8d647ee5 100644 --- a/renderer/components/Editor.tsx +++ b/renderer/components/Editor.tsx @@ -319,7 +319,12 @@ export class Editor extends React.Component { this.updateKeyboardBitmap(e.key, false); } turnCharacterOn = (e: KeyboardEvent) => { - this.updateKeyboardBitmap(e.key, true) + // Handle special ctrl + q edge case + if (e.ctrlKey && e.key === 'q') { + this.setState({ isKeyboardModeToggled: false}); + } else { + this.updateKeyboardBitmap(e.key, true); + } } upload = () => { diff --git a/renderer/components/video-feed/ShepherdOverlay.tsx b/renderer/components/video-feed/ShepherdOverlay.tsx new file mode 100644 index 00000000..fbdf6b18 --- /dev/null +++ b/renderer/components/video-feed/ShepherdOverlay.tsx @@ -0,0 +1,12 @@ +import parse from 'html-react-parser'; +import './scoreboard'; +import '../../../static/video-feed/scoreboard.css'; +import '../../../static/video-feed/scoreboard.css'; + +const shepherdScoreHtml = require('../../../static/video-feed/scoreboard.html'); + +export const ShepherdOverlay = () => { + return ( + parse(shepherdScoreHtml) + ); +}; diff --git a/renderer/components/video-feed/VideoFeed.tsx b/renderer/components/video-feed/VideoFeed.tsx index b23dac4e..8d551468 100644 --- a/renderer/components/video-feed/VideoFeed.tsx +++ b/renderer/components/video-feed/VideoFeed.tsx @@ -1,14 +1,17 @@ import React, { useEffect, useState } from 'react'; +import { Button } from 'react-bootstrap'; import OvenPlayer from 'ovenplayer'; import { keyboardButtons } from '../../consts/keyboard-buttons'; import { TooltipButton } from '../TooltipButton'; import { Input, Source } from '../../../protos/protos'; import { ipcRenderer } from 'electron'; import '../../../static/video-feed/video-feed.css'; +import { ShepherdOverlay } from './ShepherdOverlay'; export const VideoFeed = () => { const [isKeyboardModeToggled, setIsKeyboardModeToggled] = useState(false); const [keyboardBitmap, setKeyboardBitmap] = useState(0); + const [shouldShowScoreboard, setShouldShowScoreboard] = useState(true); useEffect(() => { const driverVideoFeed = OvenPlayer.create('driver-video-feed', { @@ -27,7 +30,7 @@ export const VideoFeed = () => { sources: [ { type: 'webRTC', - file: 'ws://64.227.109.107:3333/overhead/stream', + file: 'ws://161.35.224.231:3333/overhead/stream', label: '720p' } ], @@ -44,7 +47,18 @@ export const VideoFeed = () => { }); }, []); - const sendKeyboardInputsToRuntime = () => { + useEffect(() => { + const keyboardConnectionStatus = new Input({ + connected: isKeyboardModeToggled, + axes: [], + buttons: 0, + source: Source.KEYBOARD + }); + + ipcRenderer.send('stateUpdate', [keyboardConnectionStatus], Source.KEYBOARD); + }, [isKeyboardModeToggled]); + + const sendKeyboardInputsToRuntime = (keyboardBitmap: number) => { const keyboard = new Input({ connected: true, axes: [], @@ -93,7 +107,7 @@ export const VideoFeed = () => { } setKeyboardBitmap(newKeyboardBitmap); - sendKeyboardInputsToRuntime(); + sendKeyboardInputsToRuntime(newKeyboardBitmap); }; const turnCharacterOff = (e: KeyboardEvent) => { @@ -101,7 +115,12 @@ export const VideoFeed = () => { }; const turnCharacterOn = (e: KeyboardEvent) => { - updateKeyboardBitmap(e.key, true); + // Handle special ctrl + q edge case + if (e.ctrlKey && e.key === 'q') { + setIsKeyboardModeToggled(false); + } else { + updateKeyboardBitmap(e.key, true); + } }; return ( @@ -115,13 +134,15 @@ export const VideoFeed = () => { disabled={false} bsStyle={isKeyboardModeToggled ? 'info' : 'default'} /> + -
+
diff --git a/renderer/components/video-feed/index.tsx b/renderer/components/video-feed/index.tsx index b47c2744..454dffe4 100644 --- a/renderer/components/video-feed/index.tsx +++ b/renderer/components/video-feed/index.tsx @@ -4,5 +4,5 @@ import { VideoFeed } from './VideoFeed'; ReactDOM.render( , - document.getElementById("Videofeed-react") + document.getElementById("video-feed-window") ); \ No newline at end of file diff --git a/static/video-feed/video-feed.css b/static/video-feed/video-feed.css index 8aff52c2..f6405aa6 100644 --- a/static/video-feed/video-feed.css +++ b/static/video-feed/video-feed.css @@ -1,20 +1,20 @@ -.container { +#header { + display: flex; + justify-content: center; +} + +.video-feed-container { position: relative; height: 95vh; overflow: hidden; + width: 100% } .content { position: absolute; top: 0; - color:#f1f1f1; - background-color: rgba(0, 0, 0, 0.5); width: 100%; height: 100%; - display: flex; - justify-content: center; - align-items: center; - padding: 0; } #overhead-video-feed { @@ -23,8 +23,5 @@ position: absolute; top: 0; left: 0; - border-right: white; - border-bottom: white; - border-width: 1px; } diff --git a/static/video-feed/video.html b/static/video-feed/video.html index 99727e12..0bf6c5db 100644 --- a/static/video-feed/video.html +++ b/static/video-feed/video.html @@ -11,8 +11,7 @@ Video Feed -
-
+
\ No newline at end of file From 2f7d39d8db01f5862003598722e8e54fbdccee2c Mon Sep 17 00:00:00 2001 From: ewc340 Date: Sat, 27 Mar 2021 23:18:20 -0700 Subject: [PATCH 22/42] Add overlay url box --- renderer/components/ConfigBox.tsx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/renderer/components/ConfigBox.tsx b/renderer/components/ConfigBox.tsx index 5dc1b189..6d166d6e 100644 --- a/renderer/components/ConfigBox.tsx +++ b/renderer/components/ConfigBox.tsx @@ -204,12 +204,18 @@ export const ConfigBoxComponent = (props: Props) => { + {defaults.NGROK ? + Overlay URL + + + : null} +

Field Control Settings

- + {!defaults.NGROK ? Field Control IP Address - + : null} = 0 && stationNumber <= 4 ? 'success' : 'error'}> Field Control Station Number From b15de5bac9f2a7f89debed19a9f96219350d752b Mon Sep 17 00:00:00 2001 From: ewc340 Date: Sat, 27 Mar 2021 23:19:11 -0700 Subject: [PATCH 23/42] Style tweaks --- .../components/video-feed/ShepherdOverlay.tsx | 1 - renderer/components/video-feed/VideoFeed.tsx | 10 ++++-- static/video-feed/scoreboard.css | 10 +++--- static/video-feed/scoreboard.html | 4 +-- static/video-feed/video-feed.css | 34 +++++++++++++------ 5 files changed, 37 insertions(+), 22 deletions(-) diff --git a/renderer/components/video-feed/ShepherdOverlay.tsx b/renderer/components/video-feed/ShepherdOverlay.tsx index fbdf6b18..cb483381 100644 --- a/renderer/components/video-feed/ShepherdOverlay.tsx +++ b/renderer/components/video-feed/ShepherdOverlay.tsx @@ -1,7 +1,6 @@ import parse from 'html-react-parser'; import './scoreboard'; import '../../../static/video-feed/scoreboard.css'; -import '../../../static/video-feed/scoreboard.css'; const shepherdScoreHtml = require('../../../static/video-feed/scoreboard.html'); diff --git a/renderer/components/video-feed/VideoFeed.tsx b/renderer/components/video-feed/VideoFeed.tsx index fb3cf979..37e16704 100644 --- a/renderer/components/video-feed/VideoFeed.tsx +++ b/renderer/components/video-feed/VideoFeed.tsx @@ -30,7 +30,7 @@ export const VideoFeed = () => { sources: [ { type: 'webRTC', - file: 'ws://161.35.224.231:3333/overhead/stream', + file: 'ws://64.227.109.107:3333/overhead/stream', label: '720p' } ], @@ -130,6 +130,7 @@ export const VideoFeed = () => { onClick={toggleKeyboardControl} disabled={false} bsStyle={isKeyboardModeToggled ? 'info' : 'default'} + style={{ marginRight: '10px' }} > Toggle Keyboard Control Mode @@ -140,9 +141,12 @@ export const VideoFeed = () => {
-
+
+
+
-
+ +
{shouldShowScoreboard ? ShepherdOverlay() : null}
diff --git a/static/video-feed/scoreboard.css b/static/video-feed/scoreboard.css index 185d8104..10694fdf 100644 --- a/static/video-feed/scoreboard.css +++ b/static/video-feed/scoreboard.css @@ -1,16 +1,15 @@ body { - /* background-color: rgb(0, 0, 225); */ background-size: cover; - height: 100vh; + /* height: 100vh; */ width: 100%; + background-color: transparent; } - p { padding-left: 5px; } -.outer-container { +.shepherd-outer-container { position: absolute; top: 0; right: 0; @@ -18,7 +17,7 @@ p { height: 100%; } -.container { +.shepherd-container { display: flex; justify-content: space-evenly; align-items: stretch; @@ -67,7 +66,6 @@ p { font-size: 200% } - .stagename { height: calc(10vh); width: calc(45vh) diff --git a/static/video-feed/scoreboard.html b/static/video-feed/scoreboard.html index b39edaad..1256b2a8 100644 --- a/static/video-feed/scoreboard.html +++ b/static/video-feed/scoreboard.html @@ -13,8 +13,8 @@ -
-
+
+
diff --git a/static/video-feed/video-feed.css b/static/video-feed/video-feed.css index f6405aa6..67760fb4 100644 --- a/static/video-feed/video-feed.css +++ b/static/video-feed/video-feed.css @@ -1,27 +1,41 @@ #header { display: flex; - justify-content: center; + /* justify-content: center; */ + /* padding: 5px; */ + position: absolute; + bottom: 0; + right: 0; + z-index: 10; } .video-feed-container { - position: relative; - height: 95vh; - overflow: hidden; - width: 100% + /* position: relative; */ + height: 100vh; + /* overflow: hidden; */ + width: 100%; } -.content { - position: absolute; - top: 0; +.overhead-video-feed-container { width: 100%; height: 100%; + position: absolute; + overflow: hidden; + top: 0; + left: 0; } #overhead-video-feed { width: 30%; height: 30%; + /* position: absolute; */ + /* top: 0; + left: 0; */ +} + +#shepherd-overlay { position: absolute; top: 0; - left: 0; + /* margin-top: 50px; */ + width: 100%; + height: 100%; } - From a034d644bc0f75dd83b411fe904ef3b653a5a4c0 Mon Sep 17 00:00:00 2001 From: ewc340 Date: Sun, 28 Mar 2021 00:05:22 -0700 Subject: [PATCH 24/42] Fix sandstorm showing --- renderer/components/video-feed/VideoFeed.tsx | 4 +--- static/video-feed/scoreboard.css | 2 +- static/video-feed/video-feed.css | 23 +++++--------------- 3 files changed, 8 insertions(+), 21 deletions(-) diff --git a/renderer/components/video-feed/VideoFeed.tsx b/renderer/components/video-feed/VideoFeed.tsx index 37e16704..082a4497 100644 --- a/renderer/components/video-feed/VideoFeed.tsx +++ b/renderer/components/video-feed/VideoFeed.tsx @@ -141,9 +141,7 @@ export const VideoFeed = () => {
-
-
-
+
diff --git a/static/video-feed/scoreboard.css b/static/video-feed/scoreboard.css index 10694fdf..a082e875 100644 --- a/static/video-feed/scoreboard.css +++ b/static/video-feed/scoreboard.css @@ -1,6 +1,6 @@ body { background-size: cover; - /* height: 100vh; */ + height: 100vh; width: 100%; background-color: transparent; } diff --git a/static/video-feed/video-feed.css b/static/video-feed/video-feed.css index 67760fb4..92701fbf 100644 --- a/static/video-feed/video-feed.css +++ b/static/video-feed/video-feed.css @@ -1,7 +1,6 @@ #header { display: flex; - /* justify-content: center; */ - /* padding: 5px; */ + padding: 8px; position: absolute; bottom: 0; right: 0; @@ -9,33 +8,23 @@ } .video-feed-container { - /* position: relative; */ - height: 100vh; - /* overflow: hidden; */ - width: 100%; -} - -.overhead-video-feed-container { - width: 100%; + position: relative; height: 100%; - position: absolute; overflow: hidden; - top: 0; - left: 0; + width: 100%; } #overhead-video-feed { width: 30%; height: 30%; - /* position: absolute; */ - /* top: 0; - left: 0; */ + position: absolute; + top: 0; + left: 0; } #shepherd-overlay { position: absolute; top: 0; - /* margin-top: 50px; */ width: 100%; height: 100%; } From fe7dd1934ef8b895972bd89934ed62b4ac9ceee0 Mon Sep 17 00:00:00 2001 From: ewc340 Date: Tue, 30 Mar 2021 16:52:54 -0700 Subject: [PATCH 25/42] Fix video feed button click, turn off scoreboard overlay toggling, and change overlay default html --- renderer/components/video-feed/VideoFeed.tsx | 27 ++++++++++---------- static/video-feed/scoreboard.css | 4 +-- static/video-feed/scoreboard.html | 8 +++--- static/video-feed/video-feed.css | 18 ++++++------- 4 files changed, 29 insertions(+), 28 deletions(-) diff --git a/renderer/components/video-feed/VideoFeed.tsx b/renderer/components/video-feed/VideoFeed.tsx index 082a4497..5a96cc03 100644 --- a/renderer/components/video-feed/VideoFeed.tsx +++ b/renderer/components/video-feed/VideoFeed.tsx @@ -11,7 +11,7 @@ import { ShepherdOverlay } from './ShepherdOverlay'; export const VideoFeed = () => { const [isKeyboardModeToggled, setIsKeyboardModeToggled] = useState(false); const [keyboardBitmap, setKeyboardBitmap] = useState(0); - const [shouldShowScoreboard, setShouldShowScoreboard] = useState(true); + // const [shouldShowScoreboard, setShouldShowScoreboard] = useState(true); useEffect(() => { const driverVideoFeed = OvenPlayer.create('driver-video-feed', { @@ -124,7 +124,17 @@ export const VideoFeed = () => { return ( <> -