Add keyboard and mouse support

Keyboard and mouse controls will now work if you use the kbMouseSupport parameter in the config for Libretro cores. Be aware that capturing mouse and keyboard controls properly is only possible in fullscreen mode.

Note: In the case of DOSBox, a virtual filesystem handler is not yet implemented, thus each game state will be shared between all rooms (DOS game instances) of CloudRetro.

Author: sergystepanov

pkg/api/user.go

@@ -12,8 +12,9 @@ type (
 		PlayerIndex int    `json:"player_index"`
 	}
 	GameStartUserResponse struct {
-		RoomId string        `json:"roomId"`
-		Av     *AppVideoInfo `json:"av"`
+		RoomId  string        `json:"roomId"`
+		Av      *AppVideoInfo `json:"av"`
+		KbMouse bool          `json:"kb_mouse"`
 	}
 	IceServer struct {
 		Urls       string `json:"urls,omitempty"`

pkg/api/worker.go

@@ -33,8 +33,9 @@ type (
 	}
 	StartGameResponse struct {
 		Room
-		AV     *AppVideoInfo `json:"av"`
-		Record bool
+		AV      *AppVideoInfo `json:"av"`
+		Record  bool          `json:"record"`
+		KbMouse bool          `json:"kb_mouse"`
 	}
 	RecordGameRequest[T Id] struct {
 		StatefulRoom[T]

pkg/config/config.yaml

@@ -185,11 +185,13 @@ emulator:
       #   - isGlAllowed (bool)
       #   - usesLibCo (bool)
       #   - hasMultitap (bool) -- (removed)
-      #   - coreAspectRatio (bool) -- correct the aspect ratio on the client with the info from the core.
+      #   - coreAspectRatio (bool) -- (deprecated) correct the aspect ratio on the client with the info from the core.
       #   - hid (map[int][]int)
       #       A list of device IDs to bind to the input ports.
+      #       Can be seen in human readable form in the console when worker.debug is enabled.
       #       Some cores allow binding multiple devices to a single port (DosBox), but typically,
       #       you should bind just one device to one port.
+      #   - kbMouseSupport (bool) -- (temp) a flag if the core needs the keyboard and mouse on the client
       #   - vfr (bool)
       #    (experimental)
       #       Enable variable frame rate only for cores that can't produce a constant frame rate.
@@ -213,7 +215,6 @@ emulator:
             mgba_audio_low_pass_filter: enabled
             mgba_audio_low_pass_range: 40
         pcsx:
-          coreAspectRatio: true
           lib: pcsx_rearmed_libretro
           roms: [ "cue", "chd" ]
           # example of folder override
@@ -227,7 +228,6 @@ emulator:
         # https://docs.libretro.com/library/fbneo/
         mame:
           lib: fbneo_libretro
-          coreAspectRatio: true
           roms: [ "zip" ]
         nes:
           lib: nestopia_libretro
@@ -280,7 +280,7 @@ encoder:
     # see: https://trac.ffmpeg.org/wiki/Encode/H.264
     h264:
       # Constant Rate Factor (CRF) 0-51 (default: 23)
-      crf: 26
+      crf: 23
       # ultrafast, superfast, veryfast, faster, fast, medium, slow, slower, veryslow, placebo
       preset: superfast
       # baseline, main, high, high10, high422, high444

pkg/config/emulator.go

@@ -48,6 +48,7 @@ type LibretroCoreConfig struct {
 	Height          int
 	Hid             map[int][]int
 	IsGlAllowed     bool
+	KbMouseSupport  bool
 	Lib             string
 	Options         map[string]string
 	Options4rom     map[string]map[string]string // <(^_^)>

pkg/coordinator/userapi.go

@@ -37,6 +37,6 @@ func (u *User) SendWebrtcOffer(sdp string) { u.Notify(api.WebrtcOffer, sdp) }
 func (u *User) SendWebrtcIceCandidate(candidate string) { u.Notify(api.WebrtcIce, candidate) }
 
 // StartGame signals the user that everything is ready to start a game.
-func (u *User) StartGame(av *api.AppVideoInfo) {
-	u.Notify(api.StartGame, api.GameStartUserResponse{RoomId: u.w.RoomId, Av: av})
+func (u *User) StartGame(av *api.AppVideoInfo, kbMouse bool) {
+	u.Notify(api.StartGame, api.GameStartUserResponse{RoomId: u.w.RoomId, Av: av, KbMouse: kbMouse})
 }

pkg/coordinator/userhandlers.go

@@ -56,7 +56,7 @@ func (u *User) HandleStartGame(rq api.GameStartUserRequest, launcher games.Launc
 		return
 	}
 	u.log.Info().Str("id", startGameResp.Rid).Msg("Received room response from worker")
-	u.StartGame(startGameResp.AV)
+	u.StartGame(startGameResp.AV, startGameResp.KbMouse)
 
 	// send back recording status
 	if conf.Recording.Enabled && rq.Record {

pkg/network/webrtc/webrtc.go

@@ -81,11 +81,15 @@ func (p *Peer) NewCall(vCodec, aCodec string, onICECandidate func(ice any)) (sdp
 	p.log.Debug().Msgf("Added [%s] track", audio.Codec().MimeType)
 	p.a = audio
 
-	// plug in the [data] channel (in and out)
-	if err = p.addDataChannel("data"); err != nil {
+	err = p.AddChannel("data", func(data []byte) {
+		if len(data) == 0 || p.OnMessage == nil {
+			return
+		}
+		p.OnMessage(data)
+	})
+	if err != nil {
 		return "", err
 	}
-	p.log.Debug().Msg("Added [data] chan")
 
 	p.conn.OnICEConnectionStateChange(p.handleICEState(func() { p.log.Info().Msg("Connected") }))
 	// Stream provider supposes to send offer
@@ -221,6 +225,19 @@ func (p *Peer) AddCandidate(candidate string, decoder Decoder) error {
 	return nil
 }
 
+func (p *Peer) AddChannel(label string, onMessage func([]byte)) error {
+	ch, err := p.addDataChannel(label)
+	if err != nil {
+		return err
+	}
+	if label == "data" {
+		p.d = ch
+	}
+	ch.OnMessage(func(m webrtc.DataChannelMessage) { onMessage(m.Data) })
+	p.log.Debug().Msgf("Added [%v] chan", label)
+	return nil
+}
+
 func (p *Peer) Disconnect() {
 	if p.conn == nil {
 		return
@@ -232,29 +249,19 @@ func (p *Peer) Disconnect() {
 	p.log.Debug().Msg("WebRTC stop")
 }
 
-// addDataChannel creates a new WebRTC data channel for user input.
+// addDataChannel creates new WebRTC data channel.
 // Default params -- ordered: true, negotiated: false.
-func (p *Peer) addDataChannel(label string) error {
+func (p *Peer) addDataChannel(label string) (*webrtc.DataChannel, error) {
 	ch, err := p.conn.CreateDataChannel(label, nil)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	ch.OnOpen(func() {
-		p.log.Debug().Str("label", ch.Label()).Uint16("id", *ch.ID()).
-			Msg("Data channel [input] opened")
+		p.log.Debug().Uint16("id", *ch.ID()).Msgf("Data channel [%v] opened", ch.Label())
 	})
 	ch.OnError(p.logx)
-	ch.OnMessage(func(m webrtc.DataChannelMessage) {
-		if len(m.Data) == 0 {
-			return
-		}
-		if p.OnMessage != nil {
-			p.OnMessage(m.Data)
-		}
-	})
-	p.d = ch
-	ch.OnClose(func() { p.log.Debug().Msg("Data channel [input] has been closed") })
-	return nil
+	ch.OnClose(func() { p.log.Debug().Msgf("Data channel [%v] has been closed", ch.Label()) })
+	return ch, nil
 }
 
 func (p *Peer) logx(err error) { p.log.Error().Err(err) }

pkg/worker/caged/app/app.go

@@ -13,7 +13,8 @@ type App interface {
 	SetAudioCb(func(Audio))
 	SetVideoCb(func(Video))
 	SetDataCb(func([]byte))
-	SendControl(port int, data []byte)
+	Input(port int, device byte, data []byte)
+	KbMouseSupport() bool
 }
 
 type Audio struct {

pkg/worker/caged/caged.go

@@ -15,6 +15,12 @@ type Manager struct {
 	log  *logger.Logger
 }
 
+const (
+	RetroPad = libretro.RetroPad
+	Keyboard = libretro.Keyboard
+	Mouse    = libretro.Mouse
+)
+
 type ModName string
 
 const Libretro ModName = "libretro"

pkg/worker/caged/libretro/caged.go

@@ -79,15 +79,16 @@ func (c *Caged) EnableCloudStorage(uid string, storage cloud.Storage) {
 	}
 }
 
-func (c *Caged) AspectEnabled() bool               { return c.base.nano.Aspect }
-func (c *Caged) AspectRatio() float32              { return c.base.AspectRatio() }
-func (c *Caged) PixFormat() uint32                 { return c.Emulator.PixFormat() }
-func (c *Caged) Rotation() uint                    { return c.Emulator.Rotation() }
-func (c *Caged) AudioSampleRate() int              { return c.Emulator.AudioSampleRate() }
-func (c *Caged) ViewportSize() (int, int)          { return c.base.ViewportSize() }
-func (c *Caged) Scale() float64                    { return c.Emulator.Scale() }
-func (c *Caged) SendControl(port int, data []byte) { c.base.Input(port, data) }
-func (c *Caged) Start()                            { go c.Emulator.Start() }
-func (c *Caged) SetSaveOnClose(v bool)             { c.base.SaveOnClose = v }
-func (c *Caged) SetSessionId(name string)          { c.base.SetSessionId(name) }
-func (c *Caged) Close()                            { c.Emulator.Close() }
+func (c *Caged) AspectEnabled() bool              { return c.base.nano.Aspect }
+func (c *Caged) AspectRatio() float32             { return c.base.AspectRatio() }
+func (c *Caged) PixFormat() uint32                { return c.Emulator.PixFormat() }
+func (c *Caged) Rotation() uint                   { return c.Emulator.Rotation() }
+func (c *Caged) AudioSampleRate() int             { return c.Emulator.AudioSampleRate() }
+func (c *Caged) ViewportSize() (int, int)         { return c.base.ViewportSize() }
+func (c *Caged) Scale() float64                   { return c.Emulator.Scale() }
+func (c *Caged) Input(p int, d byte, data []byte) { c.base.Input(p, d, data) }
+func (c *Caged) KbMouseSupport() bool             { return c.base.KbMouseSupport() }
+func (c *Caged) Start()                           { go c.Emulator.Start() }
+func (c *Caged) SetSaveOnClose(v bool)            { c.base.SaveOnClose = v }
+func (c *Caged) SetSessionId(name string)         { c.base.SetSessionId(name) }
+func (c *Caged) Close()                           { c.Emulator.Close() }