Add initial automatic aspect ratio change

Depending on the configuration param coreAspectRatio, video streams may have automatic aspect ratio correction in the browser with the value provided by the cores themselves.

pkg/api/api.go

@@ -62,8 +62,9 @@ func (o *Out) GetPayload() any         { return o.Payload }
 
 // Packet codes:
 //
-//	x, 1xx - user codes
-//	2xx - worker codes
+// x, 1xx - user codes
+// 15x - webrtc data exchange codes
+// 2xx - worker codes
 const (
 	CheckLatency     PT = 3
 	InitSession      PT = 4
@@ -84,6 +85,7 @@ const (
 	CloseRoom        PT = 202
 	IceCandidate        = WebrtcIce
 	TerminateSession PT = 204
+	AppVideoChange   PT = 150
 )
 
 func (p PT) String() string {
@@ -124,6 +126,8 @@ func (p PT) String() string {
 		return "CloseRoom"
 	case TerminateSession:
 		return "TerminateSession"
+	case AppVideoChange:
+		return "AppVideoChange"
 	default:
 		return "Unknown"
 	}
@@ -160,3 +164,5 @@ func UnwrapChecked[T any](bytes []byte, err error) (*T, error) {
 	}
 	return Unwrap[T](bytes), nil
 }
+
+func Wrap(t any) ([]byte, error) { return json.Marshal(t) }

pkg/api/user.go

@@ -11,6 +11,10 @@ type (
 		RecordUser  string `json:"record_user,omitempty"`
 		PlayerIndex int    `json:"player_index"`
 	}
+	GameStartUserResponse struct {
+		RoomId string        `json:"roomId"`
+		Av     *AppVideoInfo `json:"av"`
+	}
 	IceServer struct {
 		Urls       string `json:"urls,omitempty"`
 		Username   string `json:"username,omitempty"`

pkg/api/worker.go

@@ -33,6 +33,7 @@ type (
 	}
 	StartGameResponse struct {
 		Room
+		AV     *AppVideoInfo `json:"av"`
 		Record bool
 	}
 	RecordGameRequest[T Id] struct {
@@ -59,4 +60,10 @@ type (
 		Stateful[T]
 	}
 	WebrtcInitResponse string
+
+	AppVideoInfo struct {
+		W int     `json:"w"`
+		H int     `json:"h"`
+		A float32 `json:"a"`
+	}
 )

pkg/config/config.yaml

@@ -118,14 +118,6 @@ emulator:
   # (removed)
   threads: 0
 
-  aspectRatio:
-    # enable aspect ratio changing
-    # (experimental)
-    keep: false
-    # recalculate emulator game frame size to the given WxH
-    width: 320
-    height: 240
-
   # enable autosave for emulator states if set to a non-zero value of seconds
   autosaveSec: 0
 
@@ -189,6 +181,7 @@ emulator:
       #   - isGlAllowed (bool)
       #   - usesLibCo (bool)
       #   - hasMultitap (bool)
+      #   - coreAspectRatio (bool) -- correct the aspect ratio on the client with the info from the core.
       #   - vfr (bool)
       #    (experimental)
       #       Enable variable frame rate only for cores that can't produce a constant frame rate.
@@ -210,6 +203,7 @@ emulator:
             mgba_audio_low_pass_filter: enabled
             mgba_audio_low_pass_range: 40
         pcsx:
+          coreAspectRatio: true
           lib: pcsx_rearmed_libretro
           roms: [ "cue", "chd" ]
           # example of folder override
@@ -227,6 +221,8 @@ emulator:
         nes:
           lib: nestopia_libretro
           roms: [ "nes" ]
+          options:
+            nestopia_aspect: "uncorrected"
         snes:
           lib: snes9x_libretro
           roms: [ "smc", "sfc", "swc", "fig", "bs" ]

pkg/config/emulator.go

@@ -8,11 +8,6 @@ import (
 
 type Emulator struct {
 	Threads     int
-	AspectRatio struct {
-		Keep   bool
-		Width  int
-		Height int
-	}
 	Storage     string
 	LocalPath   string
 	Libretro    LibretroConfig
@@ -44,20 +39,21 @@ type LibretroRepoConfig struct {
 }
 
 type LibretroCoreConfig struct {
-	AltRepo       bool
-	AutoGlContext bool // hack: keep it here to pass it down the emulator
-	Folder        string
-	Hacks         []string
-	HasMultitap   bool
-	Height        int
-	IsGlAllowed   bool
-	Lib           string
-	Options       map[string]string
-	Roms          []string
-	Scale         float64
-	UsesLibCo     bool
-	VFR           bool
-	Width         int
+	AltRepo         bool
+	AutoGlContext   bool // hack: keep it here to pass it down the emulator
+	CoreAspectRatio bool
+	Folder          string
+	Hacks           []string
+	HasMultitap     bool
+	Height          int
+	IsGlAllowed     bool
+	Lib             string
+	Options         map[string]string
+	Roms            []string
+	Scale           float64
+	UsesLibCo       bool
+	VFR             bool
+	Width           int
 }
 
 type CoreInfo struct {

pkg/coordinator/userapi.go

@@ -37,4 +37,6 @@ func (u *User) SendWebrtcOffer(sdp string) { u.Notify(api.WebrtcOffer, sdp) }
 func (u *User) SendWebrtcIceCandidate(candidate string) { u.Notify(api.WebrtcIce, candidate) }
 
 // StartGame signals the user that everything is ready to start a game.
-func (u *User) StartGame() { u.Notify(api.StartGame, u.w.RoomId) }
+func (u *User) StartGame(av *api.AppVideoInfo) {
+	u.Notify(api.StartGame, api.GameStartUserResponse{RoomId: u.w.RoomId, Av: av})
+}

pkg/coordinator/userhandlers.go

@@ -56,7 +56,7 @@ func (u *User) HandleStartGame(rq api.GameStartUserRequest, launcher games.Launc
 		return
 	}
 	u.log.Info().Str("id", startGameResp.Rid).Msg("Received room response from worker")
-	u.StartGame()
+	u.StartGame(startGameResp.AV)
 
 	// send back recording status
 	if conf.Recording.Enabled && rq.Record {

pkg/network/webrtc/webrtc.go

@@ -32,13 +32,13 @@ func (p *Peer) NewCall(vCodec, aCodec string, onICECandidate func(ice any)) (sdp
 	if p.conn != nil && p.conn.ConnectionState() == webrtc.PeerConnectionStateConnected {
 		return
 	}
-	p.log.Info().Msg("WebRTC start")
+	p.log.Debug().Msg("WebRTC start")
 	if p.conn, err = p.api.NewPeer(); err != nil {
-		return "", err
+		return
 	}
 	p.conn.OnICECandidate(p.handleICECandidate(onICECandidate))
 	// plug in the [video] track (out)
-	video, err := newTrack("video", "game-video", vCodec)
+	video, err := newTrack("video", "video", vCodec)
 	if err != nil {
 		return "", err
 	}
@@ -49,7 +49,7 @@ func (p *Peer) NewCall(vCodec, aCodec string, onICECandidate func(ice any)) (sdp
 	p.log.Debug().Msgf("Added [%s] track", video.Codec().MimeType)
 
 	// plug in the [audio] track (out)
-	audio, err := newTrack("audio", "game-audio", aCodec)
+	audio, err := newTrack("audio", "audio", aCodec)
 	if err != nil {
 		return "", err
 	}
@@ -59,21 +59,19 @@ func (p *Peer) NewCall(vCodec, aCodec string, onICECandidate func(ice any)) (sdp
 	p.log.Debug().Msgf("Added [%s] track", audio.Codec().MimeType)
 	p.a = audio
 
-	// plug in the [input] data channel (in)
-	if err = p.addInputChannel("game-input"); err != nil {
+	// plug in the [data] channel (in and out)
+	if err = p.addDataChannel("data"); err != nil {
 		return "", err
 	}
-	p.log.Debug().Msg("Added [input/bytes] chan")
+	p.log.Debug().Msg("Added [data] chan")
 
-	p.conn.OnICEConnectionStateChange(p.handleICEState(func() {
-		p.log.Info().Msg("Start streaming")
-	}))
+	p.conn.OnICEConnectionStateChange(p.handleICEState(func() { p.log.Info().Msg("Connected") }))
 	// Stream provider supposes to send offer
 	offer, err := p.conn.CreateOffer(nil)
 	if err != nil {
 		return "", err
 	}
-	p.log.Info().Msg("Created Offer")
+	p.log.Debug().Msg("Created Offer")
 
 	err = p.conn.SetLocalDescription(offer)
 	if err != nil {
@@ -210,15 +208,16 @@ func (p *Peer) Disconnect() {
 	p.log.Debug().Msg("WebRTC stop")
 }
 
-// addInputChannel creates a new WebRTC data channel for user input.
+// addDataChannel creates a new WebRTC data channel for user input.
 // Default params -- ordered: true, negotiated: false.
-func (p *Peer) addInputChannel(label string) error {
+func (p *Peer) addDataChannel(label string) error {
 	ch, err := p.conn.CreateDataChannel(label, nil)
 	if err != nil {
 		return err
 	}
 	ch.OnOpen(func() {
-		p.log.Debug().Str("label", ch.Label()).Uint16("id", *ch.ID()).Msg("Data channel [input] opened")
+		p.log.Debug().Str("label", ch.Label()).Uint16("id", *ch.ID()).
+			Msg("Data channel [input] opened")
 	})
 	ch.OnError(p.logx)
 	ch.OnMessage(func(m webrtc.DataChannelMessage) {

pkg/worker/caged/app/app.go

@@ -2,13 +2,16 @@ package app
 
 type App interface {
 	AudioSampleRate() int
+	AspectRatio() float32
+	AspectEnabled() bool
 	Init() error
 	ViewportSize() (int, int)
 	Start()
 	Close()
 
 	SetAudioCb(func(Audio))
 	SetVideoCb(func(Video))
+	SetDataCb(func([]byte))
 	SendControl(port int, data []byte)
 }
 

pkg/worker/caged/libretro/caged.go

@@ -14,9 +14,6 @@ type Caged struct {
 	base *Frontend // maintains the root for mad embedding
 	conf CagedConf
 	log  *logger.Logger
-	w, h int
-
-	OnSysInfoChange func()
 }
 
 type CagedConf struct {
@@ -78,6 +75,8 @@ func (c *Caged) EnableCloudStorage(uid string, storage cloud.Storage) {
 	}
 }
 
+func (c *Caged) AspectEnabled() bool               { return c.base.nano.Aspect }
+func (c *Caged) AspectRatio() float32              { return c.base.AspectRatio() }
 func (c *Caged) PixFormat() uint32                 { return c.Emulator.PixFormat() }
 func (c *Caged) Rotation() uint                    { return c.Emulator.Rotation() }
 func (c *Caged) AudioSampleRate() int              { return c.Emulator.AudioSampleRate() }

pkg/worker/caged/libretro/frontend.go

@@ -3,7 +3,6 @@ package libretro
 import (
 	"errors"
 	"fmt"
-	"math"
 	"path/filepath"
 	"sync"
 	"sync/atomic"
@@ -20,6 +19,7 @@ import (
 type Emulator interface {
 	SetAudioCb(func(app.Audio))
 	SetVideoCb(func(app.Video))
+	SetDataCb(func([]byte))
 	LoadCore(name string)
 	LoadGame(path string) error
 	FPS() int
@@ -57,6 +57,7 @@ type Frontend struct {
 	log     *logger.Logger
 	nano    *nanoarch.Nanoarch
 	onAudio func(app.Audio)
+	onData  func([]byte)
 	onVideo func(app.Video)
 	storage Storage
 	scale   float64
@@ -90,6 +91,7 @@ const (
 var (
 	audioPool sync.Pool
 	noAudio   = func(app.Audio) {}
+	noData    = func([]byte) {}
 	noVideo   = func(app.Video) {}
 	videoPool sync.Pool
 )
@@ -135,6 +137,7 @@ func NewFrontend(conf config.Emulator, log *logger.Logger) (*Frontend, error) {
 		input:   NewGameSessionInput(),
 		log:     log,
 		onAudio: noAudio,
+		onData:  noData,
 		onVideo: noVideo,
 		storage: store,
 		th:      conf.Threads,
@@ -153,14 +156,15 @@ func NewFrontend(conf config.Emulator, log *logger.Logger) (*Frontend, error) {
 func (f *Frontend) LoadCore(emu string) {
 	conf := f.conf.GetLibretroCoreConfig(emu)
 	meta := nanoarch.Metadata{
-		AutoGlContext: conf.AutoGlContext,
-		Hacks:         conf.Hacks,
-		HasMultitap:   conf.HasMultitap,
-		HasVFR:        conf.VFR,
-		IsGlAllowed:   conf.IsGlAllowed,
-		LibPath:       conf.Lib,
-		Options:       conf.Options,
-		UsesLibCo:     conf.UsesLibCo,
+		AutoGlContext:   conf.AutoGlContext,
+		Hacks:           conf.Hacks,
+		HasMultitap:     conf.HasMultitap,
+		HasVFR:          conf.VFR,
+		IsGlAllowed:     conf.IsGlAllowed,
+		LibPath:         conf.Lib,
+		Options:         conf.Options,
+		UsesLibCo:       conf.UsesLibCo,
+		CoreAspectRatio: conf.CoreAspectRatio,
 	}
 	f.mu.Lock()
 	scale := 1.0
@@ -224,6 +228,13 @@ func (f *Frontend) SetVideoChangeCb(fn func()) { f.nano.OnSystemAvInfo = fn }
 
 func (f *Frontend) Start() {
 	f.log.Debug().Msgf("frontend start")
+	if f.nano.Stopped.Load() {
+		f.log.Warn().Msgf("frontend stopped during the start")
+		f.mui.Lock()
+		defer f.mui.Unlock()
+		f.Shutdown()
+		return
+	}
 
 	f.mui.Lock()
 	f.done = make(chan struct{})
@@ -269,6 +280,7 @@ func (f *Frontend) Start() {
 	}
 }
 
+func (f *Frontend) AspectRatio() float32          { return f.nano.AspectRatio() }
 func (f *Frontend) AudioSampleRate() int          { return f.nano.AudioSampleRate() }
 func (f *Frontend) FPS() int                      { return f.nano.VideoFramerate() }
 func (f *Frontend) Flipped() bool                 { return f.nano.IsGL() }
@@ -286,6 +298,7 @@ func (f *Frontend) SaveGameState() error          { return f.Save() }
 func (f *Frontend) Scale() float64                { return f.scale }
 func (f *Frontend) SetAudioCb(cb func(app.Audio)) { f.onAudio = cb }
 func (f *Frontend) SetSessionId(name string)      { f.storage.SetMainSaveName(name) }
+func (f *Frontend) SetDataCb(cb func([]byte))     { f.onData = cb }
 func (f *Frontend) SetVideoCb(ff func(app.Video)) { f.onVideo = ff }
 func (f *Frontend) Tick()                         { f.mu.Lock(); f.nano.Run(); f.mu.Unlock() }
 func (f *Frontend) ToggleMultitap()               { f.nano.ToggleMultitap() }
@@ -296,18 +309,6 @@ func (f *Frontend) ViewportCalc() (nw int, nh int) {
 	w, h := f.FrameSize()
 	nw, nh = w, h
 
-	aspect, aw, ah := f.conf.AspectRatio.Keep, f.conf.AspectRatio.Width, f.conf.AspectRatio.Height
-	// calc the aspect ratio
-	if aspect && aw > 0 && ah > 0 {
-		ratio := float64(w) / float64(ah)
-		nw = int(math.Round(float64(ah)*ratio/2) * 2)
-		nh = ah
-		if nw > aw {
-			nw = aw
-			nh = int(math.Round(float64(aw)/ratio/2) * 2)
-		}
-	}
-
 	if f.IsPortrait() {
 		nw, nh = nh, nw
 	}