From 2e91feb861de737fd4f28b08051097cd84c9dddb Mon Sep 17 00:00:00 2001 From: Sergey Stepanov Date: Wed, 1 Nov 2023 01:21:58 +0300 Subject: [PATCH] Add initial automatic aspect ratio change Depending on the configuration param coreAspectRatio, video streams may have automatic aspect ratio correction in the browser with the value provided by the cores themselves. --- pkg/api/api.go | 10 +- pkg/api/user.go | 4 + pkg/api/worker.go | 7 ++ pkg/config/config.yaml | 12 +-- pkg/config/emulator.go | 34 +++---- pkg/coordinator/userapi.go | 4 +- pkg/coordinator/userhandlers.go | 2 +- pkg/network/webrtc/webrtc.go | 27 +++--- pkg/worker/caged/app/app.go | 3 + pkg/worker/caged/libretro/caged.go | 5 +- pkg/worker/caged/libretro/frontend.go | 43 ++++----- .../caged/libretro/nanoarch/nanoarch.go | 91 +++++++++++-------- pkg/worker/coordinatorhandlers.go | 44 ++++++--- pkg/worker/media/media.go | 8 ++ pkg/worker/room/room.go | 1 + web/css/main.css | 29 +++--- web/index.html | 36 ++++---- web/js/api/api.js | 13 ++- web/js/controller.js | 14 ++- web/js/event/event.js | 2 + web/js/network/webrtc.js | 27 +++--- web/js/stream/stream.js | 49 +++++++++- 22 files changed, 297 insertions(+), 168 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index 6a8e96f64..f85daa8de 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -62,8 +62,9 @@ func (o *Out) GetPayload() any { return o.Payload } // Packet codes: // -// x, 1xx - user codes -// 2xx - worker codes +// x, 1xx - user codes +// 15x - webrtc data exchange codes +// 2xx - worker codes const ( CheckLatency PT = 3 InitSession PT = 4 @@ -84,6 +85,7 @@ const ( CloseRoom PT = 202 IceCandidate = WebrtcIce TerminateSession PT = 204 + AppVideoChange PT = 150 ) func (p PT) String() string { @@ -124,6 +126,8 @@ func (p PT) String() string { return "CloseRoom" case TerminateSession: return "TerminateSession" + case AppVideoChange: + return "AppVideoChange" default: return "Unknown" } @@ -160,3 +164,5 @@ func UnwrapChecked[T any](bytes []byte, err error) (*T, error) { } return Unwrap[T](bytes), nil } + +func Wrap(t any) ([]byte, error) { return json.Marshal(t) } diff --git a/pkg/api/user.go b/pkg/api/user.go index 84d8ee62f..aef4305dc 100644 --- a/pkg/api/user.go +++ b/pkg/api/user.go @@ -11,6 +11,10 @@ type ( RecordUser string `json:"record_user,omitempty"` PlayerIndex int `json:"player_index"` } + GameStartUserResponse struct { + RoomId string `json:"roomId"` + Av *AppVideoInfo `json:"av"` + } IceServer struct { Urls string `json:"urls,omitempty"` Username string `json:"username,omitempty"` diff --git a/pkg/api/worker.go b/pkg/api/worker.go index b206c5c17..045fa4296 100644 --- a/pkg/api/worker.go +++ b/pkg/api/worker.go @@ -33,6 +33,7 @@ type ( } StartGameResponse struct { Room + AV *AppVideoInfo `json:"av"` Record bool } RecordGameRequest[T Id] struct { @@ -59,4 +60,10 @@ type ( Stateful[T] } WebrtcInitResponse string + + AppVideoInfo struct { + W int `json:"w"` + H int `json:"h"` + A float32 `json:"a"` + } ) diff --git a/pkg/config/config.yaml b/pkg/config/config.yaml index 3ffc0891d..13104af55 100644 --- a/pkg/config/config.yaml +++ b/pkg/config/config.yaml @@ -118,14 +118,6 @@ emulator: # (removed) threads: 0 - aspectRatio: - # enable aspect ratio changing - # (experimental) - keep: false - # recalculate emulator game frame size to the given WxH - width: 320 - height: 240 - # enable autosave for emulator states if set to a non-zero value of seconds autosaveSec: 0 @@ -189,6 +181,7 @@ emulator: # - isGlAllowed (bool) # - usesLibCo (bool) # - hasMultitap (bool) + # - coreAspectRatio (bool) -- correct the aspect ratio on the client with the info from the core. # - vfr (bool) # (experimental) # Enable variable frame rate only for cores that can't produce a constant frame rate. @@ -210,6 +203,7 @@ emulator: mgba_audio_low_pass_filter: enabled mgba_audio_low_pass_range: 40 pcsx: + coreAspectRatio: true lib: pcsx_rearmed_libretro roms: [ "cue", "chd" ] # example of folder override @@ -227,6 +221,8 @@ emulator: nes: lib: nestopia_libretro roms: [ "nes" ] + options: + nestopia_aspect: "uncorrected" snes: lib: snes9x_libretro roms: [ "smc", "sfc", "swc", "fig", "bs" ] diff --git a/pkg/config/emulator.go b/pkg/config/emulator.go index 84e57ed87..da8f5b2ea 100644 --- a/pkg/config/emulator.go +++ b/pkg/config/emulator.go @@ -8,11 +8,6 @@ import ( type Emulator struct { Threads int - AspectRatio struct { - Keep bool - Width int - Height int - } Storage string LocalPath string Libretro LibretroConfig @@ -44,20 +39,21 @@ type LibretroRepoConfig struct { } type LibretroCoreConfig struct { - AltRepo bool - AutoGlContext bool // hack: keep it here to pass it down the emulator - Folder string - Hacks []string - HasMultitap bool - Height int - IsGlAllowed bool - Lib string - Options map[string]string - Roms []string - Scale float64 - UsesLibCo bool - VFR bool - Width int + AltRepo bool + AutoGlContext bool // hack: keep it here to pass it down the emulator + CoreAspectRatio bool + Folder string + Hacks []string + HasMultitap bool + Height int + IsGlAllowed bool + Lib string + Options map[string]string + Roms []string + Scale float64 + UsesLibCo bool + VFR bool + Width int } type CoreInfo struct { diff --git a/pkg/coordinator/userapi.go b/pkg/coordinator/userapi.go index 4f922d9a4..ed1ebcead 100644 --- a/pkg/coordinator/userapi.go +++ b/pkg/coordinator/userapi.go @@ -37,4 +37,6 @@ func (u *User) SendWebrtcOffer(sdp string) { u.Notify(api.WebrtcOffer, sdp) } func (u *User) SendWebrtcIceCandidate(candidate string) { u.Notify(api.WebrtcIce, candidate) } // StartGame signals the user that everything is ready to start a game. -func (u *User) StartGame() { u.Notify(api.StartGame, u.w.RoomId) } +func (u *User) StartGame(av *api.AppVideoInfo) { + u.Notify(api.StartGame, api.GameStartUserResponse{RoomId: u.w.RoomId, Av: av}) +} diff --git a/pkg/coordinator/userhandlers.go b/pkg/coordinator/userhandlers.go index 426fd74f0..81b6bf4ff 100644 --- a/pkg/coordinator/userhandlers.go +++ b/pkg/coordinator/userhandlers.go @@ -56,7 +56,7 @@ func (u *User) HandleStartGame(rq api.GameStartUserRequest, launcher games.Launc return } u.log.Info().Str("id", startGameResp.Rid).Msg("Received room response from worker") - u.StartGame() + u.StartGame(startGameResp.AV) // send back recording status if conf.Recording.Enabled && rq.Record { diff --git a/pkg/network/webrtc/webrtc.go b/pkg/network/webrtc/webrtc.go index ed0c3ca67..f94d79154 100644 --- a/pkg/network/webrtc/webrtc.go +++ b/pkg/network/webrtc/webrtc.go @@ -32,13 +32,13 @@ func (p *Peer) NewCall(vCodec, aCodec string, onICECandidate func(ice any)) (sdp if p.conn != nil && p.conn.ConnectionState() == webrtc.PeerConnectionStateConnected { return } - p.log.Info().Msg("WebRTC start") + p.log.Debug().Msg("WebRTC start") if p.conn, err = p.api.NewPeer(); err != nil { - return "", err + return } p.conn.OnICECandidate(p.handleICECandidate(onICECandidate)) // plug in the [video] track (out) - video, err := newTrack("video", "game-video", vCodec) + video, err := newTrack("video", "video", vCodec) if err != nil { return "", err } @@ -49,7 +49,7 @@ func (p *Peer) NewCall(vCodec, aCodec string, onICECandidate func(ice any)) (sdp p.log.Debug().Msgf("Added [%s] track", video.Codec().MimeType) // plug in the [audio] track (out) - audio, err := newTrack("audio", "game-audio", aCodec) + audio, err := newTrack("audio", "audio", aCodec) if err != nil { return "", err } @@ -59,21 +59,19 @@ func (p *Peer) NewCall(vCodec, aCodec string, onICECandidate func(ice any)) (sdp p.log.Debug().Msgf("Added [%s] track", audio.Codec().MimeType) p.a = audio - // plug in the [input] data channel (in) - if err = p.addInputChannel("game-input"); err != nil { + // plug in the [data] channel (in and out) + if err = p.addDataChannel("data"); err != nil { return "", err } - p.log.Debug().Msg("Added [input/bytes] chan") + p.log.Debug().Msg("Added [data] chan") - p.conn.OnICEConnectionStateChange(p.handleICEState(func() { - p.log.Info().Msg("Start streaming") - })) + p.conn.OnICEConnectionStateChange(p.handleICEState(func() { p.log.Info().Msg("Connected") })) // Stream provider supposes to send offer offer, err := p.conn.CreateOffer(nil) if err != nil { return "", err } - p.log.Info().Msg("Created Offer") + p.log.Debug().Msg("Created Offer") err = p.conn.SetLocalDescription(offer) if err != nil { @@ -210,15 +208,16 @@ func (p *Peer) Disconnect() { p.log.Debug().Msg("WebRTC stop") } -// addInputChannel creates a new WebRTC data channel for user input. +// addDataChannel creates a new WebRTC data channel for user input. // Default params -- ordered: true, negotiated: false. -func (p *Peer) addInputChannel(label string) error { +func (p *Peer) addDataChannel(label string) error { ch, err := p.conn.CreateDataChannel(label, nil) if err != nil { return err } ch.OnOpen(func() { - p.log.Debug().Str("label", ch.Label()).Uint16("id", *ch.ID()).Msg("Data channel [input] opened") + p.log.Debug().Str("label", ch.Label()).Uint16("id", *ch.ID()). + Msg("Data channel [input] opened") }) ch.OnError(p.logx) ch.OnMessage(func(m webrtc.DataChannelMessage) { diff --git a/pkg/worker/caged/app/app.go b/pkg/worker/caged/app/app.go index a1917b4d5..fcf34fd9f 100644 --- a/pkg/worker/caged/app/app.go +++ b/pkg/worker/caged/app/app.go @@ -2,6 +2,8 @@ package app type App interface { AudioSampleRate() int + AspectRatio() float32 + AspectEnabled() bool Init() error ViewportSize() (int, int) Start() @@ -9,6 +11,7 @@ type App interface { SetAudioCb(func(Audio)) SetVideoCb(func(Video)) + SetDataCb(func([]byte)) SendControl(port int, data []byte) } diff --git a/pkg/worker/caged/libretro/caged.go b/pkg/worker/caged/libretro/caged.go index 20f958477..f71017bfa 100644 --- a/pkg/worker/caged/libretro/caged.go +++ b/pkg/worker/caged/libretro/caged.go @@ -14,9 +14,6 @@ type Caged struct { base *Frontend // maintains the root for mad embedding conf CagedConf log *logger.Logger - w, h int - - OnSysInfoChange func() } type CagedConf struct { @@ -78,6 +75,8 @@ func (c *Caged) EnableCloudStorage(uid string, storage cloud.Storage) { } } +func (c *Caged) AspectEnabled() bool { return c.base.nano.Aspect } +func (c *Caged) AspectRatio() float32 { return c.base.AspectRatio() } func (c *Caged) PixFormat() uint32 { return c.Emulator.PixFormat() } func (c *Caged) Rotation() uint { return c.Emulator.Rotation() } func (c *Caged) AudioSampleRate() int { return c.Emulator.AudioSampleRate() } diff --git a/pkg/worker/caged/libretro/frontend.go b/pkg/worker/caged/libretro/frontend.go index e33f6b0f2..6a729cb3c 100644 --- a/pkg/worker/caged/libretro/frontend.go +++ b/pkg/worker/caged/libretro/frontend.go @@ -3,7 +3,6 @@ package libretro import ( "errors" "fmt" - "math" "path/filepath" "sync" "sync/atomic" @@ -20,6 +19,7 @@ import ( type Emulator interface { SetAudioCb(func(app.Audio)) SetVideoCb(func(app.Video)) + SetDataCb(func([]byte)) LoadCore(name string) LoadGame(path string) error FPS() int @@ -57,6 +57,7 @@ type Frontend struct { log *logger.Logger nano *nanoarch.Nanoarch onAudio func(app.Audio) + onData func([]byte) onVideo func(app.Video) storage Storage scale float64 @@ -90,6 +91,7 @@ const ( var ( audioPool sync.Pool noAudio = func(app.Audio) {} + noData = func([]byte) {} noVideo = func(app.Video) {} videoPool sync.Pool ) @@ -135,6 +137,7 @@ func NewFrontend(conf config.Emulator, log *logger.Logger) (*Frontend, error) { input: NewGameSessionInput(), log: log, onAudio: noAudio, + onData: noData, onVideo: noVideo, storage: store, th: conf.Threads, @@ -153,14 +156,15 @@ func NewFrontend(conf config.Emulator, log *logger.Logger) (*Frontend, error) { func (f *Frontend) LoadCore(emu string) { conf := f.conf.GetLibretroCoreConfig(emu) meta := nanoarch.Metadata{ - AutoGlContext: conf.AutoGlContext, - Hacks: conf.Hacks, - HasMultitap: conf.HasMultitap, - HasVFR: conf.VFR, - IsGlAllowed: conf.IsGlAllowed, - LibPath: conf.Lib, - Options: conf.Options, - UsesLibCo: conf.UsesLibCo, + AutoGlContext: conf.AutoGlContext, + Hacks: conf.Hacks, + HasMultitap: conf.HasMultitap, + HasVFR: conf.VFR, + IsGlAllowed: conf.IsGlAllowed, + LibPath: conf.Lib, + Options: conf.Options, + UsesLibCo: conf.UsesLibCo, + CoreAspectRatio: conf.CoreAspectRatio, } f.mu.Lock() scale := 1.0 @@ -224,6 +228,13 @@ func (f *Frontend) SetVideoChangeCb(fn func()) { f.nano.OnSystemAvInfo = fn } func (f *Frontend) Start() { f.log.Debug().Msgf("frontend start") + if f.nano.Stopped.Load() { + f.log.Warn().Msgf("frontend stopped during the start") + f.mui.Lock() + defer f.mui.Unlock() + f.Shutdown() + return + } f.mui.Lock() f.done = make(chan struct{}) @@ -269,6 +280,7 @@ func (f *Frontend) Start() { } } +func (f *Frontend) AspectRatio() float32 { return f.nano.AspectRatio() } func (f *Frontend) AudioSampleRate() int { return f.nano.AudioSampleRate() } func (f *Frontend) FPS() int { return f.nano.VideoFramerate() } func (f *Frontend) Flipped() bool { return f.nano.IsGL() } @@ -286,6 +298,7 @@ func (f *Frontend) SaveGameState() error { return f.Save() } func (f *Frontend) Scale() float64 { return f.scale } func (f *Frontend) SetAudioCb(cb func(app.Audio)) { f.onAudio = cb } func (f *Frontend) SetSessionId(name string) { f.storage.SetMainSaveName(name) } +func (f *Frontend) SetDataCb(cb func([]byte)) { f.onData = cb } func (f *Frontend) SetVideoCb(ff func(app.Video)) { f.onVideo = ff } func (f *Frontend) Tick() { f.mu.Lock(); f.nano.Run(); f.mu.Unlock() } func (f *Frontend) ToggleMultitap() { f.nano.ToggleMultitap() } @@ -296,18 +309,6 @@ func (f *Frontend) ViewportCalc() (nw int, nh int) { w, h := f.FrameSize() nw, nh = w, h - aspect, aw, ah := f.conf.AspectRatio.Keep, f.conf.AspectRatio.Width, f.conf.AspectRatio.Height - // calc the aspect ratio - if aspect && aw > 0 && ah > 0 { - ratio := float64(w) / float64(ah) - nw = int(math.Round(float64(ah)*ratio/2) * 2) - nh = ah - if nw > aw { - nw = aw - nh = int(math.Round(float64(aw)/ratio/2) * 2) - } - } - if f.IsPortrait() { nw, nh = nh, nw } diff --git a/pkg/worker/caged/libretro/nanoarch/nanoarch.go b/pkg/worker/caged/libretro/nanoarch/nanoarch.go index 4b412a3c2..cd4d36498 100644 --- a/pkg/worker/caged/libretro/nanoarch/nanoarch.go +++ b/pkg/worker/caged/libretro/nanoarch/nanoarch.go @@ -52,7 +52,7 @@ type Nanoarch struct { reserved chan struct{} // limits concurrent use Rot uint serializeSize C.size_t - stopped atomic.Bool + Stopped atomic.Bool sys struct { av C.struct_retro_system_av_info i C.struct_retro_system_info @@ -70,6 +70,7 @@ type Nanoarch struct { PixFmt PixFmt } vfr bool + Aspect bool sdlCtx *graphics.SDL hackSkipHwContextDestroy bool limiter func(func()) @@ -91,14 +92,15 @@ type FrameInfo struct { } type Metadata struct { - LibPath string // the full path to some emulator lib - IsGlAllowed bool - UsesLibCo bool - AutoGlContext bool - HasMultitap bool - HasVFR bool - Options map[string]string - Hacks []string + LibPath string // the full path to some emulator lib + IsGlAllowed bool + UsesLibCo bool + AutoGlContext bool + HasMultitap bool + HasVFR bool + Options map[string]string + Hacks []string + CoreAspectRatio bool } type PixFmt struct { @@ -122,7 +124,7 @@ func (p PixFmt) String() string { // Nan0 is a global link for C callbacks to Go var Nan0 = Nanoarch{ reserved: make(chan struct{}, 1), // this thing forbids concurrent use of the emulator - stopped: atomic.Bool{}, + Stopped: atomic.Bool{}, limiter: func(fn func()) { fn() }, Handlers: Handlers{ OnDpad: func(uint, uint) int16 { return 0 }, @@ -144,13 +146,14 @@ func NewNano(localPath string) *Nanoarch { return nano } +func (n *Nanoarch) AspectRatio() float32 { return float32(n.sys.av.geometry.aspect_ratio) } func (n *Nanoarch) AudioSampleRate() int { return int(n.sys.av.timing.sample_rate) } func (n *Nanoarch) VideoFramerate() int { return int(n.sys.av.timing.fps) } func (n *Nanoarch) IsPortrait() bool { return 90 == n.Rot%180 } func (n *Nanoarch) BaseWidth() int { return int(n.sys.av.geometry.base_width) } func (n *Nanoarch) BaseHeight() int { return int(n.sys.av.geometry.base_height) } func (n *Nanoarch) WaitReady() { <-n.reserved } -func (n *Nanoarch) Close() { n.stopped.Store(true); n.reserved <- struct{}{} } +func (n *Nanoarch) Close() { n.Stopped.Store(true); n.reserved <- struct{}{} } func (n *Nanoarch) SetLogger(log *logger.Logger) { n.log = log } func (n *Nanoarch) SetVideoDebounce(t time.Duration) { n.limiter = NewLimit(t) } @@ -158,6 +161,7 @@ func (n *Nanoarch) CoreLoad(meta Metadata) { var err error n.LibCo = meta.UsesLibCo n.vfr = meta.HasVFR + n.Aspect = meta.CoreAspectRatio n.Video.gl.autoCtx = meta.AutoGlContext n.Video.gl.enabled = meta.IsGlAllowed @@ -258,12 +262,17 @@ func (n *Nanoarch) LoadGame(path string) error { return fmt.Errorf("core failed to load ROM: %v", path) } - C.bridge_retro_get_system_av_info(retroGetSystemAVInfo, &n.sys.av) + var av C.struct_retro_system_av_info + C.bridge_retro_get_system_av_info(retroGetSystemAVInfo, &av) n.log.Info().Msgf("System A/V >>> %vx%v (%vx%v), [%vfps], AR [%v], audio [%vHz]", - n.sys.av.geometry.base_width, n.sys.av.geometry.base_height, - n.sys.av.geometry.max_width, n.sys.av.geometry.max_height, - n.sys.av.timing.fps, n.sys.av.geometry.aspect_ratio, n.sys.av.timing.sample_rate, + av.geometry.base_width, av.geometry.base_height, + av.geometry.max_width, av.geometry.max_height, + av.timing.fps, av.geometry.aspect_ratio, av.timing.sample_rate, ) + if isGeometryDifferent(av.geometry) { + geometryChange(av.geometry) + } + n.sys.av = av n.serializeSize = C.bridge_retro_serialize_size(retroSerializeSize) n.log.Info().Msgf("Save file size: %v", byteCountBinary(int64(n.serializeSize))) @@ -273,7 +282,7 @@ func (n *Nanoarch) LoadGame(path string) error { n.log.Info().Msgf("variable framerate (VFR) is enabled") } - n.stopped.Store(false) + n.Stopped.Store(false) if n.Video.gl.enabled { bufS := uint(n.sys.av.geometry.max_width*n.sys.av.geometry.max_height) * n.Video.PixFmt.BPP @@ -348,6 +357,7 @@ func (n *Nanoarch) Shutdown() { } setRotation(0) + Nan0.sys.av = C.struct_retro_system_av_info{} if err := closeLib(coreLib); err != nil { n.log.Error().Err(err).Msg("lib close failed") } @@ -376,7 +386,7 @@ func (n *Nanoarch) Run() { } func (n *Nanoarch) IsGL() bool { return n.Video.gl.enabled } -func (n *Nanoarch) IsStopped() bool { return n.stopped.Load() } +func (n *Nanoarch) IsStopped() bool { return n.Stopped.Load() } func videoSetPixelFormat(format uint32) (C.bool, error) { switch format { @@ -550,7 +560,7 @@ var ( //export coreVideoRefresh func coreVideoRefresh(data unsafe.Pointer, width, height uint, packed uint) { - if Nan0.stopped.Load() { + if Nan0.Stopped.Load() { Nan0.log.Warn().Msgf(">>> skip video") return } @@ -636,7 +646,7 @@ func coreAudioSample(l, r C.int16_t) { //export coreAudioSampleBatch func coreAudioSampleBatch(data unsafe.Pointer, frames C.size_t) C.size_t { - if Nan0.stopped.Load() { + if Nan0.Stopped.Load() { if Nan0.log.GetLevel() < logger.InfoLevel { Nan0.log.Warn().Msgf(">>> skip %v audio frames", frames) } @@ -686,27 +696,18 @@ func coreEnvironment(cmd C.unsigned, data unsafe.Pointer) C.bool { switch cmd { case C.RETRO_ENVIRONMENT_SET_SYSTEM_AV_INFO: - Nan0.sys.av = *(*C.struct_retro_system_av_info)(data) - Nan0.log.Debug().Msgf(">>> system av change: %v", Nan0.sys.av) - if Nan0.OnSystemAvInfo != nil { - go Nan0.OnSystemAvInfo() + Nan0.log.Debug().Msgf("retro_set_system_av_info") + av := *(*C.struct_retro_system_av_info)(data) + if isGeometryDifferent(av.geometry) { + geometryChange(av.geometry) } return true case C.RETRO_ENVIRONMENT_SET_GEOMETRY: + Nan0.log.Debug().Msgf("retro_set_geometry") geom := *(*C.struct_retro_game_geometry)(data) - Nan0.log.Debug().Msgf(">>> geometry change: %v", geom) - // some cores are eager to change resolution too many times - // in a small period of time, thus we have some debouncer here - Nan0.limiter(func() { - lw := Nan0.sys.av.geometry.base_width - lh := Nan0.sys.av.geometry.base_height - if lw != geom.base_width || lh != geom.base_height { - Nan0.sys.av.geometry = geom - if Nan0.OnSystemAvInfo != nil { - go Nan0.OnSystemAvInfo() - } - } - }) + if isGeometryDifferent(geom) { + geometryChange(geom) + } return true case C.RETRO_ENVIRONMENT_SET_ROTATION: setRotation((*(*uint)(data) % 4) * 90) @@ -876,3 +877,21 @@ func (d *limit) push(f func()) { } d.t = time.AfterFunc(d.d, f) } + +func geometryChange(geom C.struct_retro_game_geometry) { + Nan0.limiter(func() { + old := Nan0.sys.av.geometry + Nan0.sys.av.geometry = geom + if Nan0.OnSystemAvInfo != nil { + Nan0.log.Debug().Msgf(">>> geometry change %v -> %v", old, geom) + if Nan0.Aspect { + go Nan0.OnSystemAvInfo() + } + } + }) +} + +func isGeometryDifferent(geom C.struct_retro_game_geometry) bool { + return Nan0.sys.av.geometry.base_width != geom.base_width || + Nan0.sys.av.geometry.base_height != geom.base_height +} diff --git a/pkg/worker/coordinatorhandlers.go b/pkg/worker/coordinatorhandlers.go index e675d33d0..d5d9dd8e7 100644 --- a/pkg/worker/coordinatorhandlers.go +++ b/pkg/worker/coordinatorhandlers.go @@ -112,6 +112,31 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest[com.Uid], w *Worke r.SetApp(app) + m := media.NewWebRtcMediaPipe(w.conf.Encoder.Audio, w.conf.Encoder.Video, w.log) + + // recreate the video encoder + app.VideoChangeCb(func() { + app.ViewportRecalculate() + m.VideoW, m.VideoH = app.ViewportSize() + m.VideoScale = app.Scale() + + if m.IsInitialized() { + if err := m.Reinit(); err != nil { + c.log.Error().Err(err).Msgf("reinit fail") + } + } + + data, err := api.Wrap(api.Out{T: uint8(api.AppVideoChange), Payload: api.AppVideoInfo{ + W: m.VideoW, + H: m.VideoH, + A: app.AspectRatio(), + }}) + if err != nil { + c.log.Error().Err(err).Msgf("wrap") + } + r.Send(data) + }) + w.log.Info().Msgf("Starting the game: %v", rq.Game.Name) if err := app.Load(game, w.conf.Worker.Library.BasePath); err != nil { c.log.Error().Err(err).Msgf("couldn't load the game %v", game) @@ -120,7 +145,6 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest[com.Uid], w *Worke return api.EmptyPacket } - m := media.NewWebRtcMediaPipe(w.conf.Encoder.Audio, w.conf.Encoder.Video, w.log) m.AudioSrcHz = app.AudioSampleRate() m.AudioFrame = w.conf.Encoder.Audio.Frame m.VideoW, m.VideoH = app.ViewportSize() @@ -140,16 +164,6 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest[com.Uid], w *Worke m.SetPixFmt(app.PixFormat()) m.SetRot(app.Rotation()) - // recreate the video encoder - app.VideoChangeCb(func() { - app.ViewportRecalculate() - m.VideoW, m.VideoH = app.ViewportSize() - m.VideoScale = app.Scale() - if err := m.Reinit(); err != nil { - c.log.Error().Err(err).Msgf("reinit fail") - } - }) - r.BindAppMedia() r.StartApp() } @@ -159,7 +173,13 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest[com.Uid], w *Worke c.RegisterRoom(r.Id()) - return api.Out{Payload: api.StartGameResponse{Room: api.Room{Rid: r.Id()}, Record: w.conf.Recording.Enabled}} + response := api.StartGameResponse{Room: api.Room{Rid: r.Id()}, Record: w.conf.Recording.Enabled} + if r.App().AspectEnabled() { + ww, hh := r.App().ViewportSize() + response.AV = &api.AppVideoInfo{W: ww, H: hh, A: r.App().AspectRatio()} + } + + return api.Out{Payload: response} } // HandleTerminateSession handles cases when a user has been disconnected from the websocket of coordinator. diff --git a/pkg/worker/media/media.go b/pkg/worker/media/media.go index 588259ac8..70758b1cb 100644 --- a/pkg/worker/media/media.go +++ b/pkg/worker/media/media.go @@ -115,6 +115,8 @@ type WebrtcMediaPipe struct { VideoW, VideoH int VideoScale float64 + initialized bool + // keep the old settings for reinit oldPf uint32 oldRot uint @@ -144,6 +146,7 @@ func (wmp *WebrtcMediaPipe) Init() error { return err } wmp.log.Debug().Msgf("%v", wmp.v.Info()) + wmp.initialized = true return nil } @@ -188,6 +191,10 @@ func (wmp *WebrtcMediaPipe) ProcessVideo(v app.Video) []byte { } func (wmp *WebrtcMediaPipe) Reinit() error { + if !wmp.initialized { + return nil + } + wmp.v.Stop() if err := wmp.initVideo(wmp.VideoW, wmp.VideoH, wmp.VideoScale, wmp.vConf); err != nil { return err @@ -199,6 +206,7 @@ func (wmp *WebrtcMediaPipe) Reinit() error { return nil } +func (wmp *WebrtcMediaPipe) IsInitialized() bool { return wmp.initialized } func (wmp *WebrtcMediaPipe) SetPixFmt(f uint32) { wmp.oldPf = f; wmp.v.SetPixFormat(f) } func (wmp *WebrtcMediaPipe) SetVideoFlip(b bool) { wmp.oldFlip = b; wmp.v.SetFlip(b) } func (wmp *WebrtcMediaPipe) SetRot(r uint) { wmp.oldRot = r; wmp.v.SetRot(r) } diff --git a/pkg/worker/room/room.go b/pkg/worker/room/room.go index 0f111456c..c52f091d1 100644 --- a/pkg/worker/room/room.go +++ b/pkg/worker/room/room.go @@ -81,6 +81,7 @@ func (r *Room[T]) Id() string { return r.id } func (r *Room[T]) SetApp(app app.App) { r.app = app } func (r *Room[T]) SetMedia(m MediaPipe) { r.media = m } func (r *Room[T]) StartApp() { r.app.Start() } +func (r *Room[T]) Send(data []byte) { r.users.ForEach(func(u T) { u.SendData(data) }) } func (r *Room[T]) Close() { if r == nil || r.closed { diff --git a/web/css/main.css b/web/css/main.css index d106b1be4..5efbf8750 100644 --- a/web/css/main.css +++ b/web/css/main.css @@ -22,7 +22,7 @@ body { display: flex; overflow: hidden; - width: 556px; + width: 640px; height: 286px; position: absolute; @@ -65,6 +65,11 @@ body { background-size: 100% 100%; } +#controls-right { + position: absolute; + left: 70px; +} + #circle-pad-holder { display: block; @@ -83,11 +88,10 @@ body { } #guide-txt { - color: #bababa; + color: #979797; font-size: 8px; top: 269px; - left: 30px; - width: 1000px; + left: 68px; position: absolute; user-select: none; @@ -166,7 +170,7 @@ body { align-items: center; justify-content: center; - width: 256px; + width: 320px; height: 240px; position: absolute; top: 23px; @@ -416,11 +420,11 @@ body { } .game-screen { - width: 100%; - height: 102%; /* lol */ - background-color: #222222; position: absolute; - display: flex; + object-fit: contain; + width: inherit; + height: inherit; + background-color: #222222; } #menu-screen { @@ -428,7 +432,7 @@ body { display: block; overflow: hidden; - width: 256px; + width: 320px; height: 240px; background-image: url('/img/screen_background5.png'); @@ -444,6 +448,7 @@ body { height: 36px; background-color: #FFCF9E; opacity: 0.75; + mix-blend-mode: lighten; top: 50%; left: 0; @@ -459,7 +464,7 @@ body { top: 102px; /* 240px - 36 / 2 */ left: 0; - z-index: 1; + /*z-index: 1;*/ } @@ -481,7 +486,7 @@ body { left: 15px; top: 7px; - width: 226px; + width: 288px; height: 25px; } diff --git a/web/index.html b/web/index.html index 4f44140f1..e082abdee 100644 --- a/web/index.html +++ b/web/index.html @@ -47,26 +47,28 @@
-
Arrows(move),ZXCVAS(game ABXYLR),1/2(1st/2nd player),Shift/Enter/K/L(select/start/save/load),F(fullscreen),share(copy - sharelink to clipboard) +
+ Arrows (move), ZXCVAS (game ABXYLR), 1/2 (1st/2nd player), Shift/Enter/K/L (select/start/save/load), F (fullscreen), share (copy shared link to the clipboard)
- -
player choice
-
-
-
- -
-
-
-
-
+ +
+ + +
+
+ +
+
+
+
+
+
@@ -125,16 +127,16 @@

Options

- + - + - + - + diff --git a/web/js/api/api.js b/web/js/api/api.js index bbb213180..20de8312b 100644 --- a/web/js/api/api.js +++ b/web/js/api/api.js @@ -21,6 +21,8 @@ const api = (() => { GAME_RECORDING: 110, GET_WORKER_LIST: 111, GAME_ERROR_NO_FREE_SLOTS: 112, + + APP_VIDEO_CHANGE: 150, }); const packet = (type, payload, id) => { @@ -31,18 +33,21 @@ const api = (() => { socket.send(packet); }; + const decodeBytes = (b) => String.fromCharCode.apply(null, new Uint8Array(b)) + return Object.freeze({ endpoint: endpoints, + decode: (b) => JSON.parse(decodeBytes(b)), server: - Object.freeze({ + { initWebrtc: () => packet(endpoints.INIT_WEBRTC), sendIceCandidate: (candidate) => packet(endpoints.ICE_CANDIDATE, btoa(JSON.stringify(candidate))), sendSdp: (sdp) => packet(endpoints.ANSWER, btoa(JSON.stringify(sdp))), latencyCheck: (id, list) => packet(endpoints.LATENCY_CHECK, list, id), getWorkerList: () => packet(endpoints.GET_WORKER_LIST), - }), + }, game: - Object.freeze({ + { load: () => packet(endpoints.GAME_LOAD), save: () => packet(endpoints.GAME_SAVE), setPlayerIndex: (i) => packet(endpoints.GAME_SET_PLAYER_INDEX, i), @@ -60,6 +65,6 @@ const api = (() => { user: userName, }), quit: (roomId) => packet(endpoints.GAME_QUIT, {room_id: roomId}), - }) + } }) })(socket); diff --git a/web/js/controller.js b/web/js/controller.js index 5ae5443cd..edd0e61e7 100644 --- a/web/js/controller.js +++ b/web/js/controller.js @@ -153,8 +153,8 @@ const saveGame = utils.debounce(() => api.game.save(), 1000); const loadGame = utils.debounce(() => api.game.load(), 1000); - const onMessage = (message) => { - const {id, t, p: payload} = message; + const onMessage = (m) => { + const {id, t, p: payload} = m; switch (t) { case api.endpoint.INIT: event.pub(WEBRTC_NEW_CONNECTION, payload); @@ -166,7 +166,10 @@ event.pub(WEBRTC_ICE_CANDIDATE_RECEIVED, {candidate: payload}); break; case api.endpoint.GAME_START: - event.pub(GAME_ROOM_AVAILABLE, {roomId: payload}); + if (payload.av) { + event.pub(APP_VIDEO_CHANGED, payload.av) + } + event.pub(GAME_ROOM_AVAILABLE, {roomId: payload.roomId}); break; case api.endpoint.GAME_SAVE: event.pub(GAME_SAVED); @@ -189,6 +192,9 @@ case api.endpoint.GAME_ERROR_NO_FREE_SLOTS: event.pub(GAME_ERROR_NO_FREE_SLOTS); break; + case api.endpoint.APP_VIDEO_CHANGE: + event.pub(APP_VIDEO_CHANGED, {...payload}) + break; } } @@ -429,6 +435,7 @@ event.sub(GAME_ERROR_NO_FREE_SLOTS, () => message.show("No free slots :(", 2500)); event.sub(WEBRTC_NEW_CONNECTION, (data) => { workerManager.whoami(data.wid); + webrtc.onData = (x) => onMessage(api.decode(x.data)) webrtc.start(data.ice); api.server.initWebrtc() gameList.set(data.games); @@ -438,7 +445,6 @@ event.sub(WEBRTC_SDP_OFFER, (data) => webrtc.setRemoteDescription(data.sdp, stream.video.el())); event.sub(WEBRTC_ICE_CANDIDATE_RECEIVED, (data) => webrtc.addCandidate(data.candidate)); event.sub(WEBRTC_ICE_CANDIDATES_FLUSH, () => webrtc.flushCandidates()); - // event.sub(MEDIA_STREAM_READY, () => rtcp.start()); event.sub(WEBRTC_CONNECTION_READY, onConnectionReady); event.sub(WEBRTC_CONNECTION_CLOSED, () => { input.poll.disable(); diff --git a/web/js/event/event.js b/web/js/event/event.js index 9ad45a4b1..3c412d202 100644 --- a/web/js/event/event.js +++ b/web/js/event/event.js @@ -101,3 +101,5 @@ const SETTINGS_CLOSED = 'settingsClosed'; const RECORDING_TOGGLED = 'recordingToggle' const RECORDING_STATUS_CHANGED = 'recordingStatusChanged' + +const APP_VIDEO_CHANGED = 'appVideoChanged' diff --git a/web/js/network/webrtc.js b/web/js/network/webrtc.js index 99c5432a8..8d72c8fd1 100644 --- a/web/js/network/webrtc.js +++ b/web/js/network/webrtc.js @@ -12,16 +12,16 @@ */ const webrtc = (() => { let connection; - let inputChannel; + let dataChannel; let mediaStream; - let candidates = Array(); + let candidates = []; let isAnswered = false; let isFlushing = false; let connected = false; let inputReady = false; - let onMessage; + let onData; const start = (iceservers) => { log.info('[rtc] <- ICE servers', iceservers); @@ -31,16 +31,16 @@ const webrtc = (() => { connection.ondatachannel = e => { log.debug('[rtc] ondatachannel', e.channel.label) - inputChannel = e.channel; - inputChannel.onopen = () => { + dataChannel = e.channel; + dataChannel.onopen = () => { log.info('[rtc] the input channel has been opened'); inputReady = true; event.pub(WEBRTC_CONNECTION_READY) }; - if (onMessage) { - inputChannel.onmessage = onMessage; + if (onData) { + dataChannel.onmessage = onData; } - inputChannel.onclose = () => log.info('[rtc] the input channel has been closed'); + dataChannel.onclose = () => log.info('[rtc] the input channel has been closed'); } connection.oniceconnectionstatechange = ice.onIceConnectionStateChange; connection.onicegatheringstatechange = ice.onIceStateChange; @@ -62,9 +62,9 @@ const webrtc = (() => { connection.close(); connection = null; } - if (inputChannel) { - inputChannel.close(); - inputChannel = null; + if (dataChannel) { + dataChannel.close(); + dataChannel = null; } candidates = Array(); log.info('[rtc] WebRTC has been closed'); @@ -173,10 +173,13 @@ const webrtc = (() => { // return false // } // }, - input: (data) => inputChannel.send(data), + input: (data) => dataChannel.send(data), isConnected: () => connected, isInputReady: () => inputReady, getConnection: () => connection, stop, + set onData(fn) { + onData = fn + } } })(event, log); diff --git a/web/js/stream/stream.js b/web/js/stream/stream.js index 64e65a165..9ab3dd494 100644 --- a/web/js/stream/stream.js +++ b/web/js/stream/stream.js @@ -16,6 +16,9 @@ const stream = (() => { state = { screen: screen, timerId: null, + w: 0, + h: 0, + aspect: 4/3 }; const mute = (mute) => screen.muted = mute @@ -82,6 +85,25 @@ const stream = (() => { useCustomScreen(options.mirrorMode === 'mirror'); }, false); + screen.addEventListener('fullscreenchange', () => { + const fullscreen = document.fullscreenElement + + + screen.style.padding = '0' + if (fullscreen) { + const dw = (window.innerWidth - fullscreen.clientHeight * state.aspect) / 2 + screen.style.padding = `0 ${dw}px` + // chrome bug + setTimeout(() => { + const dw = (window.innerHeight - fullscreen.clientHeight * state.aspect) / 2 + screen.style.padding = `0 ${dw}px` + }, 1) + } + + // !to flipped + + }) + const useCustomScreen = (use) => { if (use) { if (screen.paused || screen.ended) return; @@ -95,13 +117,15 @@ const stream = (() => { canvas.setAttribute('width', screen.videoWidth); canvas.setAttribute('height', screen.videoHeight); canvas.style['image-rendering'] = 'pixelated'; + canvas.style.width = '100%' + canvas.style.height = '100%' canvas.classList.add('game-screen'); // stretch depending on the video orientation // portrait -- vertically, landscape -- horizontally const isPortrait = screen.videoWidth < screen.videoHeight; canvas.style.width = isPortrait ? 'auto' : canvas.style.width; - canvas.style.height = isPortrait ? canvas.style.height : 'auto'; + // canvas.style.height = isPortrait ? canvas.style.height : 'auto'; let surface = canvas.getContext('2d'); screen.parentNode.insertBefore(canvas, screen.nextSibling); @@ -135,6 +159,27 @@ const stream = (() => { } }); + + const fit = 'contain' + + event.sub(APP_VIDEO_CHANGED, (payload) => { + const {w, h, a} = payload + + state.aspect = a + + const a2 = w / h + + const attr = a.toFixed(6) !== a2.toFixed(6) ? 'fill' : fit + state.screen.style['object-fit'] = attr + + state.h = payload.h + state.w = Math.floor(payload.h * payload.a) + // payload.a > 0 && (state.aspect = payload.a) + state.screen.setAttribute('width', payload.w) + state.screen.setAttribute('height', payload.h) + state.screen.style.aspectRatio = state.aspect + }) + return { audio: {mute}, video: {toggleFullscreen, el: getVideoEl}, @@ -144,4 +189,4 @@ const stream = (() => { init } } -)(env, gui, log, opts, settings); +)(env, event, gui, log, opts, settings);