From 65251061160c384e3eae7b992e965cdeb4cccac5 Mon Sep 17 00:00:00 2001 From: Sergey Stepanov Date: Tue, 24 Oct 2023 02:46:57 +0300 Subject: [PATCH] Update video encoders --- pkg/config/config.yaml | 2 + pkg/config/worker.go | 5 ++- pkg/encoder/encoder.go | 54 +++++++++++++++---------- pkg/encoder/h264/libx264.go | 11 +++-- pkg/encoder/h264/x264.go | 73 +++++++++++++++------------------- pkg/encoder/h264/x264_test.go | 4 +- pkg/encoder/vpx/libvpx.go | 2 + pkg/worker/media/media.go | 29 +++----------- pkg/worker/media/media_test.go | 36 +++++++++++++---- 9 files changed, 116 insertions(+), 100 deletions(-) diff --git a/pkg/config/config.yaml b/pkg/config/config.yaml index 344aa10eb..cbfe0ab5a 100644 --- a/pkg/config/config.yaml +++ b/pkg/config/config.yaml @@ -260,6 +260,8 @@ encoder: video: # h264, vpx (VP8) codec: h264 + # Threaded encoder if supported, 0 - auto, 1 - nope, >1 - multi-threaded + threads: 0 # see: https://trac.ffmpeg.org/wiki/Encode/H.264 h264: # Constant Rate Factor (CRF) 0-51 (default: 23) diff --git a/pkg/config/worker.go b/pkg/config/worker.go index ab6af2cc8..c0c39adf3 100644 --- a/pkg/config/worker.go +++ b/pkg/config/worker.go @@ -52,8 +52,9 @@ type Audio struct { } type Video struct { - Codec string - H264 struct { + Codec string + Threads int + H264 struct { Crf uint8 LogLevel int32 Preset string diff --git a/pkg/encoder/encoder.go b/pkg/encoder/encoder.go index 60e960d01..a9f9003cf 100644 --- a/pkg/encoder/encoder.go +++ b/pkg/encoder/encoder.go @@ -2,9 +2,11 @@ package encoder import ( "fmt" - "sync" "sync/atomic" + "github.com/giongto35/cloud-game/v3/pkg/config" + "github.com/giongto35/cloud-game/v3/pkg/encoder/h264" + "github.com/giongto35/cloud-game/v3/pkg/encoder/vpx" "github.com/giongto35/cloud-game/v3/pkg/encoder/yuv" "github.com/giongto35/cloud-game/v3/pkg/logger" ) @@ -16,6 +18,7 @@ type ( LoadBuf(input []byte) Encode() []byte IntraRefresh() + Info() string SetFlip(bool) Shutdown() error } @@ -28,7 +31,6 @@ type Video struct { y yuv.Conv pf yuv.PixFmt rot uint - mu sync.Mutex } type VideoCodec string @@ -36,6 +38,7 @@ type VideoCodec string const ( H264 VideoCodec = "h264" VP8 VideoCodec = "vp8" + VPX VideoCodec = "vpx" ) // NewVideoEncoder returns new video encoder. @@ -43,13 +46,27 @@ const ( // converts them into YUV I420 format, // encodes with provided video encoder, and // puts the result into the output channel. -func NewVideoEncoder(codec Encoder, w, h int, scale float64, log *logger.Logger) *Video { - return &Video{codec: codec, y: yuv.NewYuvConv(w, h, scale), log: log} +func NewVideoEncoder(w, h, dw, dh int, scale float64, conf config.Video, log *logger.Logger) (*Video, error) { + var enc Encoder + var err error + switch VideoCodec(conf.Codec) { + case H264: + opts := h264.Options(conf.H264) + enc, err = h264.NewEncoder(dw, dh, conf.Threads, &opts) + case VP8, VPX: + opts := vpx.Options(conf.Vpx) + enc, err = vpx.NewEncoder(dw, dh, &opts) + default: + err = fmt.Errorf("unsupported codec: %v", conf.Codec) + } + if err != nil { + return nil, err + } + + return &Video{codec: enc, y: yuv.NewYuvConv(w, h, scale), log: log}, nil } func (v *Video) Encode(frame InFrame) OutFrame { - v.mu.Lock() - defer v.mu.Unlock() if v.stopped.Load() { return nil } @@ -64,7 +81,9 @@ func (v *Video) Encode(frame InFrame) OutFrame { return nil } -func (v *Video) Info() string { return fmt.Sprintf("libyuv: %v", v.y.Version()) } +func (v *Video) Info() string { + return fmt.Sprintf("%v, libyuv: %v", v.codec.Info(), v.y.Version()) +} func (v *Video) SetPixFormat(f uint32) { switch f { @@ -77,16 +96,10 @@ func (v *Video) SetPixFormat(f uint32) { } } -// SetRot sets the rotation angle of the frames. -func (v *Video) SetRot(r uint) { - switch r { - // de-rotate - case 90: - v.rot = 270 - case 270: - v.rot = 90 - default: - v.rot = r +// SetRot sets the de-rotation angle of the frames. +func (v *Video) SetRot(a uint) { + if a > 0 { + v.rot = (a + 180) % 360 } } @@ -94,11 +107,12 @@ func (v *Video) SetRot(r uint) { func (v *Video) SetFlip(b bool) { v.codec.SetFlip(b) } func (v *Video) Stop() { - v.stopped.Store(true) - v.mu.Lock() - defer v.mu.Unlock() + if v.stopped.Swap(true) { + return + } v.rot = 0 + defer func() { v.codec = nil }() if err := v.codec.Shutdown(); err != nil { v.log.Error().Err(err).Msg("failed to close the encoder") } diff --git a/pkg/encoder/h264/libx264.go b/pkg/encoder/h264/libx264.go index 0539b437b..6be21eb6a 100644 --- a/pkg/encoder/h264/libx264.go +++ b/pkg/encoder/h264/libx264.go @@ -8,6 +8,11 @@ package h264 #include "stdint.h" #include "x264.h" #include + +static int x264_encode( x264_t *h, uintptr_t pp_nal, int *pi_nal, x264_picture_t *pic_in, x264_picture_t *pic_out ) { + return x264_encoder_encode(h, (x264_nal_t **)pp_nal, pi_nal, pic_in, pic_out); +} + */ import "C" import "unsafe" @@ -505,16 +510,16 @@ func EncoderOpen(param *Param) *T { // EncoderEncode - encode one picture. // Returns the number of bytes in the returned NALs, negative on error and zero if no NAL units returned. -func EncoderEncode(enc *T, ppNal []*Nal, piNal *int32, picIn *Picture, picOut *Picture) int32 { +func EncoderEncode(enc *T, ppNal **Nal, piNal *int32, picIn *Picture, picOut *Picture) int32 { cenc := enc.cptr() - cppNal := (**C.x264_nal_t)(unsafe.Pointer(&ppNal[0])) + cppNal := C.uintptr_t(uintptr(unsafe.Pointer(ppNal))) cpiNal := (*C.int)(unsafe.Pointer(piNal)) cpicIn := picIn.cptr() cpicOut := picOut.cptr() - return (int32)(C.x264_encoder_encode(cenc, cppNal, cpiNal, cpicIn, cpicOut)) + return (int32)(C.x264_encode(cenc, cppNal, cpiNal, cpicIn, cpicOut)) } // EncoderClose closes an encoder handler. diff --git a/pkg/encoder/h264/x264.go b/pkg/encoder/h264/x264.go index ae8634cbe..8b10ce58a 100644 --- a/pkg/encoder/h264/x264.go +++ b/pkg/encoder/h264/x264.go @@ -8,12 +8,10 @@ import ( type H264 struct { ref *T - width int32 - lumaSize int32 - chromaSize int32 - nnals int32 - nals []*Nal - + pnals *Nal // array of NALs + nnals int32 // number of NALs + y int32 // Y size + uv int32 // U or V size in, out *Picture } @@ -32,11 +30,11 @@ type Options struct { Tune string } -func NewEncoder(w, h int, opts *Options) (encoder *H264, err error) { +func NewEncoder(w, h int, th int, opts *Options) (encoder *H264, err error) { libVersion := LibVersion() - if libVersion < 150 { - return nil, fmt.Errorf("x264: the library version should be newer than v150, you have got version %v", libVersion) + if libVersion < 156 { + return nil, fmt.Errorf("x264: the library version should be newer than v155, you have got version %v", libVersion) } if opts == nil { @@ -63,39 +61,28 @@ func NewEncoder(w, h int, opts *Options) (encoder *H264, err error) { } } - // legacy encoder lacks of this param - param.IBitdepth = 8 + ww, hh := int32(w), int32(h) - if libVersion > 155 { - param.ICsp = CspI420 - } else { - param.ICsp = 1 - } - param.IWidth = int32(w) - param.IHeight = int32(h) + param.IBitdepth = 8 + param.ICsp = CspI420 + param.IWidth = ww + param.IHeight = hh param.ILogLevel = opts.LogLevel param.ISyncLookahead = 0 - param.IThreads = 1 - + param.IThreads = int32(th) + if th != 1 { + param.BSlicedThreads = 1 + } param.Rc.IRcMethod = RcCrf param.Rc.FRfConstant = float32(opts.Crf) encoder = &H264{ - lumaSize: param.IWidth * param.IHeight, - chromaSize: param.IWidth * param.IHeight / 4, - nals: make([]*Nal, 1), - width: param.IWidth, - out: new(Picture), + y: ww * hh, + uv: ww * hh / 4, + pnals: new(Nal), + out: new(Picture), in: &Picture{ - Img: Image{ - ICsp: param.ICsp, - IPlane: 3, - IStride: [4]int32{ - 0: param.IWidth, - 1: param.IWidth >> 1, - 2: param.IWidth >> 1, - }, - }, + Img: Image{ICsp: param.ICsp, IPlane: 3, IStride: [4]int32{0: ww, 1: ww >> 1, 2: ww >> 1}}, }, } @@ -109,23 +96,27 @@ func LibVersion() int { return int(Build) } func (e *H264) LoadBuf(yuv []byte) { e.in.Img.Plane[0] = uintptr(unsafe.Pointer(&yuv[0])) - e.in.Img.Plane[1] = uintptr(unsafe.Pointer(&yuv[e.lumaSize])) - e.in.Img.Plane[2] = uintptr(unsafe.Pointer(&yuv[e.lumaSize+e.chromaSize])) + e.in.Img.Plane[1] = uintptr(unsafe.Pointer(&yuv[e.y])) + e.in.Img.Plane[2] = uintptr(unsafe.Pointer(&yuv[e.y+e.uv])) } -func (e *H264) Encode() []byte { +func (e *H264) Encode() (b []byte) { e.in.IPts += 1 - if ret := EncoderEncode(e.ref, e.nals, &e.nnals, e.in, e.out); ret > 0 { - return unsafe.Slice((*byte)(e.nals[0].PPayload), ret) - //return C.GoBytes(e.nals[0].PPayload, C.int(ret)) + bytes := EncoderEncode(e.ref, &e.pnals, &e.nnals, e.in, e.out) + if bytes > 0 { + // we merge multiple NALs stored in **pnals into a single byte stream + // ret contains the total size of NALs in bytes, i.e. each e.pnals[...].PPayload * IPayload + b = unsafe.Slice((*byte)(e.pnals.PPayload), bytes) } - return []byte{} + return } func (e *H264) IntraRefresh() { // !to implement } +func (e *H264) Info() string { return fmt.Sprintf("x264: v%v", LibVersion()) } + func (e *H264) SetFlip(b bool) { if b { e.in.Img.ICsp |= CspVflip diff --git a/pkg/encoder/h264/x264_test.go b/pkg/encoder/h264/x264_test.go index e819ba187..b13c0bc49 100644 --- a/pkg/encoder/h264/x264_test.go +++ b/pkg/encoder/h264/x264_test.go @@ -3,7 +3,7 @@ package h264 import "testing" func TestH264Encode(t *testing.T) { - h264, err := NewEncoder(120, 120, nil) + h264, err := NewEncoder(120, 120, 0, nil) if err != nil { t.Error(err) } @@ -17,7 +17,7 @@ func TestH264Encode(t *testing.T) { func Benchmark(b *testing.B) { w, h := 1920, 1080 - h264, err := NewEncoder(w, h, nil) + h264, err := NewEncoder(w, h, 0, nil) if err != nil { b.Error(err) } diff --git a/pkg/encoder/vpx/libvpx.go b/pkg/encoder/vpx/libvpx.go index ca423e0de..81988b9c6 100644 --- a/pkg/encoder/vpx/libvpx.go +++ b/pkg/encoder/vpx/libvpx.go @@ -163,6 +163,8 @@ func (vpx *Vpx) Encode() []byte { return C.GoBytes(fb.ptr, fb.size) } +func (vpx *Vpx) Info() string { return fmt.Sprintf("vpx: %v", C.GoString(C.vpx_codec_version_str())) } + func (vpx *Vpx) IntraRefresh() { // !to implement } diff --git a/pkg/worker/media/media.go b/pkg/worker/media/media.go index 7c9a242eb..588259ac8 100644 --- a/pkg/worker/media/media.go +++ b/pkg/worker/media/media.go @@ -8,9 +8,7 @@ import ( "github.com/giongto35/cloud-game/v3/pkg/config" "github.com/giongto35/cloud-game/v3/pkg/encoder" - "github.com/giongto35/cloud-game/v3/pkg/encoder/h264" "github.com/giongto35/cloud-game/v3/pkg/encoder/opus" - "github.com/giongto35/cloud-game/v3/pkg/encoder/vpx" "github.com/giongto35/cloud-game/v3/pkg/logger" "github.com/giongto35/cloud-game/v3/pkg/worker/caged/app" ) @@ -145,6 +143,7 @@ func (wmp *WebrtcMediaPipe) Init() error { if err := wmp.initVideo(wmp.VideoW, wmp.VideoH, wmp.VideoScale, wmp.vConf); err != nil { return err } + wmp.log.Debug().Msgf("%v", wmp.v.Info()) return nil } @@ -175,29 +174,11 @@ func (wmp *WebrtcMediaPipe) encodeAudio(pcm samples) { wmp.onAudio(data) } -func (wmp *WebrtcMediaPipe) initVideo(w, h int, scale float64, conf config.Video) error { - var enc encoder.Encoder - var err error - +func (wmp *WebrtcMediaPipe) initVideo(w, h int, scale float64, conf config.Video) (err error) { sw, sh := round(w, scale), round(h, scale) - - wmp.log.Debug().Msgf("Scale: %vx%v -> %vx%v", w, h, sw, sh) - - wmp.log.Info().Msgf("Video codec: %v", conf.Codec) - if conf.Codec == string(encoder.H264) { - wmp.log.Debug().Msgf("x264: build v%v", h264.LibVersion()) - opts := h264.Options(conf.H264) - enc, err = h264.NewEncoder(sw, sh, &opts) - } else { - opts := vpx.Options(conf.Vpx) - enc, err = vpx.NewEncoder(sw, sh, &opts) - } - if err != nil { - return fmt.Errorf("couldn't create a video encoder: %w", err) - } - wmp.v = encoder.NewVideoEncoder(enc, w, h, scale, wmp.log) - wmp.log.Debug().Msgf("%v", wmp.v.Info()) - return nil + wmp.v, err = encoder.NewVideoEncoder(w, h, sw, sh, scale, conf, wmp.log) + wmp.log.Debug().Msgf("media scale: %vx%v -> %vx%v", w, h, sw, sh) + return err } func round(x int, scale float64) int { return (int(float64(x)*scale) + 1) & ^1 } diff --git a/pkg/worker/media/media_test.go b/pkg/worker/media/media_test.go index e99522efa..93568cc26 100644 --- a/pkg/worker/media/media_test.go +++ b/pkg/worker/media/media_test.go @@ -6,9 +6,8 @@ import ( "reflect" "testing" + "github.com/giongto35/cloud-game/v3/pkg/config" "github.com/giongto35/cloud-game/v3/pkg/encoder" - "github.com/giongto35/cloud-game/v3/pkg/encoder/h264" - "github.com/giongto35/cloud-game/v3/pkg/encoder/vpx" "github.com/giongto35/cloud-game/v3/pkg/logger" ) @@ -38,15 +37,36 @@ func BenchmarkH264(b *testing.B) { run(1920, 1080, encoder.H264, b.N, nil, nil, func BenchmarkVP8(b *testing.B) { run(1920, 1080, encoder.VP8, b.N, nil, nil, b) } func run(w, h int, cod encoder.VideoCodec, count int, a *image.RGBA, b *image.RGBA, backend testing.TB) { - var enc encoder.Encoder - if cod == encoder.H264 { - enc, _ = h264.NewEncoder(w, h, nil) - } else { - enc, _ = vpx.NewEncoder(w, h, nil) + conf := config.Video{ + Codec: string(cod), + Threads: 0, + H264: struct { + Crf uint8 + LogLevel int32 + Preset string + Profile string + Tune string + }{ + Crf: 30, + LogLevel: 0, + Preset: "ultrafast", + Profile: "baseline", + Tune: "zerolatency", + }, + Vpx: struct { + Bitrate uint + KeyframeInterval uint + }{ + Bitrate: 1000, + KeyframeInterval: 5, + }, } logger.SetGlobalLevel(logger.Disabled) - ve := encoder.NewVideoEncoder(enc, w, h, 1, l) + ve, err := encoder.NewVideoEncoder(w, h, w, h, 1, conf, l) + if err != nil { + backend.Error(err) + } defer ve.Stop() if a == nil {