Update video encoders

pkg/config/config.yaml

@@ -260,6 +260,8 @@ encoder:
   video:
     # h264, vpx (VP8)
     codec: h264
+    # Threaded encoder if supported, 0 - auto, 1 - nope, >1 - multi-threaded
+    threads: 0
     # see: https://trac.ffmpeg.org/wiki/Encode/H.264
     h264:
       # Constant Rate Factor (CRF) 0-51 (default: 23)

pkg/config/worker.go

@@ -52,8 +52,9 @@ type Audio struct {
 }
 
 type Video struct {
-	Codec string
-	H264  struct {
+	Codec   string
+	Threads int
+	H264    struct {
 		Crf      uint8
 		LogLevel int32
 		Preset   string

pkg/encoder/encoder.go

@@ -2,9 +2,11 @@ package encoder
 
 import (
 	"fmt"
-	"sync"
 	"sync/atomic"
 
+	"github.com/giongto35/cloud-game/v3/pkg/config"
+	"github.com/giongto35/cloud-game/v3/pkg/encoder/h264"
+	"github.com/giongto35/cloud-game/v3/pkg/encoder/vpx"
 	"github.com/giongto35/cloud-game/v3/pkg/encoder/yuv"
 	"github.com/giongto35/cloud-game/v3/pkg/logger"
 )
@@ -16,6 +18,7 @@ type (
 		LoadBuf(input []byte)
 		Encode() []byte
 		IntraRefresh()
+		Info() string
 		SetFlip(bool)
 		Shutdown() error
 	}
@@ -28,28 +31,42 @@ type Video struct {
 	y       yuv.Conv
 	pf      yuv.PixFmt
 	rot     uint
-	mu      sync.Mutex
 }
 
 type VideoCodec string
 
 const (
 	H264 VideoCodec = "h264"
 	VP8  VideoCodec = "vp8"
+	VPX  VideoCodec = "vpx"
 )
 
 // NewVideoEncoder returns new video encoder.
 // By default, it waits for RGBA images on the input channel,
 // converts them into YUV I420 format,
 // encodes with provided video encoder, and
 // puts the result into the output channel.
-func NewVideoEncoder(codec Encoder, w, h int, scale float64, log *logger.Logger) *Video {
-	return &Video{codec: codec, y: yuv.NewYuvConv(w, h, scale), log: log}
+func NewVideoEncoder(w, h, dw, dh int, scale float64, conf config.Video, log *logger.Logger) (*Video, error) {
+	var enc Encoder
+	var err error
+	switch VideoCodec(conf.Codec) {
+	case H264:
+		opts := h264.Options(conf.H264)
+		enc, err = h264.NewEncoder(dw, dh, conf.Threads, &opts)
+	case VP8, VPX:
+		opts := vpx.Options(conf.Vpx)
+		enc, err = vpx.NewEncoder(dw, dh, &opts)
+	default:
+		err = fmt.Errorf("unsupported codec: %v", conf.Codec)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return &Video{codec: enc, y: yuv.NewYuvConv(w, h, scale), log: log}, nil
 }
 
 func (v *Video) Encode(frame InFrame) OutFrame {
-	v.mu.Lock()
-	defer v.mu.Unlock()
 	if v.stopped.Load() {
 		return nil
 	}
@@ -64,7 +81,9 @@ func (v *Video) Encode(frame InFrame) OutFrame {
 	return nil
 }
 
-func (v *Video) Info() string { return fmt.Sprintf("libyuv: %v", v.y.Version()) }
+func (v *Video) Info() string {
+	return fmt.Sprintf("%v, libyuv: %v", v.codec.Info(), v.y.Version())
+}
 
 func (v *Video) SetPixFormat(f uint32) {
 	switch f {
@@ -77,28 +96,23 @@ func (v *Video) SetPixFormat(f uint32) {
 	}
 }
 
-// SetRot sets the rotation angle of the frames.
-func (v *Video) SetRot(r uint) {
-	switch r {
-	// de-rotate
-	case 90:
-		v.rot = 270
-	case 270:
-		v.rot = 90
-	default:
-		v.rot = r
+// SetRot sets the de-rotation angle of the frames.
+func (v *Video) SetRot(a uint) {
+	if a > 0 {
+		v.rot = (a + 180) % 360
 	}
 }
 
 // SetFlip tells the encoder to flip the frames vertically.
 func (v *Video) SetFlip(b bool) { v.codec.SetFlip(b) }
 
 func (v *Video) Stop() {
-	v.stopped.Store(true)
-	v.mu.Lock()
-	defer v.mu.Unlock()
+	if v.stopped.Swap(true) {
+		return
+	}
 	v.rot = 0
 
+	defer func() { v.codec = nil }()
 	if err := v.codec.Shutdown(); err != nil {
 		v.log.Error().Err(err).Msg("failed to close the encoder")
 	}

pkg/encoder/h264/libx264.go

@@ -8,6 +8,11 @@ package h264
 #include "stdint.h"
 #include "x264.h"
 #include <stdlib.h>
+
+static int x264_encode( x264_t *h, uintptr_t pp_nal, int *pi_nal, x264_picture_t *pic_in, x264_picture_t *pic_out ) {
+	return x264_encoder_encode(h, (x264_nal_t **)pp_nal, pi_nal, pic_in, pic_out);
+}
+
 */
 import "C"
 import "unsafe"
@@ -505,16 +510,16 @@ func EncoderOpen(param *Param) *T {
 
 // EncoderEncode - encode one picture.
 // Returns the number of bytes in the returned NALs, negative on error and zero if no NAL units returned.
-func EncoderEncode(enc *T, ppNal []*Nal, piNal *int32, picIn *Picture, picOut *Picture) int32 {
+func EncoderEncode(enc *T, ppNal **Nal, piNal *int32, picIn *Picture, picOut *Picture) int32 {
 	cenc := enc.cptr()
 
-	cppNal := (**C.x264_nal_t)(unsafe.Pointer(&ppNal[0]))
+	cppNal := C.uintptr_t(uintptr(unsafe.Pointer(ppNal)))
 	cpiNal := (*C.int)(unsafe.Pointer(piNal))
 
 	cpicIn := picIn.cptr()
 	cpicOut := picOut.cptr()
 
-	return (int32)(C.x264_encoder_encode(cenc, cppNal, cpiNal, cpicIn, cpicOut))
+	return (int32)(C.x264_encode(cenc, cppNal, cpiNal, cpicIn, cpicOut))
 }
 
 // EncoderClose closes an encoder handler.

pkg/encoder/h264/x264.go

@@ -8,12 +8,10 @@ import (
 type H264 struct {
 	ref *T
 
-	width      int32
-	lumaSize   int32
-	chromaSize int32
-	nnals      int32
-	nals       []*Nal
-
+	pnals   *Nal  // array of NALs
+	nnals   int32 // number of NALs
+	y       int32 // Y size
+	uv      int32 // U or V size
 	in, out *Picture
 }
 
@@ -32,11 +30,11 @@ type Options struct {
 	Tune string
 }
 
-func NewEncoder(w, h int, opts *Options) (encoder *H264, err error) {
+func NewEncoder(w, h int, th int, opts *Options) (encoder *H264, err error) {
 	libVersion := LibVersion()
 
-	if libVersion < 150 {
-		return nil, fmt.Errorf("x264: the library version should be newer than v150, you have got version %v", libVersion)
+	if libVersion < 156 {
+		return nil, fmt.Errorf("x264: the library version should be newer than v155, you have got version %v", libVersion)
 	}
 
 	if opts == nil {
@@ -63,39 +61,28 @@ func NewEncoder(w, h int, opts *Options) (encoder *H264, err error) {
 		}
 	}
 
-	// legacy encoder lacks of this param
-	param.IBitdepth = 8
+	ww, hh := int32(w), int32(h)
 
-	if libVersion > 155 {
-		param.ICsp = CspI420
-	} else {
-		param.ICsp = 1
-	}
-	param.IWidth = int32(w)
-	param.IHeight = int32(h)
+	param.IBitdepth = 8
+	param.ICsp = CspI420
+	param.IWidth = ww
+	param.IHeight = hh
 	param.ILogLevel = opts.LogLevel
 	param.ISyncLookahead = 0
-	param.IThreads = 1
-
+	param.IThreads = int32(th)
+	if th != 1 {
+		param.BSlicedThreads = 1
+	}
 	param.Rc.IRcMethod = RcCrf
 	param.Rc.FRfConstant = float32(opts.Crf)
 
 	encoder = &H264{
-		lumaSize:   param.IWidth * param.IHeight,
-		chromaSize: param.IWidth * param.IHeight / 4,
-		nals:       make([]*Nal, 1),
-		width:      param.IWidth,
-		out:        new(Picture),
+		y:     ww * hh,
+		uv:    ww * hh / 4,
+		pnals: new(Nal),
+		out:   new(Picture),
 		in: &Picture{
-			Img: Image{
-				ICsp:   param.ICsp,
-				IPlane: 3,
-				IStride: [4]int32{
-					0: param.IWidth,
-					1: param.IWidth >> 1,
-					2: param.IWidth >> 1,
-				},
-			},
+			Img: Image{ICsp: param.ICsp, IPlane: 3, IStride: [4]int32{0: ww, 1: ww >> 1, 2: ww >> 1}},
 		},
 	}
 
@@ -109,23 +96,27 @@ func LibVersion() int { return int(Build) }
 
 func (e *H264) LoadBuf(yuv []byte) {
 	e.in.Img.Plane[0] = uintptr(unsafe.Pointer(&yuv[0]))
-	e.in.Img.Plane[1] = uintptr(unsafe.Pointer(&yuv[e.lumaSize]))
-	e.in.Img.Plane[2] = uintptr(unsafe.Pointer(&yuv[e.lumaSize+e.chromaSize]))
+	e.in.Img.Plane[1] = uintptr(unsafe.Pointer(&yuv[e.y]))
+	e.in.Img.Plane[2] = uintptr(unsafe.Pointer(&yuv[e.y+e.uv]))
 }
 
-func (e *H264) Encode() []byte {
+func (e *H264) Encode() (b []byte) {
 	e.in.IPts += 1
-	if ret := EncoderEncode(e.ref, e.nals, &e.nnals, e.in, e.out); ret > 0 {
-		return unsafe.Slice((*byte)(e.nals[0].PPayload), ret)
-		//return C.GoBytes(e.nals[0].PPayload, C.int(ret))
+	bytes := EncoderEncode(e.ref, &e.pnals, &e.nnals, e.in, e.out)
+	if bytes > 0 {
+		// we merge multiple NALs stored in **pnals into a single byte stream
+		// ret contains the total size of NALs in bytes, i.e. each e.pnals[...].PPayload * IPayload
+		b = unsafe.Slice((*byte)(e.pnals.PPayload), bytes)
 	}
-	return []byte{}
+	return
 }
 
 func (e *H264) IntraRefresh() {
 	// !to implement
 }
 
+func (e *H264) Info() string { return fmt.Sprintf("x264: v%v", LibVersion()) }
+
 func (e *H264) SetFlip(b bool) {
 	if b {
 		e.in.Img.ICsp |= CspVflip

pkg/encoder/h264/x264_test.go

@@ -3,7 +3,7 @@ package h264
 import "testing"
 
 func TestH264Encode(t *testing.T) {
-	h264, err := NewEncoder(120, 120, nil)
+	h264, err := NewEncoder(120, 120, 0, nil)
 	if err != nil {
 		t.Error(err)
 	}
@@ -17,7 +17,7 @@ func TestH264Encode(t *testing.T) {
 
 func Benchmark(b *testing.B) {
 	w, h := 1920, 1080
-	h264, err := NewEncoder(w, h, nil)
+	h264, err := NewEncoder(w, h, 0, nil)
 	if err != nil {
 		b.Error(err)
 	}

pkg/encoder/vpx/libvpx.go

@@ -163,6 +163,8 @@ func (vpx *Vpx) Encode() []byte {
 	return C.GoBytes(fb.ptr, fb.size)
 }
 
+func (vpx *Vpx) Info() string { return fmt.Sprintf("vpx: %v", C.GoString(C.vpx_codec_version_str())) }
+
 func (vpx *Vpx) IntraRefresh() {
 	// !to implement
 }

pkg/worker/media/media.go

@@ -8,9 +8,7 @@ import (
 
 	"github.com/giongto35/cloud-game/v3/pkg/config"
 	"github.com/giongto35/cloud-game/v3/pkg/encoder"
-	"github.com/giongto35/cloud-game/v3/pkg/encoder/h264"
 	"github.com/giongto35/cloud-game/v3/pkg/encoder/opus"
-	"github.com/giongto35/cloud-game/v3/pkg/encoder/vpx"
 	"github.com/giongto35/cloud-game/v3/pkg/logger"
 	"github.com/giongto35/cloud-game/v3/pkg/worker/caged/app"
 )
@@ -145,6 +143,7 @@ func (wmp *WebrtcMediaPipe) Init() error {
 	if err := wmp.initVideo(wmp.VideoW, wmp.VideoH, wmp.VideoScale, wmp.vConf); err != nil {
 		return err
 	}
+	wmp.log.Debug().Msgf("%v", wmp.v.Info())
 	return nil
 }
 
@@ -175,29 +174,11 @@ func (wmp *WebrtcMediaPipe) encodeAudio(pcm samples) {
 	wmp.onAudio(data)
 }
 
-func (wmp *WebrtcMediaPipe) initVideo(w, h int, scale float64, conf config.Video) error {
-	var enc encoder.Encoder
-	var err error
-
+func (wmp *WebrtcMediaPipe) initVideo(w, h int, scale float64, conf config.Video) (err error) {
 	sw, sh := round(w, scale), round(h, scale)
-
-	wmp.log.Debug().Msgf("Scale: %vx%v -> %vx%v", w, h, sw, sh)
-
-	wmp.log.Info().Msgf("Video codec: %v", conf.Codec)
-	if conf.Codec == string(encoder.H264) {
-		wmp.log.Debug().Msgf("x264: build v%v", h264.LibVersion())
-		opts := h264.Options(conf.H264)
-		enc, err = h264.NewEncoder(sw, sh, &opts)
-	} else {
-		opts := vpx.Options(conf.Vpx)
-		enc, err = vpx.NewEncoder(sw, sh, &opts)
-	}
-	if err != nil {
-		return fmt.Errorf("couldn't create a video encoder: %w", err)
-	}
-	wmp.v = encoder.NewVideoEncoder(enc, w, h, scale, wmp.log)
-	wmp.log.Debug().Msgf("%v", wmp.v.Info())
-	return nil
+	wmp.v, err = encoder.NewVideoEncoder(w, h, sw, sh, scale, conf, wmp.log)
+	wmp.log.Debug().Msgf("media scale: %vx%v -> %vx%v", w, h, sw, sh)
+	return err
 }
 
 func round(x int, scale float64) int { return (int(float64(x)*scale) + 1) & ^1 }