diff --git a/Makefile b/Makefile index 12f0e64443..70e88127ad 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ output_dir := out/Default target_dir := target target_lib_dir := $(target_dir)/lib target_bin_dir := $(target_dir)/bin +target_pylib_dir := $(target_dir)/pylib compile_docker := alphartc-compile release_docker := alphartc @@ -37,7 +38,8 @@ peerconnection_serverless: make docker-$@ \ output_dir=$(output_dir) \ target_lib_dir=$(target_lib_dir) \ - target_bin_dir=$(target_bin_dir) + target_bin_dir=$(target_bin_dir) \ + target_pylib_dir=$(target_pylib_dir) # Docker internal command @@ -51,8 +53,14 @@ docker-app: docker-peerconnection_serverless docker-peerconnection_serverless: ninja -C $(output_dir) peerconnection_serverless + mkdir -p $(target_lib_dir) cp modules/third_party/onnxinfer/lib/*.so $(target_lib_dir) cp modules/third_party/onnxinfer/lib/*.so.* $(target_lib_dir) + mkdir -p $(target_bin_dir) - cp $(output_dir)/peerconnection_serverless $(target_bin_dir) + cp $(output_dir)/peerconnection_serverless $(target_bin_dir)/peerconnection_serverless.origin + cp examples/peerconnection/serverless/peerconnection_serverless $(target_bin_dir) + + mkdir -p $(target_pylib_dir) + cp modules/third_party/cmdinfer/*.py $(target_pylib_dir)/ diff --git a/README.md b/README.md index 1d61e26af8..d2d8320755 100644 --- a/README.md +++ b/README.md @@ -39,10 +39,12 @@ AlphaRTC is a fork of Google's WebRTC project using ML-based bandwidth estimation, delivered by the OpenNetLab team. By equipping WebRTC with a more accurate bandwidth estimator, our mission is to eventually increase the quality of transmission. -AlphaRTC replaces Google Congestion Control (GCC) with ONNXInfer, an ML-powered bandwidth estimator, which takes in an ONNX model to make bandwidth estimation more accurate. ONNXInfer is proudly powered by Microsoft's [ONNXRuntime](https://github.com/microsoft/onnxruntime). +AlphaRTC replaces Google Congestion Control (GCC) with two customized congestion control interfaces, PyInfer and ONNXInfer. The PyInfer provides an opportunity to load external bandwidth estimator written by Python. The external bandwidth estimator could be based on ML framework, like PyTorch or TensorFlow, or a pure Python algorithm without any dependencies. And the ONNXInfer is an ML-powered bandwidth estimator, which takes in an ONNX model to make bandwidth estimation more accurate. ONNXInfer is proudly powered by Microsoft's [ONNXRuntime](https://github.com/microsoft/onnxruntime). ## Environment +**We recommend you directly fetch the pre-provided Docker images from [Github release](https://github.com/OpenNetLab/AlphaRTC/releases/latest/download/alphartc.tar.gz)** + Ubuntu 18.04 is the only officially supported distro at this moment. For other distros, you may be able to compile your own binary, or use our pre-provided Docker images. ## Compilation @@ -107,7 +109,7 @@ Note: all commands below work for both Linux (sh) and Windows (pwsh), unless oth gn gen out/Default ``` -5. Comile +5. Compile ```shell ninja -C out/Default peerconnection_serverless ``` @@ -151,9 +153,6 @@ This section describes required fields for the json configuration file. - **bwe_feedback_duration**: The duration the receiver sends its estimated target rate every time(*in millisecond*) -- **onnx** - - **onnx_model_path**: The path of the [onnx](https://www.onnxruntime.ai/) model - - **video_source** - **video_disabled**: - **enabled**: If set to `true`, the client will not take any video source as input @@ -188,11 +187,57 @@ This section describes required fields for the json configuration file. - **fps**: Frames per second of the output video file - **file_path**: The file path of the output video file in YUV format +#### Use PyInfer or ONNXInfer + +##### PyInfer + +The default bandwidth estimator is PyInfer, You should implement your Python class named `Estimator` with required methods `report_states` and `get_estimated_bandwidth` in Python file `BandwidthEstimator.py ` and put this file in your workspace. +There is an example of Estimator with fixed estimated bandwidth 1Mbps. Here is an example [BandwidthEstimator.py](examples/peerconnection/serverless/corpus/BandwidthEstimator.py). + +```python +class Estimator(object): + def report_states(self, stats: dict): + ''' + stats is a dict with the following items + { + "send_time_ms": uint, + "arrival_time_ms": uint, + "payload_type": int, + "sequence_number": uint, + "ssrc": int, + "padding_length": uint, + "header_length": uint, + "payload_size": uint + } + ''' + pass + + def get_estimated_bandwidth(self)->int: + return int(1e6) # 1Mbps + +``` + +##### ONNXInfer + +If you want to use the ONNXInfer as the bandwidth estimator, you should specify the path of onnx model in the config file. Here is an example configuration [receiver.json](examples/peerconnection/serverless/corpus/receiver.json) + +- **onnx** + - **onnx_model_path**: The path of the [onnx](https://www.onnxruntime.ai/) model + + #### Run peerconnection_serverless - Dockerized environment To better demonstrate the usage of peerconnection_serverless, we provide an all-inclusive corpus in `examples/peerconnection/serverless/corpus`. You can use the following commands to execute a tiny example. After these commands terminates, you will get `outvideo.yuv` and `outaudio.wav`. - + + + PyInfer: + ```shell + sudo docker run -d --rm -v `pwd`/examples/peerconnection/serverless/corpus:/app -w /app --name alphartc alphartc peerconnection_serverless receiver_pyinfer.json + sudo docker exec alphartc peerconnection_serverless sender_pyinfer.json + ``` + + ONNXInfer: ``` shell sudo docker run -d --rm -v `pwd`/examples/peerconnection/serverless/corpus:/app -w /app --name alphartc alphartc peerconnection_serverless receiver.json sudo docker exec alphartc peerconnection_serverless sender.json diff --git a/api/alphacc_config.cc b/api/alphacc_config.cc index cf5d5690db..79bd24f812 100644 --- a/api/alphacc_config.cc +++ b/api/alphacc_config.cc @@ -68,10 +68,10 @@ bool ParseAlphaCCConfig(const std::string& file_path) { RETURN_ON_FAIL( GetInt(top, "bwe_feedback_duration", &config->bwe_feedback_duration_ms)); - RETURN_ON_FAIL(GetValue(top, "onnx", &second)); - RETURN_ON_FAIL( - GetString(second, "onnx_model_path", &config->onnx_model_path)); - second.clear(); + if (GetValue(top, "onnx", &second)) { + GetString(second, "onnx_model_path", &config->onnx_model_path); + second.clear(); + } bool enabled = false; RETURN_ON_FAIL(GetValue(top, "video_source", &second)); diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 887ee3b6eb..2e54a1aea7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -33,3 +33,28 @@ steps: - script: docker run -d --rm -v `pwd`/examples/peerconnection/serverless/corpus:/app -w /app --name alphartc alphartc peerconnection_serverless receiver.json && docker exec alphartc peerconnection_serverless sender.json displayName: 'run example' + +- script: docker run -d --rm -v `pwd`/examples/peerconnection/serverless/corpus:/app -w /app --name alphartc_pyinfer alphartc peerconnection_serverless receiver_pyinfer.json + && docker exec alphartc_pyinfer peerconnection_serverless sender_pyinfer.json + displayName: 'run pyinfer example' + +- script: docker save alphartc | gzip > alphartc.tar.gz + displayName: "Export alphartc docker image" + +- publish: $(System.DefaultWorkingDirectory)/alphartc.tar.gz + continueOnError: true + artifact: alphartc.tar.gz + displayName: "Archive AlphaRTC Peerconnection" + +- task: GitHubRelease@0 + inputs: + gitHubConnection: 'Pterosaur (1)' + repositoryName: '$(Build.Repository.Name)' + action: 'create' + tagSource: manual + tag: $(Build.BuildNumber) + title: alphartc + assets: '$(System.DefaultWorkingDirectory)/alphartc.tar.gz' + changeLogCompareToRelease: 'lastFullRelease' + changeLogType: 'commitBased' + displayName: "Release target" diff --git a/dockers/Dockerfile.release b/dockers/Dockerfile.release index 73c3d175fd..f33228ed3a 100644 --- a/dockers/Dockerfile.release +++ b/dockers/Dockerfile.release @@ -1,9 +1,10 @@ FROM ubuntu:18.04 RUN apt-get update && apt-get install -y \ - libx11-6 libgomp1 + libx11-6 libgomp1 python3 COPY lib /usr/lib/ COPY bin /usr/bin/ +COPY pylib /usr/lib/python3/dist-packages/ diff --git a/examples/peerconnection/serverless/corpus/BandwidthEstimator.py b/examples/peerconnection/serverless/corpus/BandwidthEstimator.py new file mode 100644 index 0000000000..9b30c979c6 --- /dev/null +++ b/examples/peerconnection/serverless/corpus/BandwidthEstimator.py @@ -0,0 +1,20 @@ + +class Estimator(object): + def report_states(self, stats: dict): + ''' + stats is a dict with the following items + { + "send_time_ms": uint, + "arrival_time_ms": uint, + "payload_type": int, + "sequence_number": uint, + "ssrc": int, + "padding_length": uint, + "header_length": uint, + "payload_size": uint + } + ''' + pass + + def get_estimated_bandwidth(self)->int: + return int(1e6) # 1Mbps diff --git a/examples/peerconnection/serverless/corpus/receiver_pyinfer.json b/examples/peerconnection/serverless/corpus/receiver_pyinfer.json new file mode 100644 index 0000000000..75e19cd8d9 --- /dev/null +++ b/examples/peerconnection/serverless/corpus/receiver_pyinfer.json @@ -0,0 +1,54 @@ +{ + "serverless_connection": { + "autoclose": 20, + "sender": { + "enabled": false + }, + "receiver": { + "enabled": true, + "listening_ip": "0.0.0.0", + "listening_port": 8000 + } + }, + "bwe_feedback_duration": 200, + "video_source": { + "video_disabled": { + "enabled": true + }, + "webcam": { + "enabled": false + }, + "video_file": { + "enabled": false, + "height": 240, + "width": 320, + "fps": 10, + "file_path": "testmedia/test.yuv" + } + }, + "audio_source": { + "microphone": { + "enabled": false + }, + "audio_file": { + "enabled": true, + "file_path": "testmedia/test.wav" + } + }, + "save_to_file": { + "enabled": true, + "audio": { + "file_path": "outaudio.wav" + }, + "video": { + "width": 320, + "height": 240, + "fps": 10, + "file_path": "outvideo.yuv" + } + }, + "logging": { + "enabled": true, + "log_output_path": "webrtc.log" + } +} \ No newline at end of file diff --git a/examples/peerconnection/serverless/corpus/sender_pyinfer.json b/examples/peerconnection/serverless/corpus/sender_pyinfer.json new file mode 100644 index 0000000000..3a1ccc7436 --- /dev/null +++ b/examples/peerconnection/serverless/corpus/sender_pyinfer.json @@ -0,0 +1,44 @@ +{ + "serverless_connection": { + "autoclose": 20, + "sender": { + "enabled": true, + "dest_ip": "0.0.0.0", + "dest_port": 8000 + }, + "receiver": { + "enabled": false + } + }, + "bwe_feedback_duration": 200, + "video_source": { + "video_disabled": { + "enabled": false + }, + "webcam": { + "enabled": false + }, + "video_file": { + "enabled": true, + "height": 240, + "width": 320, + "fps": 10, + "file_path": "testmedia/test.yuv" + } + }, + "audio_source": { + "microphone": { + "enabled": false + }, + "audio_file": { + "enabled": true, + "file_path": "testmedia/test.wav" + } + }, + "save_to_file": { + "enabled": false + }, + "logging": { + "enabled": false + } +} \ No newline at end of file diff --git a/examples/peerconnection/serverless/peerconnection_serverless b/examples/peerconnection/serverless/peerconnection_serverless new file mode 100755 index 0000000000..6924ded5be --- /dev/null +++ b/examples/peerconnection/serverless/peerconnection_serverless @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import sys +import os +import subprocess +import traceback +import json + +sys.path.append(os.getcwd()) + +import cmdinfer + + +def main(): + app = subprocess.Popen( + ["peerconnection_serverless.origin"] + sys.argv[1:], + bufsize=1, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + try: + cmdinfer.main(app.stdout, app.stdin) + app.wait() + except: + app.terminate() + app.wait() + error_message = traceback.format_exc() + error_message = "\n{}".format(error_message) + sys.stderr.write(error_message) + if len(sys.argv[1:]) == 0: + return + config_file = sys.argv[1] + config_file = json.load(open(config_file, "r")) + if "logging" not in config_file: + return + if "enabled" not in config_file["logging"] or not config_file["logging"]["enabled"]: + return + with open(config_file["logging"]["log_output_path"], "a") as log_file: + log_file.write(error_message) + + +if __name__ == "__main__": + main() diff --git a/modules/remote_bitrate_estimator/BUILD.gn b/modules/remote_bitrate_estimator/BUILD.gn index 2a8adab41c..3fbaa72c8e 100644 --- a/modules/remote_bitrate_estimator/BUILD.gn +++ b/modules/remote_bitrate_estimator/BUILD.gn @@ -62,7 +62,8 @@ rtc_library("remote_bitrate_estimator") { "../../system_wrappers:metrics", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", - "//modules/third_party/statcollect:stat_collect" + "//modules/third_party/statcollect:stat_collect", + "//modules/third_party/cmdinfer:cmdinfer" ] if (is_linux) { diff --git a/modules/remote_bitrate_estimator/remote_estimator_proxy.cc b/modules/remote_bitrate_estimator/remote_estimator_proxy.cc index 5787b17a10..6955963467 100644 --- a/modules/remote_bitrate_estimator/remote_estimator_proxy.cc +++ b/modules/remote_bitrate_estimator/remote_estimator_proxy.cc @@ -13,11 +13,13 @@ #endif // WIN32 #include "modules/remote_bitrate_estimator/remote_estimator_proxy.h" +#include "modules/third_party/cmdinfer/cmdinfer.h" #include #include #include #include +#include #include "api/alphacc_config.h" #include "modules/rtp_rtcp/source/rtcp_packet/transport_feedback.h" @@ -54,11 +56,15 @@ RemoteEstimatorProxy::RemoteEstimatorProxy( last_bwe_sendback_ms_(clock->TimeInMilliseconds()), stats_collect_(StatCollect::SC_TYPE_STRUCT), cycles_(-1), - max_abs_send_time_(0) { - onnx_infer_ = onnxinfer::CreateONNXInferInterface( - GetAlphaCCConfig()->onnx_model_path.c_str()); - if (!onnxinfer::IsReady(onnx_infer_)) { - RTC_LOG(LS_ERROR) << "Failed to create onnx_infer_."; + max_abs_send_time_(0), + onnx_infer_(nullptr) { + + if (!GetAlphaCCConfig()->onnx_model_path.empty()) { + onnx_infer_ = onnxinfer::CreateONNXInferInterface( + GetAlphaCCConfig()->onnx_model_path.c_str()); + if (!onnxinfer::IsReady(onnx_infer_)) { + RTC_LOG(LS_ERROR) << "Failed to create onnx_infer_."; + } } RTC_LOG(LS_INFO) << "Maximum interval between transport feedback RTCP messages (ms): " @@ -89,16 +95,32 @@ void RemoteEstimatorProxy::IncomingPacket(int64_t arrival_time_ms, // lossCound and RTT field for onnxinfer::OnReceived() are set to -1 since // no available lossCound and RTT in webrtc - onnxinfer::OnReceived(onnx_infer_, header.payloadType, header.sequenceNumber, - send_time_ms, header.ssrc, header.paddingLength, - header.headerLength, arrival_time_ms, payload_size, -1, -1); + if (onnx_infer_) { + onnxinfer::OnReceived(onnx_infer_, header.payloadType, header.sequenceNumber, + send_time_ms, header.ssrc, header.paddingLength, + header.headerLength, arrival_time_ms, payload_size, -1, -1); + } else { + cmdinfer::ReportStates( + send_time_ms, + arrival_time_ms, + payload_size, + header.payloadType, + header.sequenceNumber, + header.ssrc, + header.paddingLength, + header.headerLength); + } //--- BandWidthControl: Send back bandwidth estimation into to sender --- bool time_to_send_bew_message = TimeToSendBweMessage(); float estimation = 0; if (time_to_send_bew_message) { BweMessage bwe; - estimation = onnxinfer::GetBweEstimate(onnx_infer_); + if (onnx_infer_) { + estimation = onnxinfer::GetBweEstimate(onnx_infer_); + } else { + estimation = cmdinfer::GetEstimatedBandwidth(); + } bwe.pacing_rate = bwe.padding_rate = bwe.target_rate = estimation; bwe.timestamp_ms = clock_->TimeInMilliseconds(); SendbackBweEstimation(bwe); diff --git a/modules/third_party/cmdinfer/BUILD.gn b/modules/third_party/cmdinfer/BUILD.gn new file mode 100644 index 0000000000..60320c80e3 --- /dev/null +++ b/modules/third_party/cmdinfer/BUILD.gn @@ -0,0 +1,9 @@ +static_library("cmdinfer") { + deps = [ + "//modules/third_party/statcollect:stat_collect", + ] + sources = [ + "cmdinfer.h", + "cmdinfer.cc", + ] +} diff --git a/modules/third_party/cmdinfer/cmdinfer.cc b/modules/third_party/cmdinfer/cmdinfer.cc new file mode 100644 index 0000000000..1c7192f1a6 --- /dev/null +++ b/modules/third_party/cmdinfer/cmdinfer.cc @@ -0,0 +1,38 @@ +#include "cmdinfer.h" + +#include "modules/third_party/statcollect/json.hpp" + +#include + + +const char * RequestBandwidthCommand = "RequestBandwidth"; + +void cmdinfer::ReportStates( + std::uint64_t sendTimeMs, + std::uint64_t receiveTimeMs, + std::size_t payloadSize, + std::uint8_t payloadType, + std::uint16_t sequenceNumber, + std::uint32_t ssrc, + std::size_t paddingLength, + std::size_t headerLength) { + + nlohmann::json j; + j["send_time_ms"] = sendTimeMs; + j["arrival_time_ms"] = receiveTimeMs; + j["payload_type"] = payloadType; + j["sequence_number"] = sequenceNumber; + j["ssrc"] = ssrc; + j["padding_length"] = paddingLength; + j["header_length"] = headerLength; + j["payload_size"] = payloadSize; + + std::cout << j.dump() << std::endl; +} + +float cmdinfer::GetEstimatedBandwidth() { + std::uint64_t bandwidth = 0; + std::cout << RequestBandwidthCommand << std::endl; + std::cin >> bandwidth; + return static_cast(bandwidth); +} diff --git a/modules/third_party/cmdinfer/cmdinfer.h b/modules/third_party/cmdinfer/cmdinfer.h new file mode 100644 index 0000000000..0847fe4869 --- /dev/null +++ b/modules/third_party/cmdinfer/cmdinfer.h @@ -0,0 +1,21 @@ +#ifndef MODULES_THIRD_PARTY_CMDINFER_H_ +#define MODULES_THIRD_PARTY_CMDINFER_H_ + +#include +#include + +namespace cmdinfer { + void ReportStates( + std::uint64_t sendTimeMs, + std::uint64_t receiveTimeMs, + std::size_t payloadSize, + std::uint8_t payloadType, + std::uint16_t sequenceNumber, + std::uint32_t ssrc, + std::size_t paddingLength, + std::size_t headerLength); + + float GetEstimatedBandwidth(); +} + +#endif diff --git a/modules/third_party/cmdinfer/cmdinfer.py b/modules/third_party/cmdinfer/cmdinfer.py new file mode 100755 index 0000000000..6779e1e78d --- /dev/null +++ b/modules/third_party/cmdinfer/cmdinfer.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import sys +import json +import glob + + +RequestBandwidthCommand = "RequestBandwidth" + + +def fetch_stats(line: str)->dict: + line = line.strip() + try: + stats = json.loads(line) + return stats + except json.decoder.JSONDecodeError: + return None + + +def request_estimated_bandwidth(line: str)->bool: + line = line.strip() + if RequestBandwidthCommand == line: + return True + return False + + +def find_estimator_class(): + import BandwidthEstimator + return BandwidthEstimator.Estimator + + +def main(ifd = sys.stdin, ofd = sys.stdout): + estimator_class = find_estimator_class() + estimator = estimator_class() + while True: + line = ifd.readline() + if not line: + break + if isinstance(line, bytes): + line = line.decode("utf-8") + stats = fetch_stats(line) + if stats: + estimator.report_states(stats) + continue + request = request_estimated_bandwidth(line) + if request: + bandwidth = estimator.get_estimated_bandwidth() + ofd.write("{}\n".format(int(bandwidth)).encode("utf-8")) + ofd.flush() + continue + sys.stdout.write(line) + sys.stdout.flush() + + +if __name__ == '__main__': + main() diff --git a/modules/third_party/statcollect/StatCollect.cpp b/modules/third_party/statcollect/StatCollect.cpp index d3cbfc3960..b9a1ad292a 100644 --- a/modules/third_party/statcollect/StatCollect.cpp +++ b/modules/third_party/statcollect/StatCollect.cpp @@ -207,7 +207,7 @@ namespace StatCollect { totalSamplesReceived, concealedSamples, concealmentEvents); - std::string* tempStrPrt = &collectInfoJson; + std::string* tempStrPrt = new std::string(collectInfoJson); resultPtr = static_cast(tempStrPrt); } else {