-
Notifications
You must be signed in to change notification settings - Fork 214
feat: upgrade libwebrtc to m150. #1284
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
5650af3
da62c24
c7723dd
baae74a
11f6bd0
49dbff1
f4c43f2
b216c99
2ebb2bc
7bc323e
ffd1d5a
1b652e9
27a5748
d1b7fdb
28558ee
bf52ad6
9413ea5
5ec3038
774a9bc
2ce8677
fc7c1c3
f1eb9e2
504e46b
309c4b3
6e3c62e
09062d7
6a894c2
b494632
50da6c6
0c3c326
2bb70b5
c08c998
60a261d
7454992
9500e32
6643976
c410744
cbec2b8
2da3891
0b5d9be
b75ae7e
d40f016
9a57219
988c5e5
db162a7
6691a78
f6083e6
4e9896c
b7e9c5a
75555bc
29ba605
3e01861
9f0c332
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| --- | ||
| libwebrtc: patch | ||
| livekit: patch | ||
| livekit-ffi: patch | ||
| webrtc-sys: patch | ||
| webrtc-sys-build: patch | ||
| --- | ||
|
|
||
| feat: upgrade libwebrtc to m150. - #1284 (@cloudwebrtc) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| #!/usr/bin/env bash | ||
| # | ||
| # Install a clang new enough to build webrtc-sys on Linux. | ||
| # | ||
| # webrtc-sys compiles against the hermetic libc++ shipped inside the libwebrtc | ||
| # artifact (use_custom_libcxx=true). That libc++ tracks LLVM trunk and uses | ||
| # builtins that only exist in a recent clang, so the distro clang in our build | ||
| # images (AlmaLinux 8: clang 17, Ubuntu 24: clang 18) fails deep inside <limits> | ||
| # and <span> instead of saying the compiler is too old. webrtc-sys/build.rs | ||
| # checks the version up front and reports the real floor, which it reads from | ||
| # the artifact's own __configuration/compiler.h. | ||
| # | ||
| # The distro package managers have nothing recent enough, so pull the official | ||
| # LLVM release tarball instead. If a build image ever rejects it for glibc | ||
| # reasons, LLVM_VERSION is the knob to turn. | ||
| # | ||
| # Prints the bin directory on stdout; everything else goes to stderr. Also | ||
| # exports CC/CXX via $GITHUB_ENV when running as a workflow step. | ||
| # | ||
| # Usage: | ||
| # runner: .github/scripts/install-clang.sh # sets CC/CXX for later steps | ||
| # docker: export LLVM_ROOT=/opt/llvm | ||
| # .github/scripts/install-clang.sh | ||
| # export CC=$LLVM_ROOT/bin/clang CXX=$LLVM_ROOT/bin/clang++ | ||
|
|
||
| set -euo pipefail | ||
|
|
||
| LLVM_VERSION="${LLVM_VERSION:-21.1.8}" | ||
| LLVM_ROOT="${LLVM_ROOT:-/opt/llvm-$LLVM_VERSION}" | ||
|
|
||
| case "$(uname -m)" in | ||
| x86_64) llvm_arch=X64 ;; | ||
| aarch64 | arm64) llvm_arch=ARM64 ;; | ||
| *) echo "install-clang.sh: unsupported architecture $(uname -m)" >&2; exit 1 ;; | ||
| esac | ||
|
|
||
| if [ "$(id -u)" -eq 0 ]; then | ||
| sudo="" | ||
| else | ||
| sudo="sudo" | ||
| fi | ||
|
|
||
| if [ ! -x "$LLVM_ROOT/bin/clang++" ]; then | ||
| url="https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/LLVM-$LLVM_VERSION-Linux-$llvm_arch.tar.xz" | ||
| echo "install-clang.sh: fetching $url" >&2 | ||
| $sudo mkdir -p "$LLVM_ROOT" | ||
| # --strip-components=1 drops the LLVM-<version>-Linux-<arch>/ prefix. | ||
| curl --fail --location --silent --show-error "$url" \ | ||
| | $sudo tar -xJ --strip-components=1 -C "$LLVM_ROOT" | ||
|
devin-ai-integration[bot] marked this conversation as resolved.
devin-ai-integration[bot] marked this conversation as resolved.
devin-ai-integration[bot] marked this conversation as resolved.
|
||
| fi | ||
|
devin-ai-integration[bot] marked this conversation as resolved.
|
||
|
|
||
| "$LLVM_ROOT/bin/clang++" --version >&2 | ||
|
|
||
| if [ -n "${GITHUB_ENV:-}" ]; then | ||
| { | ||
| echo "CC=$LLVM_ROOT/bin/clang" | ||
| echo "CXX=$LLVM_ROOT/bin/clang++" | ||
| } >> "$GITHUB_ENV" | ||
| fi | ||
|
|
||
| echo "$LLVM_ROOT/bin" | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -216,12 +216,15 @@ jobs: | |
| docker run --rm -e CARGO_RUST_PROFILE="$CARGO_RUST_PROFILE" -v $PWD:/workspace -w /workspace ${{ matrix.build_image }} bash -c "\ | ||
| uname -a; \ | ||
| export PATH=/root/.cargo/bin:\$PATH; \ | ||
| yum install llvm llvm-libs lld -y; \ | ||
| yum install clang -y; \ | ||
| yum install clang clang-devel lld -y; \ | ||
| yum install protobuf-compiler -y; \ | ||
| yum groupinstall 'Development Tools' -y; \ | ||
| clang --version; \ | ||
| yum install openssl-devel libX11-devel mesa-libGL-devel libXext-devel libva-devel libdrm-devel libgbm-devel libXdamage-devel libXrandr-devel libXfixes-devel libXcomposite-devel -y; \ | ||
| clang --version; \ | ||
| clang++ --version; \ | ||
| export LIBCLANG_PATH=/usr/lib64; \ | ||
| export CC=clang; \ | ||
| export CXX=clang++; \ | ||
|
Comment on lines
+219
to
+227
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🔴 Linux release binaries can no longer be built because the build container installs an outdated compiler The Linux packaging container still installs and uses the distribution's own compiler ( Compiler floor enforced by webrtc-sys/build.rs vs. the manylinux images
The FFI Linux job builds inside Prompt for agentsWas this helpful? React with 👍 or 👎 to provide feedback. |
||
| curl --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y; \ | ||
| cd livekit-ffi && cargo build --profile \"\$CARGO_RUST_PROFILE\" --target ${{ matrix.target }} ${{ matrix.buildargs }}" | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -176,6 +176,8 @@ fn main() { | |
| println!("cargo:rustc-link-lib=dylib=pthread"); | ||
| println!("cargo:rustc-link-lib=dylib=m"); | ||
|
|
||
| configure_hermetic_libcxx(&mut builder, &webrtc_include); | ||
|
|
||
| // In order to avoid any ABI mismatches we use the sysroot's headers. | ||
| add_gio_headers(&mut builder); | ||
|
|
||
|
|
@@ -374,7 +376,12 @@ fn main() { | |
| println!("cargo:rustc-link-lib=c++abi"); | ||
|
|
||
| configure_android_sysroot(&mut builder); | ||
| builder.file("src/android.cpp").flag("-std=c++20"); | ||
| builder | ||
| .file("src/android.cpp") | ||
| // Provides a weak stub for std::__ndk1::__hash_memory, which | ||
| // was removed from libc++_static.a exports in NDK r28. | ||
| .file("src/ndk_compat.cpp") | ||
| .flag("-std=c++20"); | ||
| } | ||
| _ => { | ||
| panic!("Unsupported target, {}", target_os); | ||
|
|
@@ -491,6 +498,133 @@ fn add_lazy_load_so(builder: &mut cc::Build, name: &str, libraries: Vec<String>) | |
| } | ||
| } | ||
|
|
||
| /// Compile against the same hermetic libc++ that is baked into libwebrtc.a. | ||
| /// | ||
| /// The Linux libwebrtc build sets `use_custom_libcxx=true`, so every std type in | ||
| /// its public API lives in the `std::__Cr` ABI namespace with libc++ layouts. | ||
| /// Using the host's libstdc++ here instead is not merely a mangling mismatch that | ||
| /// the linker would catch: `std::span` is layout-different between the two, so a | ||
| /// span handed to libwebrtc silently arrives with its pointer and size swapped. | ||
| /// | ||
| /// Mirrors the flags in the WebRTC checkout's `build/config/c++/BUILD.gn`. The | ||
| /// matching `_LIBCPP_*` defines come from webrtc.ninja via `webrtc_defines()`. | ||
| fn configure_hermetic_libcxx(builder: &mut cc::Build, webrtc_include: &path::Path) { | ||
| let libcxx = webrtc_include.join("third_party/libc++/src/include"); | ||
| let libcxxabi = webrtc_include.join("third_party/libc++abi/src/include"); | ||
| if !libcxx.join("span").exists() { | ||
| panic!( | ||
| "hermetic libc++ headers missing from {}.\n\ | ||
| This libwebrtc artifact predates use_custom_libcxx=true; rebuild it with \ | ||
| build_linux.sh or point LK_CUSTOM_WEBRTC at a newer one.", | ||
| libcxx.display() | ||
| ); | ||
| } | ||
|
|
||
| // Chromium's libc++ is clang-only. At _LIBCPP_ABI_VERSION 2 it marks unique_ptr | ||
| // and shared_ptr __attribute__((trivial_abi)), which GCC accepts and silently | ||
| // ignores (a -Wattributes warning that cc's `-w` swallows). That attribute | ||
| // changes the calling convention, not just layout: libwebrtc.a returns | ||
| // std::unique_ptr in a register, while a GCC caller reads it back from an sret | ||
| // slot the callee never wrote, yielding a garbage pointer at the first use. | ||
| if env::var_os("CXX").is_none() { | ||
| if Command::new("clang++").arg("--version").output().is_err() { | ||
| panic!( | ||
| "clang++ is required to build webrtc-sys on Linux: libwebrtc.a is built \ | ||
| against Chromium's hermetic libc++, whose trivial_abi annotations GCC \ | ||
| ignores, which silently breaks the calling convention for std::unique_ptr \ | ||
| and std::shared_ptr. Install clang, or set CXX to a clang.", | ||
| ); | ||
| } | ||
| builder.compiler("clang++"); | ||
| } | ||
|
Comment on lines
+529
to
+539
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟡 Cross-compiler settings are ignored when building for Linux, forcing the host compiler The Linux build forces the host cc crate env-var precedence not covered by the CXX checkThe Prompt for agentsWas this helpful? React with 👍 or 👎 to provide feedback. |
||
|
|
||
| check_clang_version(builder, &libcxx); | ||
|
|
||
| builder | ||
| .flag("-nostdinc++") | ||
| .flag(format!("-isystem{}", libcxx.display())) | ||
| .flag(format!("-isystem{}", libcxxabi.display())) | ||
| // Holds __config_site, which pins _LIBCPP_ABI_NAMESPACE=__Cr. | ||
| .include(webrtc_include.join("buildtools/third_party/libc++")); | ||
|
|
||
| // libc++/libc++abi are already archived into libwebrtc.a, so linking the | ||
| // host libstdc++ on top would only add a second, incompatible stdlib. | ||
| builder.cpp_link_stdlib(None); | ||
|
|
||
| // The cxx crate builds its own runtime (cxx.cc) with the host default stdlib, | ||
| // so the rust::String <-> std::string conversions it exports are mangled for | ||
| // libstdc++ and cannot satisfy the std::__Cr call sites in the generated | ||
| // bridges. Compile a second copy with the flags above to provide those. | ||
| // DEP_CXXBRIDGE1_HEADER is `cargo:HEADER` from the cxx crate: <root>/include/cxx.h. | ||
| let cxx_h = env::var("DEP_CXXBRIDGE1_HEADER") | ||
| .expect("cxx crate did not export HEADER; cannot locate its cxx.cc"); | ||
| let cxx_root = path::Path::new(&cxx_h) | ||
| .parent() | ||
| .and_then(path::Path::parent) | ||
| .expect("unexpected DEP_CXXBRIDGE1_HEADER layout"); | ||
| builder.file(cxx_root.join("src/cxx.cc")); | ||
| } | ||
|
|
||
| /// The hermetic libc++ tracks LLVM trunk, so it freely uses builtins that only | ||
| /// exist in a recent clang (`__builtin_popcountg`, `__is_nothrow_convertible`, | ||
| /// `__GCC_DESTRUCTIVE_SIZE`, ...). A compiler below its floor does not fail with | ||
| /// "your clang is too old" — it fails deep inside <limits> and <span> with | ||
| /// hundreds of lines about `dynamic_extent` not being a constant expression, in | ||
| /// headers the user never wrote. Catch it up front instead. | ||
| fn check_clang_version(builder: &cc::Build, libcxx: &path::Path) { | ||
| let min = libcxx_min_clang_major(libcxx); | ||
|
|
||
| let compiler = builder.get_compiler(); | ||
| let defines = compiler | ||
| .to_command() | ||
| .args(["-dM", "-E", "-x", "c++", "/dev/null"]) | ||
| .output() | ||
| .unwrap_or_else(|e| panic!("failed to run {}: {e}", compiler.path().display())); | ||
|
|
||
| let major = String::from_utf8_lossy(&defines.stdout).lines().find_map(|line| { | ||
| line.strip_prefix("#define __clang_major__ ").and_then(|v| v.trim().parse::<u32>().ok()) | ||
| }); | ||
|
|
||
| match major { | ||
| Some(major) if major >= min => {} | ||
| Some(major) => panic!( | ||
| "{} is clang {major}, but the hermetic libc++ shipped with this libwebrtc \ | ||
| requires clang {min} or later. Install a newer clang and point CC/CXX at it, \ | ||
| or use the exact toolchain libwebrtc was built with (see CR_CLANG_REVISION in \ | ||
| the artifact's webrtc.ninja).", | ||
| compiler.path().display(), | ||
| ), | ||
| None => panic!( | ||
| "{} does not define __clang_major__, so it is not a clang. libwebrtc.a is built \ | ||
| against Chromium's hermetic libc++, which requires clang {min} or later; GCC \ | ||
| additionally ignores its trivial_abi annotations, silently breaking the calling \ | ||
| convention for std::unique_ptr and std::shared_ptr.", | ||
| compiler.path().display(), | ||
| ), | ||
| } | ||
| } | ||
|
|
||
| /// libc++ states its own floor in `__configuration/compiler.h`, as | ||
| /// `#if _LIBCPP_CLANG_VER < 2101` (major * 100 + minor). Read it from the | ||
| /// artifact rather than hardcoding, so a libwebrtc bump moves the floor with it. | ||
| fn libcxx_min_clang_major(libcxx: &path::Path) -> u32 { | ||
| const FALLBACK: u32 = 21; | ||
|
|
||
| let header = libcxx.join("__configuration/compiler.h"); | ||
| let Ok(source) = std::fs::read_to_string(&header) else { | ||
| return FALLBACK; | ||
| }; | ||
|
|
||
| source | ||
| .lines() | ||
| .find_map(|line| { | ||
| let (_, rest) = line.split_once("_LIBCPP_CLANG_VER < ")?; | ||
| let ver: u32 = rest.trim().parse().ok()?; | ||
| Some(ver / 100) | ||
| }) | ||
| .unwrap_or(FALLBACK) | ||
|
Comment on lines
+618
to
+625
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟡 Compiler version check accepts a compiler one point release too old The required compiler version is rounded down to whole numbers when it is read from the bundled library ( Minor version is discarded from _LIBCPP_CLANG_VER
Prompt for agentsWas this helpful? React with 👍 or 👎 to provide feedback. |
||
| } | ||
|
|
||
| fn add_gio_headers(builder: &mut cc::Build) { | ||
| let webrtc_dir = webrtc_sys_build::webrtc_dir(); | ||
| let target_arch = webrtc_sys_build::target_arch(); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
do we need to provide information on the clang requirement in our webrtc repo's readme ?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Clang 21 is only required for compilation on Ubuntu 24 or earlier Linux distributions, so I updated README.md.
https://github.com/livekit/rust-sdks/pull/1284/changes#diff-b335630551682c19a781afebcf4d07bf978fb1f8ac04c6bf87428ed5106870f5R185