From 559981d0b8118895924f6b4470b5d7c55ee9b134 Mon Sep 17 00:00:00 2001 From: Luis Cruz Date: Wed, 5 Jun 2024 14:55:11 +0100 Subject: [PATCH 1/5] Remove stripped android binaries split into different folders --- .github/workflows/tests.yml | 4 +- rust_build_utils/android_build_utils.py | 55 ++++++++++++++----------- rust_build_utils/rust_utils.py | 15 ++++--- rust_build_utils/rust_utils_config.py | 3 +- 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6901cbf..fae387e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -137,7 +137,7 @@ jobs: with: name: rust-sample-android-armv7 path: rust_sample/dist - - run: python3 rust_sample/ci/build_sample.py aar rust_sample com.nordsec.rust_sample rust_sample v1.2.3 $(pwd)/rust_sample/dist/android/java $(pwd)/rust_sample/dist/android/release/stripped + - run: python3 rust_sample/ci/build_sample.py aar rust_sample com.nordsec.rust_sample rust_sample v1.2.3 $(pwd)/rust_sample/dist/android/java $(pwd)/rust_sample/dist/android/release - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 with: name: rust-sample-aar @@ -173,7 +173,7 @@ jobs: with: name: rust-sample-android-armv7 path: rust_sample/dist - - run: python3 rust_sample/ci/build_sample.py aar rust_sample com.nordsec.rust_sample rust_sample v1.2.3 $(pwd)/rust_sample/dist/android/java $(pwd)/rust_sample/dist/android/release/stripped --settings_gradle_path $(pwd)/rust_sample/templates/__settings.gradle --build_gradle_path $(pwd)/rust_sample/templates/__build.gradle --init_gradle_path $(pwd)/rust_sample/templates/__init.gradle + - run: python3 rust_sample/ci/build_sample.py aar rust_sample com.nordsec.rust_sample rust_sample v1.2.3 $(pwd)/rust_sample/dist/android/java $(pwd)/rust_sample/dist/android/release --settings_gradle_path $(pwd)/rust_sample/templates/__settings.gradle --build_gradle_path $(pwd)/rust_sample/templates/__build.gradle --init_gradle_path $(pwd)/rust_sample/templates/__init.gradle - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 with: name: rust-sample-aar-custom diff --git a/rust_build_utils/android_build_utils.py b/rust_build_utils/android_build_utils.py index d77215c..2f0211c 100644 --- a/rust_build_utils/android_build_utils.py +++ b/rust_build_utils/android_build_utils.py @@ -2,47 +2,52 @@ import shutil import subprocess import rust_build_utils.rust_utils as rutils -from rust_build_utils.rust_utils_config import GLOBAL_CONFIG, NDK_IMAGE_PATH +from rust_build_utils.rust_utils_config import ( + GLOBAL_CONFIG, + NDK_IMAGE_PATH, + NDK_VERSION, +) from string import Template from typing import Optional -NDK_VERSION = "r26" + TOOLCHAIN = ( f"{NDK_IMAGE_PATH}/android-ndk-{NDK_VERSION}/toolchains/llvm/prebuilt/linux-x86_64" ) -def strip_android(project: rutils.Project, config: rutils.CargoConfig, packages=None): - strip_dir = project.get_distribution_path( - config.target_os, config.arch, f"../stripped/", config.debug - ) - unstrip_dir = project.get_distribution_path( - config.target_os, config.arch, f"../unstripped/", config.debug - ) - if not os.path.exists(strip_dir): - os.makedirs(strip_dir) - if not os.path.exists(unstrip_dir): - os.makedirs(unstrip_dir) +def strip(project: rutils.Project, config: rutils.CargoConfig, packages=None): + if config.target_os != "android" or config.debug or packages == None: + return - arch_dir = project.get_distribution_path( + strip_bin = f"{TOOLCHAIN}/bin/llvm-strip" + dist_dir = project.get_distribution_path( config.target_os, config.arch, "", config.debug ) renamed_arch = GLOBAL_CONFIG[config.target_os]["archs"][config.arch]["dist"] - shutil.copytree( - arch_dir, - f"{unstrip_dir}/{renamed_arch}", - ) - shutil.copytree( - arch_dir, - f"{strip_dir}/{renamed_arch}", - ) - shutil.rmtree(arch_dir) - strip = f"{TOOLCHAIN}/bin/llvm-strip" + def _create_debug_symbols(bin_path: str): + create_debug_symbols_cmd = [ + f"{strip_bin}", + "--only-keep-debug", + f"{bin_path}", + "-o", + f"{bin_path}.debug", + ] + rutils.run_command(create_debug_symbols_cmd) + + set_read_only_cmd = ["chmod", "0444", f"{bin_path}.debug"] + rutils.run_command(set_read_only_cmd) + + def _strip_debug_symbols(bin_path: str): + strip_cmd = [f"{strip_bin}", "--strip-all", f"{bin_path}"] + rutils.run_command(strip_cmd) for _, bins in packages.items(): for _, bin in bins.items(): - rutils.run_command([strip, f"{strip_dir}/{renamed_arch}/{bin}"]) + bin_path = f"{dist_dir}/{bin}" + _create_debug_symbols(bin_path) + _strip_debug_symbols(bin_path) def _process_template( diff --git a/rust_build_utils/rust_utils.py b/rust_build_utils/rust_utils.py index 5f754d5..db7fc39 100644 --- a/rust_build_utils/rust_utils.py +++ b/rust_build_utils/rust_utils.py @@ -377,10 +377,17 @@ def _cargo( if not packages: raise ValueError("No packages specified") + arch = ( + config.arch + if config.target_os != "android" + else GLOBAL_CONFIG[config.target_os]["archs"][config.arch]["dist"] + ) + pre_build(config) distribution_dir = project.get_distribution_path( - config.target_os, config.arch, "", config.debug + config.target_os, arch, "", config.debug ) + if os.path.isdir(distribution_dir): shutil.rmtree(distribution_dir) os.makedirs(distribution_dir) @@ -406,7 +413,7 @@ def _cargo( msvc_context = None if config.rust_target.endswith("-msvc") and not is_msvc_active(): # For msvc based toolchains msvc development environment needs activation - msvc_context = activate_msvc(config.arch) + msvc_context = activate_msvc(arch) _build_packages(config, list(packages.keys()), extra_args, subcommand) @@ -415,9 +422,7 @@ def _cargo( # copies executable permissions shutil.copy2( project.get_cargo_path(config.rust_target, bin, config.debug), - project.get_distribution_path( - config.target_os, config.arch, bin, config.debug - ), + distribution_dir, ) post_build(project, config, packages) diff --git a/rust_build_utils/rust_utils_config.py b/rust_build_utils/rust_utils_config.py index f2b0b8a..bc8fa5e 100644 --- a/rust_build_utils/rust_utils_config.py +++ b/rust_build_utils/rust_utils_config.py @@ -1,6 +1,7 @@ from typing import Dict, Any NDK_IMAGE_PATH = "/source/.build" +NDK_VERSION = "r26" # This is the global configuration file that will be used for most Rust projects, only apply changes which are needed for all projects. # Every single OS has this general structure: @@ -42,7 +43,7 @@ }, }, "env": {"PATH": (f":{NDK_IMAGE_PATH}", "append")}, - "post_build": ["rust_build_utils.android_build_utils.strip_android"], + "post_build": ["rust_build_utils.android_build_utils.strip"], }, "linux": { "archs": { From 7aa5c7a6efd7bd4b66715f7833583c45c0b7e17e Mon Sep 17 00:00:00 2001 From: Luis Cruz Date: Wed, 5 Jun 2024 14:56:21 +0100 Subject: [PATCH 2/5] Strip linux binaries on post_build task --- rust_build_utils/linux_build_utils.py | 40 +++++++++++++++++++++++++++ rust_build_utils/rust_utils_config.py | 6 ++++ 2 files changed, 46 insertions(+) create mode 100644 rust_build_utils/linux_build_utils.py diff --git a/rust_build_utils/linux_build_utils.py b/rust_build_utils/linux_build_utils.py new file mode 100644 index 0000000..b924a01 --- /dev/null +++ b/rust_build_utils/linux_build_utils.py @@ -0,0 +1,40 @@ +from os import path +import rust_build_utils.rust_utils as rutils +from rust_build_utils.rust_utils_config import GLOBAL_CONFIG + + +def strip(project: rutils.Project, config: rutils.CargoConfig, packages=None): + if config.target_os != "linux" or config.debug or packages == None: + return + + strip_bin = GLOBAL_CONFIG["linux"]["archs"][config.arch]["strip_path"] + if not path.isfile(strip_bin): + # fallback to default strip + strip_bin = "strip" + + dist_dir = project.get_distribution_path( + config.target_os, config.arch, "", config.debug + ) + + def _create_debug_symbols(bin_path: str): + create_debug_symbols_cmd = [ + f"{strip_bin}", + "--only-keep-debug", + f"{bin_path}", + "-o", + f"{bin_path}.debug", + ] + rutils.run_command(create_debug_symbols_cmd) + + set_read_only_cmd = ["chmod", "0444", f"{bin_path}.debug"] + rutils.run_command(set_read_only_cmd) + + def _strip_debug_symbols(bin_path: str): + strip_cmd = [f"{strip_bin}", "--strip-all", f"{bin_path}"] + rutils.run_command(strip_cmd) + + for _, bins in packages.items(): + for _, bin in bins.items(): + bin_path = f"{dist_dir}/{bin}" + _create_debug_symbols(bin_path) + _strip_debug_symbols(bin_path) diff --git a/rust_build_utils/rust_utils_config.py b/rust_build_utils/rust_utils_config.py index bc8fa5e..59ca047 100644 --- a/rust_build_utils/rust_utils_config.py +++ b/rust_build_utils/rust_utils_config.py @@ -48,21 +48,27 @@ "linux": { "archs": { "x86_64": { + "strip_path": "/usr/bin/strip", "rust_target": "x86_64-unknown-linux-gnu", }, "aarch64": { + "strip_path": "/usr/aarch64-linux-gnu/bin/strip", "rust_target": "aarch64-unknown-linux-gnu", }, "i686": { + "strip_path": "/usr/i686-linux-gnu/bin/strip", "rust_target": "i686-unknown-linux-gnu", }, "armv7hf": { + "strip_path": "/usr/arm-linux-gnueabihf/bin/strip", "rust_target": "armv7-unknown-linux-gnueabihf", }, "armv5": { + "strip_path": "/usr/arm-linux-gnueabi/bin/strip", "rust_target": "arm-unknown-linux-gnueabi", }, }, + "post_build": ["rust_build_utils.linux_build_utils.strip"], }, "windows": { "archs": { From 7bd801ecfe26e5dd52fbc98d2ff7e4123971b336 Mon Sep 17 00:00:00 2001 From: Luis Cruz Date: Wed, 5 Jun 2024 21:58:04 +0100 Subject: [PATCH 3/5] Add debug symbols compression --- rust_build_utils/android_build_utils.py | 11 +++++------ rust_build_utils/linux_build_utils.py | 4 ++-- rust_build_utils/rust_utils_config.py | 10 +++++----- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/rust_build_utils/android_build_utils.py b/rust_build_utils/android_build_utils.py index 2f0211c..1ee15ad 100644 --- a/rust_build_utils/android_build_utils.py +++ b/rust_build_utils/android_build_utils.py @@ -20,18 +20,17 @@ def strip(project: rutils.Project, config: rutils.CargoConfig, packages=None): if config.target_os != "android" or config.debug or packages == None: return - strip_bin = f"{TOOLCHAIN}/bin/llvm-strip" - dist_dir = project.get_distribution_path( - config.target_os, config.arch, "", config.debug - ) - renamed_arch = GLOBAL_CONFIG[config.target_os]["archs"][config.arch]["dist"] + strip_bin = f"{TOOLCHAIN}/bin/llvm-objcopy" + + arch = GLOBAL_CONFIG[config.target_os]["archs"][config.arch]["dist"] + dist_dir = project.get_distribution_path(config.target_os, arch, "", config.debug) def _create_debug_symbols(bin_path: str): create_debug_symbols_cmd = [ f"{strip_bin}", "--only-keep-debug", + "--compress-debug-sections=zlib", f"{bin_path}", - "-o", f"{bin_path}.debug", ] rutils.run_command(create_debug_symbols_cmd) diff --git a/rust_build_utils/linux_build_utils.py b/rust_build_utils/linux_build_utils.py index b924a01..99b13af 100644 --- a/rust_build_utils/linux_build_utils.py +++ b/rust_build_utils/linux_build_utils.py @@ -10,7 +10,7 @@ def strip(project: rutils.Project, config: rutils.CargoConfig, packages=None): strip_bin = GLOBAL_CONFIG["linux"]["archs"][config.arch]["strip_path"] if not path.isfile(strip_bin): # fallback to default strip - strip_bin = "strip" + strip_bin = "objcopy" dist_dir = project.get_distribution_path( config.target_os, config.arch, "", config.debug @@ -20,8 +20,8 @@ def _create_debug_symbols(bin_path: str): create_debug_symbols_cmd = [ f"{strip_bin}", "--only-keep-debug", + "--compress-debug-sections=zlib", f"{bin_path}", - "-o", f"{bin_path}.debug", ] rutils.run_command(create_debug_symbols_cmd) diff --git a/rust_build_utils/rust_utils_config.py b/rust_build_utils/rust_utils_config.py index 59ca047..1230121 100644 --- a/rust_build_utils/rust_utils_config.py +++ b/rust_build_utils/rust_utils_config.py @@ -48,23 +48,23 @@ "linux": { "archs": { "x86_64": { - "strip_path": "/usr/bin/strip", + "strip_path": "/usr/bin/objcopy", "rust_target": "x86_64-unknown-linux-gnu", }, "aarch64": { - "strip_path": "/usr/aarch64-linux-gnu/bin/strip", + "strip_path": "/usr/aarch64-linux-gnu/bin/objcopy", "rust_target": "aarch64-unknown-linux-gnu", }, "i686": { - "strip_path": "/usr/i686-linux-gnu/bin/strip", + "strip_path": "/usr/i686-linux-gnu/bin/objcopy", "rust_target": "i686-unknown-linux-gnu", }, "armv7hf": { - "strip_path": "/usr/arm-linux-gnueabihf/bin/strip", + "strip_path": "/usr/arm-linux-gnueabihf/bin/objcopy", "rust_target": "armv7-unknown-linux-gnueabihf", }, "armv5": { - "strip_path": "/usr/arm-linux-gnueabi/bin/strip", + "strip_path": "/usr/arm-linux-gnueabi/bin/objcopy", "rust_target": "arm-unknown-linux-gnueabi", }, }, From ed537a4a2b08e4ca9aba1c4d6538170d4d36a45a Mon Sep 17 00:00:00 2001 From: Luis Cruz Date: Wed, 5 Jun 2024 22:22:34 +0100 Subject: [PATCH 4/5] Split and strip symbols from darwin binaries --- rust_build_utils/rust_utils_config.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/rust_build_utils/rust_utils_config.py b/rust_build_utils/rust_utils_config.py index 1230121..bccb40d 100644 --- a/rust_build_utils/rust_utils_config.py +++ b/rust_build_utils/rust_utils_config.py @@ -97,6 +97,10 @@ }, }, }, + "env": { + "CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO": (["packed"], "set"), + "CARGO_PROFILE_RELEASE_STRIP": (["true"], "set"), + }, "post_build": ["rust_build_utils.darwin_build_utils.assert_version"], }, "ios": { @@ -109,6 +113,10 @@ }, }, }, + "env": { + "CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO": (["packed"], "set"), + "CARGO_PROFILE_RELEASE_STRIP": (["true"], "set"), + }, "pre_build": ["rust_build_utils.darwin_build_utils.set_sdk"], "post_build": ["rust_build_utils.darwin_build_utils.assert_version"], }, @@ -122,6 +130,10 @@ }, }, }, + "env": { + "CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO": (["packed"], "set"), + "CARGO_PROFILE_RELEASE_STRIP": (["true"], "set"), + }, "pre_build": ["rust_build_utils.darwin_build_utils.set_sdk"], "post_build": ["rust_build_utils.darwin_build_utils.assert_version"], }, @@ -135,6 +147,10 @@ }, }, }, + "env": { + "CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO": (["packed"], "set"), + "CARGO_PROFILE_RELEASE_STRIP": (["true"], "set"), + }, "pre_build": ["rust_build_utils.darwin_build_utils.set_sdk"], "post_build": ["rust_build_utils.darwin_build_utils.assert_version"], }, From 997b89be0ea063464ab969e7a6c57c28bf00f890 Mon Sep 17 00:00:00 2001 From: Luis Cruz Date: Thu, 27 Jun 2024 16:36:56 -0300 Subject: [PATCH 5/5] Keep dSYM bundles when creating universal macOS binaries --- rust_build_utils/darwin_build_utils.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/rust_build_utils/darwin_build_utils.py b/rust_build_utils/darwin_build_utils.py index a5ba3bd..caa4d4b 100644 --- a/rust_build_utils/darwin_build_utils.py +++ b/rust_build_utils/darwin_build_utils.py @@ -95,13 +95,15 @@ def lipo( packages: rutils.PackageList, ): archs = GLOBAL_CONFIG[target_os]["archs"] + universal_binary_dist_path = get_universal_library_distribution_directory( + project, target_os, debug + ) for _, bins in packages.items(): for _, binary in bins.items(): create_fat_binary( project, - get_universal_library_distribution_directory(project, target_os, debug) - / binary, + universal_binary_dist_path / binary, target_os, archs.keys(), binary, @@ -109,7 +111,17 @@ def lipo( ) for arch in archs: - shutil.rmtree(project.get_distribution_path(target_os, arch, "", debug)) + dist_path = project.get_distribution_path(target_os, arch, "", debug) + + for _, bins in packages.items(): + for _, binary in bins.items(): + dsym_dir = f"{dist_path}/{binary}.dSYM" + if os.path.isdir(dsym_dir): + dst_dir = f"{universal_binary_dist_path}/{binary}.dSYM/{arch}" + os.makedirs(dst_dir, exist_ok=True) + shutil.copytree(dsym_dir, dst_dir, dirs_exist_ok=True) + + shutil.rmtree(dist_path) def create_fat_binary(