Merge pull request #23 from NordSecurity/LLT-5072_investigate_binary_…

…strip_strategies

LLT-5072: Investigate binary strip strategies

.github/workflows/tests.yml

@@ -137,7 +137,7 @@ jobs:
         with:
           name: rust-sample-android-armv7
           path: rust_sample/dist
-      - run: python3 rust_sample/ci/build_sample.py aar rust_sample com.nordsec.rust_sample rust_sample v1.2.3 $(pwd)/rust_sample/dist/android/java $(pwd)/rust_sample/dist/android/release/stripped
+      - run: python3 rust_sample/ci/build_sample.py aar rust_sample com.nordsec.rust_sample rust_sample v1.2.3 $(pwd)/rust_sample/dist/android/java $(pwd)/rust_sample/dist/android/release
       - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
           name: rust-sample-aar
@@ -173,7 +173,7 @@ jobs:
         with:
           name: rust-sample-android-armv7
           path: rust_sample/dist
-      - run: python3 rust_sample/ci/build_sample.py aar rust_sample com.nordsec.rust_sample rust_sample v1.2.3 $(pwd)/rust_sample/dist/android/java $(pwd)/rust_sample/dist/android/release/stripped --settings_gradle_path $(pwd)/rust_sample/templates/__settings.gradle --build_gradle_path $(pwd)/rust_sample/templates/__build.gradle --init_gradle_path $(pwd)/rust_sample/templates/__init.gradle
+      - run: python3 rust_sample/ci/build_sample.py aar rust_sample com.nordsec.rust_sample rust_sample v1.2.3 $(pwd)/rust_sample/dist/android/java $(pwd)/rust_sample/dist/android/release --settings_gradle_path $(pwd)/rust_sample/templates/__settings.gradle --build_gradle_path $(pwd)/rust_sample/templates/__build.gradle --init_gradle_path $(pwd)/rust_sample/templates/__init.gradle
       - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
           name: rust-sample-aar-custom

rust_build_utils/android_build_utils.py

@@ -2,47 +2,51 @@
 import shutil
 import subprocess
 import rust_build_utils.rust_utils as rutils
-from rust_build_utils.rust_utils_config import GLOBAL_CONFIG, NDK_IMAGE_PATH
+from rust_build_utils.rust_utils_config import (
+    GLOBAL_CONFIG,
+    NDK_IMAGE_PATH,
+    NDK_VERSION,
+)
 from string import Template
 from typing import Optional
 
-NDK_VERSION = "r26"
+
 TOOLCHAIN = (
     f"{NDK_IMAGE_PATH}/android-ndk-{NDK_VERSION}/toolchains/llvm/prebuilt/linux-x86_64"
 )
 
 
-def strip_android(project: rutils.Project, config: rutils.CargoConfig, packages=None):
-    strip_dir = project.get_distribution_path(
-        config.target_os, config.arch, f"../stripped/", config.debug
-    )
-    unstrip_dir = project.get_distribution_path(
-        config.target_os, config.arch, f"../unstripped/", config.debug
-    )
-    if not os.path.exists(strip_dir):
-        os.makedirs(strip_dir)
-    if not os.path.exists(unstrip_dir):
-        os.makedirs(unstrip_dir)
+def strip(project: rutils.Project, config: rutils.CargoConfig, packages=None):
+    if config.target_os != "android" or config.debug or packages == None:
+        return
 
-    arch_dir = project.get_distribution_path(
-        config.target_os, config.arch, "", config.debug
-    )
-    renamed_arch = GLOBAL_CONFIG[config.target_os]["archs"][config.arch]["dist"]
-    shutil.copytree(
-        arch_dir,
-        f"{unstrip_dir}/{renamed_arch}",
-    )
-    shutil.copytree(
-        arch_dir,
-        f"{strip_dir}/{renamed_arch}",
-    )
+    strip_bin = f"{TOOLCHAIN}/bin/llvm-objcopy"
+
+    arch = GLOBAL_CONFIG[config.target_os]["archs"][config.arch]["dist"]
+    dist_dir = project.get_distribution_path(config.target_os, arch, "", config.debug)
+
+    def _create_debug_symbols(bin_path: str):
+        create_debug_symbols_cmd = [
+            f"{strip_bin}",
+            "--only-keep-debug",
+            "--compress-debug-sections=zlib",
+            f"{bin_path}",
+            f"{bin_path}.debug",
+        ]
+        rutils.run_command(create_debug_symbols_cmd)
+
+        set_read_only_cmd = ["chmod", "0444", f"{bin_path}.debug"]
+        rutils.run_command(set_read_only_cmd)
 
-    shutil.rmtree(arch_dir)
-    strip = f"{TOOLCHAIN}/bin/llvm-strip"
+    def _strip_debug_symbols(bin_path: str):
+        strip_cmd = [f"{strip_bin}", "--strip-all", f"{bin_path}"]
+        rutils.run_command(strip_cmd)
 
     for _, bins in packages.items():
         for _, bin in bins.items():
-            rutils.run_command([strip, f"{strip_dir}/{renamed_arch}/{bin}"])
+            bin_path = f"{dist_dir}/{bin}"
+            _create_debug_symbols(bin_path)
+            _strip_debug_symbols(bin_path)
 
 
 def _process_template(

rust_build_utils/darwin_build_utils.py

@@ -95,21 +95,33 @@ def lipo(
     packages: rutils.PackageList,
 ):
     archs = GLOBAL_CONFIG[target_os]["archs"]
+    universal_binary_dist_path = get_universal_library_distribution_directory(
+        project, target_os, debug
+    )
 
     for _, bins in packages.items():
         for _, binary in bins.items():
             create_fat_binary(
                 project,
-                get_universal_library_distribution_directory(project, target_os, debug)
-                / binary,
+                universal_binary_dist_path / binary,
                 target_os,
                 archs.keys(),
                 binary,
                 debug,
             )
 
     for arch in archs:
-        shutil.rmtree(project.get_distribution_path(target_os, arch, "", debug))
+        dist_path = project.get_distribution_path(target_os, arch, "", debug)
+
+        for _, bins in packages.items():
+            for _, binary in bins.items():
+                dsym_dir = f"{dist_path}/{binary}.dSYM"
+                if os.path.isdir(dsym_dir):
+                    dst_dir = f"{universal_binary_dist_path}/{binary}.dSYM/{arch}"
+                    os.makedirs(dst_dir, exist_ok=True)
+                    shutil.copytree(dsym_dir, dst_dir, dirs_exist_ok=True)
+
+        shutil.rmtree(dist_path)
 
 
 def create_fat_binary(

rust_build_utils/linux_build_utils.py

@@ -0,0 +1,40 @@
+from os import path
+import rust_build_utils.rust_utils as rutils
+from rust_build_utils.rust_utils_config import GLOBAL_CONFIG
+
+
+def strip(project: rutils.Project, config: rutils.CargoConfig, packages=None):
+    if config.target_os != "linux" or config.debug or packages == None:
+        return
+
+    strip_bin = GLOBAL_CONFIG["linux"]["archs"][config.arch]["strip_path"]
+    if not path.isfile(strip_bin):
+        # fallback to default strip
+        strip_bin = "objcopy"
+
+    dist_dir = project.get_distribution_path(
+        config.target_os, config.arch, "", config.debug
+    )
+
+    def _create_debug_symbols(bin_path: str):
+        create_debug_symbols_cmd = [
+            f"{strip_bin}",
+            "--only-keep-debug",
+            "--compress-debug-sections=zlib",
+            f"{bin_path}",
+            f"{bin_path}.debug",
+        ]
+        rutils.run_command(create_debug_symbols_cmd)
+
+        set_read_only_cmd = ["chmod", "0444", f"{bin_path}.debug"]
+        rutils.run_command(set_read_only_cmd)
+
+    def _strip_debug_symbols(bin_path: str):
+        strip_cmd = [f"{strip_bin}", "--strip-all", f"{bin_path}"]
+        rutils.run_command(strip_cmd)
+
+    for _, bins in packages.items():
+        for _, bin in bins.items():
+            bin_path = f"{dist_dir}/{bin}"
+            _create_debug_symbols(bin_path)
+            _strip_debug_symbols(bin_path)

rust_build_utils/rust_utils.py

@@ -377,10 +377,17 @@ def _cargo(
     if not packages:
         raise ValueError("No packages specified")
 
+    arch = (
+        config.arch
+        if config.target_os != "android"
+        else GLOBAL_CONFIG[config.target_os]["archs"][config.arch]["dist"]
+    )
+
     pre_build(config)
     distribution_dir = project.get_distribution_path(
-        config.target_os, config.arch, "", config.debug
+        config.target_os, arch, "", config.debug
     )
+
     if os.path.isdir(distribution_dir):
         shutil.rmtree(distribution_dir)
     os.makedirs(distribution_dir)
@@ -406,7 +413,7 @@ def _cargo(
     msvc_context = None
     if config.rust_target.endswith("-msvc") and not is_msvc_active():
         # For msvc based toolchains msvc development environment needs activation
-        msvc_context = activate_msvc(config.arch)
+        msvc_context = activate_msvc(arch)
 
     _build_packages(config, list(packages.keys()), extra_args, subcommand)
 
@@ -415,9 +422,7 @@ def _cargo(
             # copies executable permissions
             shutil.copy2(
                 project.get_cargo_path(config.rust_target, bin, config.debug),
-                project.get_distribution_path(
-                    config.target_os, config.arch, bin, config.debug
-                ),
+                distribution_dir,
             )
 
     post_build(project, config, packages)

rust_build_utils/rust_utils_config.py

@@ -1,6 +1,7 @@
 from typing import Dict, Any
 
 NDK_IMAGE_PATH = "/source/.build"
+NDK_VERSION = "r26"
 # This is the global configuration file that will be used for most Rust projects, only apply changes which are needed for all projects.
 
 # Every single OS has this general structure:
@@ -42,26 +43,32 @@
             },
         },
         "env": {"PATH": (f":{NDK_IMAGE_PATH}", "append")},
-        "post_build": ["rust_build_utils.android_build_utils.strip_android"],
+        "post_build": ["rust_build_utils.android_build_utils.strip"],
     },
     "linux": {
         "archs": {
             "x86_64": {
+                "strip_path": "/usr/bin/objcopy",
                 "rust_target": "x86_64-unknown-linux-gnu",
             },
             "aarch64": {
+                "strip_path": "/usr/aarch64-linux-gnu/bin/objcopy",
                 "rust_target": "aarch64-unknown-linux-gnu",
             },
             "i686": {
+                "strip_path": "/usr/i686-linux-gnu/bin/objcopy",
                 "rust_target": "i686-unknown-linux-gnu",
             },
             "armv7hf": {
+                "strip_path": "/usr/arm-linux-gnueabihf/bin/objcopy",
                 "rust_target": "armv7-unknown-linux-gnueabihf",
             },
             "armv5": {
+                "strip_path": "/usr/arm-linux-gnueabi/bin/objcopy",
                 "rust_target": "arm-unknown-linux-gnueabi",
             },
         },
+        "post_build": ["rust_build_utils.linux_build_utils.strip"],
     },
     "windows": {
         "archs": {
@@ -90,6 +97,10 @@
                 },
             },
         },
+        "env": {
+            "CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO": (["packed"], "set"),
+            "CARGO_PROFILE_RELEASE_STRIP": (["true"], "set"),
+        },
         "post_build": ["rust_build_utils.darwin_build_utils.assert_version"],
     },
     "ios": {
@@ -102,6 +113,10 @@
                 },
             },
         },
+        "env": {
+            "CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO": (["packed"], "set"),
+            "CARGO_PROFILE_RELEASE_STRIP": (["true"], "set"),
+        },
         "pre_build": ["rust_build_utils.darwin_build_utils.set_sdk"],
         "post_build": ["rust_build_utils.darwin_build_utils.assert_version"],
     },
@@ -115,6 +130,10 @@
                 },
             },
         },
+        "env": {
+            "CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO": (["packed"], "set"),
+            "CARGO_PROFILE_RELEASE_STRIP": (["true"], "set"),
+        },
         "pre_build": ["rust_build_utils.darwin_build_utils.set_sdk"],
         "post_build": ["rust_build_utils.darwin_build_utils.assert_version"],
     },
@@ -128,6 +147,10 @@
                 },
             },
         },
+        "env": {
+            "CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO": (["packed"], "set"),
+            "CARGO_PROFILE_RELEASE_STRIP": (["true"], "set"),
+        },
         "pre_build": ["rust_build_utils.darwin_build_utils.set_sdk"],
         "post_build": ["rust_build_utils.darwin_build_utils.assert_version"],
     },