fix: ci fixes (#2)

* added new scripts * add empty license ignore file * add licenses * small fixes * small update * small update * small update
tari-project · Nov 21, 2024 · e08907c · e08907c
1 parent 59ad697
commit e08907c
Show file tree

Hide file tree

Showing 8 changed files with 457 additions and 18 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -222,7 +222,7 @@ jobs:
         run: cargo test --no-run --locked --all-features --release
 
       - name: cargo test
-        run: cargo nextest run --all-features --release -E "not package(integration_tests)" --profile ci
+        run: cargo nextest run --all-features --release --profile ci
 
       - name: upload artifact
         uses: actions/upload-artifact@v4 # upload test results as artifact
@@ -231,23 +231,6 @@ jobs:
           name: test-results
           path: ${{ github.workspace }}/target/nextest/ci/junit.xml
 
-      - name: cargo test cucumber
-        run: cargo test --release --package integration_tests --test cucumber -- --tags "not @ignore and not @flaky"
-
-      - name: upload test result artifact
-        uses: actions/upload-artifact@v4 # upload test results as artifact
-        if: success() || failure()
-        with:
-          name: cucumber-test-results
-          path: ${{ github.workspace }}/integration_tests/cucumber-output-junit.xml
-
-      - name: Upload cucumber log artifacts
-        uses: actions/upload-artifact@v4
-        if: success() || failure()
-        with:
-          name: cucumber-log-artifacts
-          path: ${{ github.workspace }}/integration_tests/tests/temp/cucumber_*/*.log
-
   # needed for test results
   event_file:
     runs-on: [ ubuntu-latest ]

diff --git a/.license.ignore b/.license.ignore
diff --git a/Cross.toml b/Cross.toml
@@ -0,0 +1,54 @@
+[build.env]
+passthrough = [
+  "CFLAGS",
+  "TARGET",
+  "RUST_TARGET",
+  "BUILD_TARGET",
+  "CARGO_BUILD_TARGET",
+  "TARGET_CFLAGS",
+  "CC_aarch64_unknown_linux_gnu",
+  "PKG_CONFIG_SYSROOT_DIR",
+  "PKG_CONFIG_ALLOW_CROSS",
+  "RUSTFLAGS",
+  "RUST_BACKTRACE",
+  "RUST_DEBUG",
+  "RUST_LOG",
+  "ARCH",
+  "FEATURES",
+  "ROARING_ARCH",
+  "TARI_NETWORK",
+  "TARI_TARGET_NETWORK",
+  "TARI_NETWORK_DIR",
+]
+
+# Currently needs cross-rs from git
+# ```cargo install cross --git https://github.com/cross-rs/cross```
+[target.aarch64-unknown-linux-gnu]
+image.name = "ubuntu:18.04"
+# targetting is needed for apple silicon
+image.toolchain = ["linux/arm64=aarch64-unknown-linux-gnu", "linux/amd64=x86_64-unknown-linux-gnu"]
+pre-build = "./scripts/cross_compile_ubuntu_18-pre-build.sh"
+
+[target.aarch64-unknown-linux-gnu.env]
+passthrough = [
+  "CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc",
+  "PKG_CONFIG_SYSROOT_DIR=/usr/lib/aarch64-linux-gnu/",
+  "PKG_CONFIG_ALLOW_CROSS=true",
+]
+
+[target.x86_64-unknown-linux-gnu]
+image = "ubuntu:18.04"
+pre-build = "./scripts/cross_compile_ubuntu_18-pre-build.sh"
+
+[target.riscv64gc-unknown-linux-gnu]
+#image.name = "riscv64/ubuntu:20.04"
+image.name = "ubuntu:20.04"
+image.toolchain = ["linux/riscv64=riscv64gc-unknown-linux-gnu", "linux/arm64=aarch64-unknown-linux-gnu", "linux/amd64=x86_64-unknown-linux-gnu"]
+pre-build = "./scripts/cross_compile_ubuntu_18-pre-build.sh"
+
+[target.riscv64gc-unknown-linux-gnu.env]
+passthrough = [
+  "RISCV64GC_UNKNOWN_LINUX_GNU_OPENSSL_NO_VENDOR=true",
+  "PKG_CONFIG_SYSROOT_DIR=/usr/lib/riscv64-linux-gnu/",
+  "PKG_CONFIG_ALLOW_CROSS=true",
+]
diff --git a/scripts/code_coverage.sh b/scripts/code_coverage.sh
@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+# Copyright 2024 The Tari Project
+# SPDX-License-Identifier: BSD-3-Clause
+
+set -e
+
+# Get member source directory and name from command line arguments
+if [ "$1" == "" -o "$2" == "" ]; then
+    echo "Command line argument member_crate_name or member_source_dir missing, Usage: ./codecoverage.sh member_crate_name member_source_dir"
+    exit 1
+fi
+member_crate_name=$1
+member_source_dir=$2
+source_root_dir="tari"
+build_dir="target/debug/"
+report_dir="report/$member_crate_name"
+
+echo "Check if in correct directory":
+path=$(pwd)
+primary_dir=$(basename $path)
+if [ "$primary_dir" == "scripts" ]; then
+    echo "    + Moving to correct directory.."
+    cd ..
+fi
+
+# Check if in Tari primary folder before proceeding
+path=$(pwd)
+primary_dir=$(basename $path)
+if [ "$primary_dir" == "$source_root_dir" ]; then
+    echo "    + Correct directory"
+else
+    echo "    + Error: Incorrect directory -> start code_coverage from script or tari folder!"
+    exit 1
+fi
+
+echo "Check if grcov installed:"
+if [ "$(command -v grcov)" ]
+then
+    echo "    + Already installed"
+else
+    echo "    + Installing.."
+    cargo install grcov
+fi
+
+echo "Check if lcov installed:"
+if [ "$(command -v lcov)" ]
+then
+    echo "    + Already installed"
+else
+    echo "    + Installing.."
+    ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" < /dev/null 2> /dev/null
+    brew install lcov
+fi
+
+echo "Clear Build and Report directories:"
+if [ -d "$build_dir" ]; then
+    rm -rf $build_dir
+    echo "    + Build directory removed"
+else
+    echo "    + Build directory already cleared"
+fi
+if [ -d "$report_dir" ]; then
+    rm -rf $report_dir
+    echo "    + Report directory removed"
+else
+    echo "    + Report directory already cleared"
+fi
+# Make clean directories for Build and Report
+mkdir $build_dir
+mkdir -p $report_dir
+
+echo "Setup project.."
+export CARGO_INCREMENTAL=0
+export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Coverflow-checks=off"
+echo "Build project.."
+cargo +nightly build $CARGO_OPTIONS
+
+echo "Perform project Tests.."
+cargo_filename="Cargo.toml"
+cargo +nightly test $CARGO_OPTIONS --manifest-path="$member_source_dir$cargo_filename"
+
+echo "Acquire all build and test files for coverage check.."
+ccov_filename="ccov.zip"
+ccov_path="$report_dir$ccov_filename"
+
+zip $ccov_path `find $build_dir \( -name "$member_crate_name*.gc*" \) -print`;
+
+echo "Perform grcov code coverage.."
+lcov_filename="lcov.info"
+lcov_path="$report_dir$lcov_filename"
+grcov $ccov_path -s . -t lcov --llvm --branch --ignore-not-existing --ignore "/*" > $lcov_path;
+
+echo "Generate report from code coverage.."
+local_lcov_path="$report_dir$lcov_filename"
+genhtml -o $report_dir --show-details --highlight --title $member_crate_name --legend $local_lcov_path
+
+echo "Launch report in browser.."
+index_str="index.html"
+open "$report_dir/$index_str"
diff --git a/scripts/cross_compile_ubuntu_18-pre-build.sh b/scripts/cross_compile_ubuntu_18-pre-build.sh
@@ -0,0 +1,188 @@
+#!/usr/bin/env bash
+# Copyright 2024 The Tari Project
+# SPDX-License-Identifier: BSD-3-Clause
+
+#
+# Single script for Ubuntu 18.04 package setup, mostly used for cross-compiling
+#
+
+set -e
+
+USAGE="Usage: $0 {target build}
+ where target build is one of the following:
+  x86_64-unknown-linux-gnu or
+  aarch64-unknown-linux-gnu or
+  riscv64gc-unknown-linux-gnu
+"
+
+if [ "$#" == "0" ]; then
+  echo -e "${USAGE}"
+  exit 1
+fi
+
+if [ -z "${CROSS_DEB_ARCH}" ]; then
+  echo -e "Should be run from cross, which sets the env 'CROSS_DEB_ARCH'
+  amd64
+  arm64
+  riscv64
+"
+  exit 1
+fi
+
+DEBIAN_FRONTEND=${DEBIAN_FRONTEND:-noninteractive}
+# Hack of Note!
+TimeZone=${TimeZone:-"Etc/GMT"}
+ln -snf /usr/share/zoneinfo/${TimeZone} /etc/localtime
+echo ${TimeZone} > /etc/timezone
+
+targetBuild="${1}"
+nativeRunTime=$(uname -m)
+echo "Native RunTime is ${nativeRunTime}"
+
+if [ "${nativeRunTime}" == "x86_64" ]; then
+  nativeArch=amd64
+elif [ "${nativeRunTime}" == "aarch64" ]; then
+  nativeArch=arm64
+elif [ "${nativeRunTime}" == "riscv64" ]; then
+  nativeArch=riscv64
+  echo "ToDo!"
+else
+  echo "!!Unsupport platform!!"
+  exit 1
+fi
+
+if [ "${targetBuild}" == "aarch64-unknown-linux-gnu" ]; then
+  targetArch=arm64
+  targetPlatform=aarch64
+elif [ "${targetBuild}" == "x86_64-unknown-linux-gnu" ]; then
+  targetArch=amd64
+  targetPlatform=x86-64
+elif [ "${targetBuild}" == "riscv64gc-unknown-linux-gnu" ]; then
+  targetArch=riscv64
+  targetPlatform=riscv64
+  echo "ToDo!"
+else
+  echo "!!Unsupport platform!!"
+  exit 1
+fi
+
+crossArch=${CROSS_DEB_ARCH}
+apt-get update
+
+# Base install packages
+# scripts/install_ubuntu_dependencies.sh
+apt-get install --no-install-recommends --assume-yes \
+  apt-transport-https \
+  ca-certificates \
+  curl \
+  gpg \
+  bash \
+  less \
+  openssl \
+  libssl-dev \
+  pkg-config \
+  libsqlite3-dev \
+  libsqlite3-0 \
+  libreadline-dev \
+  git \
+  cmake \
+  dh-autoreconf \
+  clang \
+  g++ \
+  libc++-dev \
+  libc++abi-dev \
+  libprotobuf-dev \
+  protobuf-compiler \
+  libncurses5-dev \
+  libncursesw5-dev \
+  libudev-dev \
+  libhidapi-dev \
+  zip
+
+echo "Installing rust ..."
+mkdir -p "$HOME/.cargo/bin/"
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+export PATH="$HOME/.cargo/bin:$PATH"
+. "$HOME/.cargo/env"
+
+# Cross-CPU compile setup
+if [ "${CROSS_DEB_ARCH}" != "${nativeArch}" ]; then
+  echo "Setup Cross CPU Compile ..."
+  sed -i.save -e "s/^deb\ http/deb [arch="${nativeArch}"] http/g" /etc/apt/sources.list
+
+  . /etc/lsb-release
+  ubuntu_tag=${DISTRIB_CODENAME}
+
+  if [[ "${crossArch}" =~ ^(arm|riscv)64$ ]]; then
+    cat << EoF > /etc/apt/sources.list.d/${ubuntu_tag}-${crossArch}.list
+deb [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag} main restricted universe multiverse
+# deb-src [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag} main restricted universe multiverse
+
+deb [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag}-updates main restricted universe multiverse
+# deb-src [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag}-updates main restricted universe multiverse
+
+deb [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag}-backports main restricted universe multiverse
+# deb-src [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag}-backports main restricted universe multiverse
+
+deb [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag}-security main restricted universe multiverse
+# deb-src [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag}-security main restricted universe multiverse
+
+deb [arch=${crossArch}] http://archive.canonical.com/ubuntu ${ubuntu_tag} partner
+# deb-src [arch=${crossArch}] http://archive.canonical.com/ubuntu ${ubuntu_tag} partner
+EoF
+  fi
+
+  if [ "${crossArch}" == "amd64" ]; then
+    cat << EoF > /etc/apt/sources.list.d/${ubuntu_tag}-${crossArch}.list
+deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag} main restricted
+# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag} main restricted
+
+deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-updates main restricted
+# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-updates main restricted
+
+deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag} universe
+# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag} universe
+deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-updates universe
+# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-updates universe
+
+deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag} multiverse
+# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag} multiverse
+deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-updates multiverse
+# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-updates multiverse
+
+deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-backports main restricted universe multiverse
+# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-backports main restricted universe multiverse
+
+# deb http://archive.canonical.com/ubuntu ${ubuntu_tag} partner
+# deb-src http://archive.canonical.com/ubuntu ${ubuntu_tag} partner
+
+deb [arch=amd64] http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security main restricted
+# deb-src http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security main restricted
+deb [arch=amd64] http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security universe
+# deb-src http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security universe
+deb [arch=amd64] http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security multiverse
+# deb-src http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security multiverse
+EoF
+  fi
+
+  dpkg --add-architecture ${CROSS_DEB_ARCH}
+  apt-get update
+
+  # scripts/install_ubuntu_dependencies-cross_compile.sh x86-64
+  apt-get --assume-yes install \
+    pkg-config-${targetPlatform}-linux-gnu \
+    gcc-${targetPlatform}-linux-gnu \
+    g++-${targetPlatform}-linux-gnu
+
+  # packages needed for Ledger and hidapi
+  apt-get --assume-yes install \
+    libudev-dev:${CROSS_DEB_ARCH} \
+    libhidapi-dev:${CROSS_DEB_ARCH}
+
+fi
+
+rustup target add ${targetBuild}
+rustup toolchain install stable-${targetBuild} --force-non-host
+
+rustup target list --installed
+rustup toolchain list