From aef6efc8118ee939db7d120c729e924cfe70fb42 Mon Sep 17 00:00:00 2001 From: Derek G Foster Date: Thu, 26 Oct 2023 12:02:08 -0700 Subject: [PATCH] Replace CI pipeline (#320) - Replace the P4 Control Plane CI pipeline. The new pipeline uses precompiled copies of the DPDK SDE and Stratum dependencies instead of building them from scratch every time, and it moves the job to run the Stratum unit tests to the stratum repository. The net effect is to reduce the run time of the networking-recipe commit check from ~1.2-1.5 hours to ~10-15 minutes. Signed-off-by: Derek G Foster --- .github/dpdk-tests.txt | 13 -- .github/es2k-tests.txt | 7 - .github/tofino-tests.txt | 7 - .github/workflows/build_and_test.yml | 192 --------------------------- .github/workflows/pipeline.yml | 150 +++++++++++++++++++++ 5 files changed, 150 insertions(+), 219 deletions(-) delete mode 100644 .github/dpdk-tests.txt delete mode 100644 .github/es2k-tests.txt delete mode 100644 .github/tofino-tests.txt delete mode 100644 .github/workflows/build_and_test.yml create mode 100644 .github/workflows/pipeline.yml diff --git a/.github/dpdk-tests.txt b/.github/dpdk-tests.txt deleted file mode 100644 index 47b30288..00000000 --- a/.github/dpdk-tests.txt +++ /dev/null @@ -1,13 +0,0 @@ -//stratum/glue/... -//stratum/hal/lib/p4/... -//stratum/lib/... -//stratum/public/... -//stratum/hal/lib/tdi:tdi_action_profile_manager_test -//stratum/hal/lib/tdi:tdi_counter_manager_test -//stratum/hal/lib/tdi:tdi_packetio_manager_test -//stratum/hal/lib/tdi:tdi_pipeline_utils_test -//stratum/hal/lib/tdi:tdi_pre_manager_test -//stratum/hal/lib/tdi:tdi_table_manager_test -//stratum/hal/lib/tdi:utils_test -//stratum/hal/lib/tdi/dpdk:all -//stratum/hal/lib/yang/... diff --git a/.github/es2k-tests.txt b/.github/es2k-tests.txt deleted file mode 100644 index f40849c2..00000000 --- a/.github/es2k-tests.txt +++ /dev/null @@ -1,7 +0,0 @@ -//stratum/glue/... -//stratum/hal/lib/p4/... -//stratum/lib/... -//stratum/public/... -//stratum/hal/lib/tdi:all -//stratum/hal/lib/tdi/es2k:all -//stratum/hal/lib/yang/... diff --git a/.github/tofino-tests.txt b/.github/tofino-tests.txt deleted file mode 100644 index d8e8a7af..00000000 --- a/.github/tofino-tests.txt +++ /dev/null @@ -1,7 +0,0 @@ -//stratum/glue/... -//stratum/hal/lib/p4/... -//stratum/lib/... -//stratum/public/... -//stratum/hal/lib/tdi:all -//stratum/hal/lib/tdi/tofino:all -//stratum/hal/lib/yang/... diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml deleted file mode 100644 index 872e00a0..00000000 --- a/.github/workflows/build_and_test.yml +++ /dev/null @@ -1,192 +0,0 @@ -name: "Networking Recipe CI Pipeline" - -on: - push: - branches: - - main - - ipdk_v* - pull_request: - branches: - - main - - ipdk_v* - -concurrency: - # if workflow for PR or push is already running stop it, and start new one - group: build_and_test-${{ github.ref }} - cancel-in-progress: true - -permissions: read-all - -jobs: - - build_and_stratum_ut_dpdk_target: - timeout-minutes: 100 - runs-on: ubuntu-20.04 - steps: - - name: 'Check Out p4-dpdk repo' - uses: actions/checkout@v3 - with: - repository: p4lang/p4-dpdk-target - submodules: 'recursive' - path: p4sde - - - name: 'Install dependencies' - working-directory: ./p4sde/tools/setup - run: | - sudo apt update -y - sudo apt install python python3-setuptools python3-pip python3-wheel python3-cffi sudo libssl-dev git zip libatomic1 libnl-route-3-dev curl tree -y - pip3 install distro - python3 install_dep.py - cmake --version - sudo apt-get purge cmake - sudo rm -rf /usr/local/bin/cmake || true - sudo rm -rf ~/.local/bin/cmake || true - # removing cmake 3.25 and reinstalling 3.20, as build is failing with cmake 3.25 - wget https://cmake.org/files/v3.20/cmake-3.20.2-linux-x86_64.sh -O ${GITHUB_WORKSPACE}/cmake-3.20.2-linux-x86_64.sh - chmod a+x ${GITHUB_WORKSPACE}/cmake-3.20.2-linux-x86_64.sh - sudo mkdir -p /opt/rh/cmake - cd ${GITHUB_WORKSPACE} && sudo sh cmake-3.20.2-linux-x86_64.sh --prefix=/opt/rh/cmake --skip-license - sudo ln -s /opt/rh/cmake/bin/cmake /usr/local/bin/cmake - cmake --version - - - name: 'Check Out Networking Recipe' - uses: actions/checkout@v3 - with: - submodules: 'recursive' - path: ipdk.recipe - - - name: 'Compile P4 dpdk target' - run: | - export SDE=${GITHUB_WORKSPACE} - mkdir install deps - export SDE_INSTALL=${GITHUB_WORKSPACE}/install - cd ${GITHUB_WORKSPACE}/p4sde - ./autogen.sh - ./configure --prefix=$SDE_INSTALL - make -j6 - make install - - - name: 'Build,Configure and Execute infrap4d dependencies' - run: | - cmake --version - cd ${GITHUB_WORKSPACE}/ipdk.recipe - export IPDK_RECIPE=`pwd` - cd $IPDK_RECIPE/setup - cmake -B build -DCMAKE_INSTALL_PREFIX=${GITHUB_WORKSPACE}/deps - cmake --build build -j6 - - - name: 'Install bazel and run Stratum unit tests' - run: | - cd /tmp - curl -o bazel-4.2.3-installer-linux-x86_64.sh https://github.com/bazelbuild/bazel/releases/download/4.2.3/bazel-4.2.3-installer-linux-x86_64.sh - chmod +x bazel-4.2.3-installer-linux-x86_64.sh - ./bazel-4.2.3-installer-linux-x86_64.sh --user - export PATH="$PATH:$HOME/bin" - cd ${GITHUB_WORKSPACE}/ipdk.recipe/stratum/stratum - export DPDK_INSTALL=${GITHUB_WORKSPACE}/install - echo "DPDK" > ${GITHUB_WORKSPACE}/install/share/TARGET - xargs -a ${GITHUB_WORKSPACE}/ipdk.recipe/.github/dpdk-tests.txt bazel test --define target=dpdk - cd bazel-testlogs/stratum - zip stratum_ut_logs.zip `find . -name test.xml -print` - - - name: 'Upload stratum-ut-result artifact' - uses: actions/upload-artifact@v3 - with: - name: stratum-ut-result - path: ipdk.recipe/stratum/stratum/bazel-testlogs/stratum/stratum_ut_logs.zip - - - name: 'Execute dpdk target build' - run: | - cd ${GITHUB_WORKSPACE}/ipdk.recipe - export SDE_INSTALL=${GITHUB_WORKSPACE}/install - export DEPEND_INSTALL=${GITHUB_WORKSPACE}/deps - ./make-all.sh --target=dpdk --prefix=${GITHUB_WORKSPACE}/dpdk-target - - build_and_krnlmon_ut_dpdk_target: - timeout-minutes: 60 - runs-on: ubuntu-20.04 - steps: - - name: 'Check Out p4-dpdk repo' - uses: actions/checkout@v3 - with: - repository: p4lang/p4-dpdk-target - submodules: 'recursive' - path: p4sde - - - name: 'Install dependencies' - working-directory: ./p4sde/tools/setup - run: | - sudo apt update -y - sudo apt install python python3-setuptools python3-pip python3-wheel python3-cffi sudo libssl-dev git zip libatomic1 libnl-route-3-dev curl tree -y - pip3 install distro - python3 install_dep.py - cmake --version - sudo apt-get purge cmake - sudo rm -rf /usr/local/bin/cmake || true - sudo rm -rf ~/.local/bin/cmake || true - # removing cmake 3.25 and reinstalling 3.20, as build is failing with cmake 3.25 - wget https://cmake.org/files/v3.20/cmake-3.20.2-linux-x86_64.sh -O ${GITHUB_WORKSPACE}/cmake-3.20.2-linux-x86_64.sh - chmod a+x ${GITHUB_WORKSPACE}/cmake-3.20.2-linux-x86_64.sh - sudo mkdir -p /opt/rh/cmake - cd ${GITHUB_WORKSPACE} && sudo sh cmake-3.20.2-linux-x86_64.sh --prefix=/opt/rh/cmake --skip-license - sudo ln -s /opt/rh/cmake/bin/cmake /usr/local/bin/cmake - cmake --version - - - name: 'Check Out Networking Recipe' - uses: actions/checkout@v3 - with: - submodules: 'recursive' - path: ipdk.recipe - - - name: 'Compile P4 dpdk target' - run: | - export SDE=${GITHUB_WORKSPACE} - mkdir install deps - export SDE_INSTALL=${GITHUB_WORKSPACE}/install - cd ${GITHUB_WORKSPACE}/p4sde - ./autogen.sh - ./configure --prefix=$SDE_INSTALL - make -j6 - make install - - - name: 'Build,Configure and Execute infrap4d dependencies' - run: | - cmake --version - cd ${GITHUB_WORKSPACE}/ipdk.recipe - export IPDK_RECIPE=`pwd` - cd $IPDK_RECIPE/setup - cmake -B build -DCMAKE_INSTALL_PREFIX=${GITHUB_WORKSPACE}/deps - cmake --build build -j6 - - - name: 'Run Krnlmon Unit Tests for DPDK Target' - run: | - cd ${GITHUB_WORKSPACE}/ipdk.recipe - export IPDK_RECIPE=${GITHUB_WORKSPACE}/ipdk.recipe - export SDE_INSTALL=${GITHUB_WORKSPACE}/install - export DEPEND_INSTALL=${GITHUB_WORKSPACE}/deps - cmake -B build -DDPDK_TARGET=on -DWITH_OVSP4RT=off - cmake --build build -j4 --target krnlmon-test || true - cat build/Testing/Temporary/LastTest.log - zip krnlmon_ut_logs.zip build/Testing/Temporary/LastTest.log - - - name: 'Upload krnlmon-ut-result artifact' - uses: actions/upload-artifact@v3 - with: - name: krnlmon_ut_logs - path: ipdk.recipe/krnlmon_ut_logs.zip - - py_bandit_test: - timeout-minutes: 10 - runs-on: ubuntu-20.04 - steps: - - name: 'Checkout Networking Recipe' - uses: actions/checkout@v3 - with: - path: ipdk.recipe - - - name: 'Run Bandit Test' - uses: tj-actions/bandit@v5.1 - with: - targets: | # or a single string "." - ./ipdk.recipe/clients/p4rt-ctl/p4rt-ctl.in - options: "-v" diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml new file mode 100644 index 00000000..9325e62f --- /dev/null +++ b/.github/workflows/pipeline.yml @@ -0,0 +1,150 @@ +name: "P4 Control Plane CI Pipeline" + +on: + push: + branches: + - main + - ipdk_v* + pull_request: + branches: + - main + - ipdk_v* + +concurrency: + # If workflow for PR or push is already running, stop it and start a new one. + group: build-${{ github.ref }} + cancel-in-progress: true + +permissions: read-all + +env: + #--------------------------------------------------------------------- + # Environment variables + #--------------------------------------------------------------------- + SDE_REPOSITORY: ipdk-io/p4dev.dpdk-sde + SDE_TAG: 2023.10.1 + SDE_FILENAME: dpdk-sde-dev-ubuntu-20.04.tar.gz + SDE_INSTALL_DIR: /opt/p4dev/dpdk-sde + + DEPS_REPOSITORY: ipdk-io/stratum-deps + DEPS_TAG: v1.2.1 + DEPS_FILENAME: deps-ubuntu-latest-x86_64.tar.gz + DEPS_INSTALL_DIR: /opt/p4dev/x86deps + + PREREQS: libbsd-dev libnl-3-dev libnl-route-3-dev libnl-genl-3-dev + +jobs: + #--------------------------------------------------------------------- + # 1-build_dpdk_p4cp + #--------------------------------------------------------------------- + build_dpdk_p4cp: + runs-on: ubuntu-latest + + steps: + - name: Clone networking-recipe + uses: actions/checkout@v3 + with: + submodules: recursive + path: recipe + + - name: Install prerequisites + run: | + sudo apt install $PREREQS + + - name: Install DPDK SDE + uses: robinraju/release-downloader@v1.8 + with: + repository: ${{ env.SDE_REPOSITORY }} + tag: ${{ env.SDE_TAG }} + fileName: ${{ env.SDE_FILENAME }} + + - run: | + sudo tar -xzf $SDE_FILENAME -C / + rm $SDE_FILENAME + + - name: Install stratum dependencies + uses: robinraju/release-downloader@v1.8 + with: + repository: ${{ env.DEPS_REPOSITORY }} + tag: ${{ env.DEPS_TAG }} + fileName: ${{ env.DEPS_FILENAME }} + + - run: | + sudo tar -xzf $DEPS_FILENAME -C / + rm $DEPS_FILENAME + + - name: Build networking-recipe + working-directory: recipe + run: | + export DEPEND_INSTALL=$DEPS_INSTALL_DIR + export SDE_INSTALL=$SDE_INSTALL_DIR + ./make-all.sh --target=dpdk --rpath + + #--------------------------------------------------------------------- + # 2-krnlmon_unit_tests + #--------------------------------------------------------------------- + krnlmon_unit_tests: + runs-on: ubuntu-latest + + steps: + - name: Clone networking-recipe + uses: actions/checkout@v3 + with: + path: recipe + + - name: Initialize krnlmon submodule + working-directory: recipe + run: git submodule update --init --checkout --depth 1 krnlmon + + - name: Install prerequisites + run: | + sudo apt install $PREREQS + + - name: Install DPDK SDE + uses: robinraju/release-downloader@v1.8 + with: + repository: ${{ env.SDE_REPOSITORY }} + tag: ${{ env.SDE_TAG }} + fileName: ${{ env.SDE_FILENAME }} + + - run: | + sudo tar -xzf $SDE_FILENAME -C / + rm $SDE_FILENAME + + - name: Install stratum dependencies + uses: robinraju/release-downloader@v1.8 + with: + repository: ${{ env.DEPS_REPOSITORY }} + tag: ${{ env.DEPS_TAG }} + fileName: ${{ env.DEPS_FILENAME }} + + - run: | + sudo tar -xzf $DEPS_FILENAME -C / + rm $DEPS_FILENAME + + - name: Run krnlmon unit tests + working-directory: recipe/krnlmon/krnlmon + run: | + export DEPEND_INSTALL=$DEPS_INSTALL_DIR + export SDE_INSTALL=$SDE_INSTALL_DIR + cmake -S . -B build -DTDI_TARGET=DPDK + cmake --build build --target krnlmon-test + + #--------------------------------------------------------------------- + # 3-py_bandit_check + #--------------------------------------------------------------------- + py_bandit_check: + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - name: Clone networking-recipe + uses: actions/checkout@v3 + with: + path: recipe + + - name: Check p4rt-ctl + uses: tj-actions/bandit@v5.1 + with: + targets: | # or a single string "." + ./recipe/clients/p4rt-ctl/p4rt-ctl.in + options: "-v"