Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Build p4runtime protobufs #328

Merged
merged 45 commits into from
Nov 15, 2023
Merged
Show file tree
Hide file tree
Changes from 44 commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
d27a013
Compile protobufs for c++, python, and go
ffoulkes Nov 4, 2023
e1dd491
Continue development
ffoulkes Nov 5, 2023
f1031aa
Eliminate cmake subdirectory
ffoulkes Nov 5, 2023
e35dafd
Major refactoring
ffoulkes Nov 5, 2023
96ff125
Add protobuf-tarballs target
ffoulkes Nov 5, 2023
6b93d63
Invoke protobuf build from main listfile
ffoulkes Nov 5, 2023
731a760
Rename proto directory to protobufs
ffoulkes Nov 5, 2023
fc9d423
Move tarball generation to an include file
ffoulkes Nov 5, 2023
21e82d4
Remove superfluous progess messages
ffoulkes Nov 5, 2023
8ad6539
Set up Go in GitHub workflow
ffoulkes Nov 5, 2023
d3882c9
Continue development
ffoulkes Nov 5, 2023
acaa242
Tweak Go support
ffoulkes Nov 5, 2023
ce17cf9
Debug protobufs build
ffoulkes Nov 5, 2023
27fcecb
Syntax error in github workflow
ffoulkes Nov 5, 2023
527cd2a
Syntax error in github workflow
ffoulkes Nov 6, 2023
41ea6be
Install gRPC plugins
ffoulkes Nov 6, 2023
5b03ce3
Clean up GitHub workflow
ffoulkes Nov 6, 2023
721dd9b
Continue development
ffoulkes Nov 6, 2023
5aea21a
Display version numbers of GO plugins
ffoulkes Nov 7, 2023
c277ddc
Revise handling of C++ an Python plugins
ffoulkes Nov 7, 2023
7f24471
Refine protobufs project
ffoulkes Nov 7, 2023
14d9886
Add a README file for the protobufs build
ffoulkes Nov 9, 2023
b9f4000
Minor edits to protobufs README
ffoulkes Nov 9, 2023
c5af5c6
Bump minimum Python version to 3.8
ffoulkes Nov 9, 2023
03b07cf
Generate Python wheel
ffoulkes Nov 12, 2023
e481d21
Replace README.md in P4Runtime wheel
ffoulkes Nov 12, 2023
901d50d
Update files for 'protobuf' target
ffoulkes Nov 12, 2023
f886628
Refine p4runtime protobuf generation
ffoulkes Nov 13, 2023
80030ea
Further refinements
ffoulkes Nov 13, 2023
bebcfc8
More refinements
ffoulkes Nov 13, 2023
2c0affd
Update version numbers
ffoulkes Nov 13, 2023
a6dd5c5
Show Protobuf version
ffoulkes Nov 13, 2023
b294691
Check p4runtime protobuf build
ffoulkes Nov 14, 2023
c3064f1
Supply missing prerequisites
ffoulkes Nov 14, 2023
1d8dec8
Install Python dependencies
ffoulkes Nov 14, 2023
7ab3cb9
Address issues with Python dependencies
ffoulkes Nov 14, 2023
8f2f611
Edited the wrong jobs
ffoulkes Nov 14, 2023
9d353dd
Install all the prerequisites
ffoulkes Nov 14, 2023
4c0746c
Specify timeouts on workflows
ffoulkes Nov 14, 2023
76d9923
Refactor StratumDependencies.cmake
ffoulkes Nov 14, 2023
f5eb796
Install packages in share/p4runtime
ffoulkes Nov 15, 2023
ec44107
Drop redundant Python source tarball
ffoulkes Nov 15, 2023
57e1627
Clean up a bit
ffoulkes Nov 15, 2023
13f0cb2
Merge branch 'main' into compile-protobufs
ffoulkes Nov 15, 2023
fe8566b
Update README file
ffoulkes Nov 15, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 75 additions & 5 deletions .github/workflows/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ jobs:
#---------------------------------------------------------------------
build_dpdk_p4cp:
runs-on: ubuntu-latest
timeout-minutes: 20

steps:
- name: Clone networking-recipe
Expand All @@ -51,13 +52,23 @@ jobs:
run: |
sudo apt install $PREREQS

- name: Set up Go environment
ffoulkes marked this conversation as resolved.
Show resolved Hide resolved
uses: actions/[email protected]
with:
go-version: 'stable'

- name: Install gRPC plugins
run: |
go version
go install google.golang.org/protobuf/cmd/[email protected]
go install google.golang.org/grpc/cmd/[email protected]

- name: Install DPDK SDE
uses: robinraju/[email protected]
with:
repository: ${{ env.SDE_REPOSITORY }}
tag: ${{ env.SDE_TAG }}
fileName: ${{ env.SDE_FILENAME }}

- run: |
sudo tar -xzf $SDE_FILENAME -C /
rm $SDE_FILENAME
Expand All @@ -68,7 +79,6 @@ jobs:
repository: ${{ env.DEPS_REPOSITORY }}
tag: ${{ env.DEPS_TAG }}
fileName: ${{ env.DEPS_FILENAME }}

- run: |
sudo tar -xzf $DEPS_FILENAME -C /
rm $DEPS_FILENAME
Expand All @@ -85,6 +95,7 @@ jobs:
#---------------------------------------------------------------------
krnlmon_unit_tests:
runs-on: ubuntu-latest
timeout-minutes: 5

steps:
- name: Clone networking-recipe
Expand All @@ -106,7 +117,6 @@ jobs:
repository: ${{ env.SDE_REPOSITORY }}
tag: ${{ env.SDE_TAG }}
fileName: ${{ env.SDE_FILENAME }}

- run: |
sudo tar -xzf $SDE_FILENAME -C /
rm $SDE_FILENAME
Expand All @@ -117,7 +127,6 @@ jobs:
repository: ${{ env.DEPS_REPOSITORY }}
tag: ${{ env.DEPS_TAG }}
fileName: ${{ env.DEPS_FILENAME }}

- run: |
sudo tar -xzf $DEPS_FILENAME -C /
rm $DEPS_FILENAME
Expand All @@ -134,7 +143,7 @@ jobs:
# 3-py_bandit_check
#---------------------------------------------------------------------
py_bandit_check:
timeout-minutes: 10
timeout-minutes: 5
runs-on: ubuntu-latest
steps:
- name: Clone networking-recipe
Expand All @@ -148,3 +157,64 @@ jobs:
targets: | # or a single string "."
./recipe/clients/p4rt-ctl/p4rt-ctl.in
options: "-v"

#---------------------------------------------------------------------
# 4-build_p4runtime_protos
#---------------------------------------------------------------------
build_p4runtime_protos:
runs-on: ubuntu-latest
timeout-minutes: 5

steps:
- name: Clone networking-recipe
uses: actions/checkout@v3
with:
submodules: recursive
path: recipe

- name: Install prerequisites
run: |
sudo apt install $PREREQS

- name: Install python dependencies
working-directory: recipe
run: |
python -m pip install setuptools build wheel

- name: Set up Go environment
uses: actions/[email protected]
with:
go-version: 'stable'

- name: Install DPDK SDE
uses: robinraju/[email protected]
with:
repository: ${{ env.SDE_REPOSITORY }}
tag: ${{ env.SDE_TAG }}
fileName: ${{ env.SDE_FILENAME }}
- run: |
sudo tar -xzf $SDE_FILENAME -C /
rm $SDE_FILENAME

- name: Install gRPC plugins
run: |
go install google.golang.org/protobuf/cmd/[email protected]
go install google.golang.org/grpc/cmd/[email protected]

- name: Install stratum dependencies
uses: robinraju/[email protected]
with:
repository: ${{ env.DEPS_REPOSITORY }}
tag: ${{ env.DEPS_TAG }}
fileName: ${{ env.DEPS_FILENAME }}
- run: |
sudo tar -xzf $DEPS_FILENAME -C /
rm $DEPS_FILENAME

- name: Build p4runtime protobufs
working-directory: recipe
run: |
export DEPEND_INSTALL=$DEPS_INSTALL_DIR
export SDE_INSTALL=$SDE_INSTALL_DIR
./make-all.sh --target=dpdk --no-ovs --no-build
cmake --build build --target protobufs
4 changes: 2 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
.vscode
.vslick
/build
/install
build
install
17 changes: 17 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
include(CMakePrintHelpers)
include(CompilerSettings)
include(CTest)
include(ExternalProject)
include(FindPkgConfig)
include(GNUInstallDirs)

Expand Down Expand Up @@ -146,8 +147,17 @@ cmake_print_variables(CMAKE_PREFIX_PATH)
# External packages #
#####################

# Stratum dependencies for the target system.
include(StratumDependencies)

# Protobuf compiler for the development system.
find_package(HostProtoc)

# TLS libraries.
find_package(OpenSSL REQUIRED)
mark_as_advanced(OpenSSL_DIR)

# P4 SDE for the target system.
if(DPDK_TARGET)
find_package(DpdkDriver)
elseif(ES2K_TARGET)
Expand All @@ -156,6 +166,7 @@ elseif(TOFINO_TARGET)
find_package(TofinoDriver)
endif()

# Open vSwitch.
if(WITH_OVSP4RT)
find_package(OVS)
endif()
Expand Down Expand Up @@ -243,6 +254,12 @@ function(set_install_rpath TARGET)
endif()
endfunction()

#####################
# Protobufs project #
#####################

include(P4RuntimeProtobufs)

##################
# Subdirectories #
##################
Expand Down
90 changes: 90 additions & 0 deletions cmake/FindHostProtoc.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
# Import host Protobuf compiler.
#
# Copyright 2022-2023 Intel Corporation
# SPDX-License-Identifier: Apache 2.0
#

include_guard(GLOBAL)

#-----------------------------------------------------------------------
# Protobuf compiler.
# Runs on the development system.
#-----------------------------------------------------------------------
find_program(HOST_PROTOC_COMMAND "protoc" NO_CMAKE_FIND_ROOT_PATH)
mark_as_advanced(HOST_PROTOC_COMMAND)

if(HOST_PROTOC_COMMAND)
execute_process(
COMMAND ${HOST_PROTOC_COMMAND} --version
OUTPUT_VARIABLE protoc_output
)
string(STRIP "${protoc_output}" protoc_output)
string(REGEX REPLACE
"^libprotoc +([0-9.]+)$" "\\1" PROTOC_VERSION ${protoc_output})
message(STATUS "Found protoc: ${HOST_PROTOC_COMMAND} (version found \"${PROTOC_VERSION}\")")
unset(protoc_output)
else()
message(FATAL_ERROR "protoc not found")
endif()

#-----------------------------------------------------------------------
# gRPC c++ plugin for the Protobuf compiler.
# Runs on the development (host) system.
#-----------------------------------------------------------------------
find_program(HOST_GRPC_CPP_PLUGIN "grpc_cpp_plugin" NO_CMAKE_FIND_ROOT_PATH)
if(HOST_GRPC_CPP_PLUGIN)
message(STATUS "Found grpc_cpp_plugin")
else()
message(FATAL_ERROR "grpc_cpp_plugin not found")
endif()
mark_as_advanced(HOST_GRPC_CPP_PLUGIN)

#-----------------------------------------------------------------------
# gRPC Python plugin for the Protobuf compiler.
# Runs on the development (host) system.
#-----------------------------------------------------------------------
find_program(HOST_GRPC_PY_PLUGIN "grpc_python_plugin" NO_CMAKE_FIND_ROOT_PATH)
if(HOST_GRPC_PY_PLUGIN)
message(STATUS "Found grpc_python_plugin")
else()
message(WARNING "grpc_python_plugin not found")
endif()
mark_as_advanced(HOST_GRPC_PY_PLUGIN)

#-----------------------------------------------------------------------
# Go plugin for the Protobuf compiler.
# Runs on the development (host) system.
#-----------------------------------------------------------------------
find_program(HOST_PROTOC_GO_PLUGIN "protoc-gen-go" NO_CMAKE_FIND_ROOT_PATH)
mark_as_advanced(HOST_PROTOC_GO_PLUGIN)

if(HOST_PROTOC_GO_PLUGIN)
execute_process(
COMMAND ${HOST_PROTOC_GO_PLUGIN} --version
OUTPUT_VARIABLE plugin_output
)
string(STRIP "${plugin_output}" plugin_output)
message(STATUS "Found ${plugin_output}")
unset(plugin_output)
else()
message(STATUS "protoc-gen-go not found")
endif()

#-----------------------------------------------------------------------
# Go gRPC plugin for the Protobuf compiler.
# Runs on the development (host) system.
#-----------------------------------------------------------------------
find_program(HOST_GRPC_GO_PLUGIN "protoc-gen-go-grpc" NO_CMAKE_FIND_ROOT_PATH)
mark_as_advanced(HOST_GRPC_GO_PLUGIN)

if(HOST_GRPC_GO_PLUGIN)
execute_process(
COMMAND ${HOST_GRPC_GO_PLUGIN} --version
OUTPUT_VARIABLE plugin_output
)
string(STRIP "${plugin_output}" plugin_output)
message(STATUS "Found ${plugin_output}")
unset(plugin_output)
else()
message(STATUS "protoc-gen-go-grpc plugin not found")
endif()
28 changes: 28 additions & 0 deletions cmake/P4RuntimeProtobufs.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#
# Copyright 2022-2023 Intel Corporation
# SPDX-License-Identifier: Apache 2.0
#
# Compile p4runtime protobufs for use by clients.
#

if(DEFINED GEN_GO_PROTOBUFS)
# Don't forward option to external project if it's undefined.
set(gen_go_protobufs -DGEN_GO_PROTOBUFS=${GEN_GO_PROTOBUFS})
endif()

ExternalProject_Add(protobufs
SOURCE_DIR
${CMAKE_CURRENT_SOURCE_DIR}/protobufs
CMAKE_ARGS
-DDEPEND_INSTALL_DIR=${DEPEND_INSTALL_DIR}
-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
-DCPP_OUT=${CMAKE_CURRENT_BINARY_DIR}/cpp_out
-DGO_OUT=${CMAKE_CURRENT_BINARY_DIR}/go_out
-DPY_OUT=${CMAKE_CURRENT_BINARY_DIR}/py_out
${gen_go_protobufs}
INSTALL_COMMAND
${CMAKE_MAKE_PROGRAM} install
EXCLUDE_FROM_ALL TRUE
)

unset(gen_go_protobufs)
8 changes: 5 additions & 3 deletions cmake/ProtobufCompile.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@
# Directory in which the input .proto files should be installed
# HOST_GRPC_CPP_PLUGIN
# Path to the host grpc_cpp_plugin executable
# HOST_PROTOC
# HOST_GRPC_PY_PLUGIN
# Path to the host grpc_python_plugin executable
# HOST_PROTOC_COMMAND
# Path to the host protobuf compiler
# PB_HEADER_INSTALL_DIR
# Directory in which the generated pb.h files should be installed
Expand Down Expand Up @@ -45,7 +47,7 @@ function(generate_proto_files PROTO_FILES SRC_DIR)
OUTPUT
${_src} ${_hdr}
COMMAND
${HOST_PROTOC}
${HOST_PROTOC_COMMAND}
--proto_path=${PROTO_IMPORT_PATH}
--cpp_out=${PB_OUT_DIR}
-I${STRATUM_SOURCE_DIR}
Expand Down Expand Up @@ -97,7 +99,7 @@ function(generate_grpc_files PROTO_FILES SRC_DIR)
OUTPUT
${_src} ${_hdr}
COMMAND
${HOST_PROTOC}
${HOST_PROTOC_COMMAND}
--proto_path=${PROTO_IMPORT_PATH}
--grpc_out=${PB_OUT_DIR}
--plugin=protoc-gen-grpc=${HOST_GRPC_CPP_PLUGIN}
Expand Down
36 changes: 2 additions & 34 deletions cmake/StratumDependencies.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ mark_as_advanced(glog_DIR)
find_package(Protobuf CONFIG REQUIRED)
mark_as_advanced(Protobuf_DIR)

message(STATUS "Found Protobuf version ${Protobuf_VERSION}")

#-----------------------------------------------------------------------
# Google RPC (gRPC).
#-----------------------------------------------------------------------
Expand All @@ -48,37 +50,3 @@ mark_as_advanced(gRPC_DIR)
mark_as_advanced(c-ares_DIR)

message(STATUS "Found gRPC version ${gRPC_VERSION}")

#-----------------------------------------------------------------------
# Protobuf compiler.
# Runs on the development system.
#-----------------------------------------------------------------------
find_program(HOST_PROTOC "protoc" NO_CMAKE_FIND_ROOT_PATH)
mark_as_advanced(HOST_PROTOC)

if(HOST_PROTOC)
message(STATUS "Found protoc: ${HOST_PROTOC}")
else()
message(FATAL_ERROR "protoc not found")
endif()

#-----------------------------------------------------------------------
# gRPC plugin for Protobuf compiler.
# Runs on the development system.
#-----------------------------------------------------------------------
find_program(HOST_GRPC_CPP_PLUGIN "grpc_cpp_plugin" NO_CMAKE_FIND_ROOT_PATH)
mark_as_advanced(HOST_GRPC_CPP_PLUGIN)

if(HOST_GRPC_CPP_PLUGIN)
message(STATUS "Found grpc_cpp_plugin: ${HOST_GRPC_CPP_PLUGIN}")
else()
message(FATAL_ERROR "grpc_cpp_plugin not found")
endif()

#-----------------------------------------------------------------------
# SSL/TLS library (OpenSSL).
#-----------------------------------------------------------------------
find_package(OpenSSL REQUIRED)
mark_as_advanced(OpenSSL_DIR)

set(WITH_OPENSSL TRUE)
1 change: 1 addition & 0 deletions protobufs/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/build
Loading
Loading