Merge branch 'latest' into net

Signed-off-by: Matthieu Baerts (NGI0) <[email protected]>

Dockerfile

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-FROM ubuntu:23.10
+FROM ubuntu:24.04
 
 LABEL name=mptcp-upstream-virtme-docker
 
@@ -9,7 +9,7 @@ RUN apt-get update && \
 	apt-get dist-upgrade -y && \
 	DEBIAN_FRONTEND=noninteractive \
 	apt-get install -y --no-install-recommends \
-		build-essential libncurses5-dev gcc libssl-dev bc bison automake \
+		build-essential libncurses5-dev gcc libssl-dev bc bison byacc automake \
 		libelf-dev flex git curl tar hashalot qemu-kvm sudo expect \
 		python3 python3-pip python3-pkg-resources file virtiofsd \
 		busybox-static coreutils python3-requests libvirt-clients udev \
@@ -18,8 +18,8 @@ RUN apt-get update && \
 		libdbus-1-dev libnl-genl-3-dev libibverbs-dev \
 		tcpdump \
 		pkg-config libmnl-dev \
-		clang lld llvm llvm-dev libcap-dev \
-		gdb crash dwarves strace \
+		clang clangd lld llvm llvm-dev libcap-dev \
+		gdb gdb-multiarch crash dwarves strace trace-cmd \
 		iptables ebtables nftables vim psmisc bash-completion less jq \
 		gettext-base libevent-dev libtraceevent-dev libnewt0.52 libslang2 libutempter0 python3-newt tmux \
 		libdwarf-dev libbfd-dev libnuma-dev libzstd-dev libunwind-dev libdw-dev libslang2-dev python3-dev python3-setuptools binutils-dev libiberty-dev libbabeltrace-dev systemtap-sdt-dev libperl-dev python3-docutils \

README.md

@@ -75,12 +75,24 @@ These scripts are sourced and can use functions from the virtme script.
 
 ### Env vars
 
+Env vars can be set to change the behaviour of the script. When using the Docker
+command, you need to specify the `-e` parameter, e.g. to set
+`INPUT_BUILD_SKIP=1`:
+
+```bash
+docker run -e INPUT_BUILD_SKIP=1 (...) mptcp/mptcp-upstream-virtme-docker:latest (...)
+```
+
 #### Skip kernel build
 
 If you didn't change the kernel code, it can be useful to skip the compilation
 part. You can then set `INPUT_BUILD_SKIP=1` to save a few seconds to start the
 VM.
 
+#### Use CLang instead of GCC
+
+Simply set `INPUT_CLANG=1` env var with all the commands you use.
+
 #### Not blocking with questions
 
 You can set `INPUT_NO_BLOCK=1` env var not to block if these files are present.
@@ -217,7 +229,29 @@ This can happen when switching between major versions of the compiler. In this
 case, it will be required to clean the build dir in `.virtme/build`, e.g.
 
 ```bash
-docker run -v "${PWD}:${PWD}:rw" -w "${PWD}"--rm -it \
+docker run -v "${PWD}:${PWD}:rw" -w "${PWD}" --rm -it \
   mptcp/mptcp-upstream-virtme-docker:latest \
-  cmd rm -r .virtme/build/tools
+  cmd rm -r .virtme/build*/tools
 ```
+
+## Working with VSCode
+
+If you use [VSCode for Linux kernel development](https://github.com/FlorentRevest/linux-kernel-vscode)
+add-on, you can configure it to use this docker image: simply copy all files
+from the [`vscode`](/tree/main/vscode) directory in your `.vscode` dir from the
+kernel source (or use symbolic links).
+
+Notes:
+- The VSCode add-on needs some modifications, see
+  [PR #5](https://github.com/FlorentRevest/linux-kernel-vscode/pull/5) and
+  [PR #6](https://github.com/FlorentRevest/linux-kernel-vscode/pull/6). If these
+  PRs are not merged, you can use
+  [this fork](https://github.com/matttbe/linux-kernel-vscode/) (`virtme-support`
+  branch) for the moment.
+- If you build the Docker image yourself, you can use a symbolic link instead of
+  the `clangd` script:
+  ```bash
+  ln -s /PATH/TO/mptcp-upstream-virtme-docker/run-tests-dev-cmd.sh .vscode/mptcp-virtme-clangd
+  ```
+- CLang will be used by VSCode, e.g. to generate `compile_commands.json`. You
+  then need to launch docker commands with `-e INPUT_CLANG=1`

build.sh

@@ -1,4 +1,18 @@
 #!/bin/bash -ex
 # SPDX-License-Identifier: GPL-2.0
+
 cd "$(dirname "$(realpath -P "${0}")")"
-docker build -t "${DOCKER_VIRTME_NAME:-mptcp/mptcp-upstream-virtme-docker:latest}" -f Dockerfile .
+
+ARGS=(
+	-t "${DOCKER_VIRTME_NAME:-mptcp/mptcp-upstream-virtme-docker:latest}"
+	-f Dockerfile
+)
+
+if [[ "${-}" =~ "x" ]]; then
+	ARGS+=(--progress plain)
+else
+	echo "Building Docker image" >&2
+	ARGS+=(--quiet)
+fi
+
+docker buildx build "${ARGS[@]}" "${@}" .

entrypoint.sh

@@ -35,6 +35,10 @@ set_trace_off() {
 	fi
 }
 
+with_clang() {
+	[ "${INPUT_CLANG}" = "1" ]
+}
+
 set_trace_on
 
 # The behaviour can be changed with 'input' env var
@@ -83,6 +87,7 @@ BASH_PROFILE="/root/.bash_profile"
 
 VIRTME_WORKDIR="${KERNEL_SRC}/.virtme"
 VIRTME_BUILD_DIR="${VIRTME_WORKDIR}/build"
+with_clang && VIRTME_BUILD_DIR+="-clang"
 VIRTME_SCRIPTS_DIR="${VIRTME_WORKDIR}/scripts"
 VIRTME_PERF_DIR="${VIRTME_BUILD_DIR}/tools/perf"
 VIRTME_TOOLS_SBIN_DIR="${VIRTME_BUILD_DIR}/tools/sbin"
@@ -102,6 +107,7 @@ BPFTESTS_CONFIG="${BPFTESTS_DIR}/config"
 
 export CCACHE_MAXSIZE="${INPUT_CCACHE_MAXSIZE}"
 export CCACHE_DIR="${VIRTME_WORKDIR}/ccache"
+with_clang && CCACHE_DIR+="-clang"
 
 export KBUILD_OUTPUT="${VIRTME_BUILD_DIR}"
 export KCONFIG_CONFIG="${VIRTME_KCONFIG}"
@@ -299,12 +305,20 @@ check_source_exec_all() {
 	log_section_end
 }
 
-_make() {
+read -ra MAKE_ARGS <<< "${INPUT_MAKE_ARGS}"
+with_clang && MAKE_ARGS+=(LLVM=1 LLVM_IAS=1 CC=clang ARCH="${VIRTME_ARCH}")
+MAKE_ARGS_O=("${MAKE_ARGS[@]}" O="${VIRTME_BUILD_DIR}")
+
+_make_j() {
 	make -j"$(nproc)" -l"$(nproc)" "${@}"
 }
 
+_make() {
+	_make_j "${MAKE_ARGS[@]}" "${@}"
+}
+
 _make_o() {
-	_make O="${VIRTME_BUILD_DIR}" "${@}"
+	_make_j "${MAKE_ARGS_O[@]}" "${@}"
 }
 
 # $1: source ; $2: target
@@ -348,7 +362,13 @@ gen_kconfig() { local mode kconfig=() vck rc=0
 	kconfig+=(-e PANIC_ON_OOPS)
 
 	# Debug info for developers
-	kconfig+=(-e DEBUG_INFO -e DEBUG_INFO_DWARF4 -e GDB_SCRIPTS)
+	kconfig+=(-e DEBUG_INFO -e DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT -e GDB_SCRIPTS)
+	if with_clang; then
+		kconfig+=(-e DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT)
+	else
+		# decode_stacktrace.sh script reports '??:?' with GCC and DWARF5
+		kconfig+=(-e DEBUG_INFO_DWARF4)
+	fi
 
 	# Compressed (old/new option)
 	kconfig+=(-e DEBUG_INFO_COMPRESSED -e DEBUG_INFO_COMPRESSED_ZLIB)
@@ -400,7 +420,7 @@ gen_kconfig() { local mode kconfig=() vck rc=0
 	kconfig+=("${@}")
 
 	# KBUILD_OUTPUT is used by virtme
-	"${VIRTME_CONFIGKERNEL}" "${vck[@]}" || rc=${?}
+	"${VIRTME_CONFIGKERNEL}" "${vck[@]}" "${MAKE_ARGS_O[@]}" || rc=${?}
 
 	./scripts/config --file "${VIRTME_KCONFIG}" "${kconfig[@]}" || rc=${?}
 
@@ -1593,6 +1613,9 @@ case "${INPUT_MODE}" in
 			COMPILER="${COMPILER}" \
 				"${MAKE_CROSS}" "${@}"
 		;;
+	"defconfig")
+		gen_kconfig "${@:-normal}"
+		;;
 	"cmd" | "command")
 		"${@}"
 		;;

run-tests-clang.sh

@@ -0,0 +1,6 @@
+#!/bin/bash -e
+# SPDX-License-Identifier: GPL-2.0
+DIR="$(dirname "$(realpath -P "${0}")")"
+export INPUT_CLANG=1
+
+bash "-${-}" "${DIR}/run-tests.sh" "${@}"

run-tests-dev-clang.sh

@@ -0,0 +1,6 @@
+#!/bin/bash -ex
+# SPDX-License-Identifier: GPL-2.0
+DIR="$(dirname "$(realpath -P "${0}")")"
+export INPUT_CLANG=1
+
+bash "-${-}" "${DIR}/run-tests-dev.sh" "${@}"

run-tests-dev-cmd.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -e
+# SPDX-License-Identifier: GPL-2.0
+DIR="$(dirname "$(realpath -P "${0}")")"
+CMD="$(basename "${0}")"
+
+if [ "${CMD}" = "run-tests-dev-cmd.sh" ]; then
+	if [ -n "${1}" ]; then
+		CMD="${1}"
+		shift
+	else
+		CMD="bash"
+	fi
+elif [[ "${CMD}" = ".virtme-run-"* ]]; then
+	export VIRTME_NO_INTERACTIVE=1
+	CMD="${CMD:12}"
+elif [[ "${CMD}" = "mptcp-virtme-"* ]]; then
+	export VIRTME_NO_INTERACTIVE=1
+	CMD="${CMD:13}"
+fi
+
+bash "-${-}" "${DIR}/run-tests-dev.sh" cmd "${CMD}" "${@}"

run-tests-dev.sh

@@ -9,6 +9,6 @@ fi
 export INPUT_NO_BLOCK="${INPUT_NO_BLOCK:-1}"
 
 bash "-${-}" "${DIR}/build.sh"
-docker system prune --filter "label=name=mptcp-upstream-virtme-docker" -f
+docker system prune --filter "label=name=mptcp-upstream-virtme-docker" -f >&2
 
 bash "-${-}" "${DIR}/run.sh" "${@}"

run.sh

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: GPL-2.0
 
 VIRTME_INTERACTIVE=""
+test -t 1 && VIRTME_INTERACTIVE="-t"
 [ "${VIRTME_NO_INTERACTIVE}" != 1 ] && VIRTME_INTERACTIVE="-it"
 
 # host is different if worktree are used
@@ -15,13 +16,15 @@ docker run \
 	${VIRTME_PACKETDRILL_PATH:+-v "${VIRTME_PACKETDRILL_PATH}:/opt/packetdrill:rw"} \
 	-v "${HOME_DIR}:/root" \
 	-w "${PWD}" \
+	-e "INPUT_CLANG" \
 	-e "INPUT_TRACE" \
 	-e "INPUT_NO_BLOCK" \
 	-e "INPUT_RUN_LOOP_CONTINUE" \
 	-e "INPUT_BUILD_SKIP" \
 	-e "INPUT_BUILD_SKIP_PERF" \
 	-e "INPUT_BUILD_SKIP_SELFTESTS" \
 	-e "INPUT_BUILD_SKIP_PACKETDRILL" \
+	-e "INPUT_MAKE_ARGS" \
 	-e "INPUT_RUN_TESTS_ONLY" \
 	-e "INPUT_RUN_TESTS_EXCEPT" \
 	-e "INPUT_SELFTESTS_DIR" \

vscode/local-virtme.sh

@@ -0,0 +1,33 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+# shellcheck disable=SC2317
+
+export VIRTME_NO_INTERACTIVE=1
+export INPUT_CLANG="1"
+export MAKE="./.virtme.sh make"
+export SILENT_BUILD_FLAG=" "
+export SPINNER=0
+
+defconfig() {
+	./.virtme.sh defconfig
+}
+
+case "${COMMAND}" in
+	"build" | "clean" | "menuconfig" | "update" | "systemtap-build")
+		echo "local: allow: ${COMMAND}"
+		;;
+	"gdb-index")
+		echo "local: skip: ${COMMAND}"
+		exit 0
+		;;
+	"defconfig")
+		echo "local: custom: ${COMMAND}"
+		${COMMAND}
+		exit
+		;;
+	*)
+		echo "local: block: ${COMMAND}"
+		exit 1
+		;;
+esac

vscode/mptcp-virtme-clangd

@@ -0,0 +1,6 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+docker run -v "${PWD}:${PWD}:rw" -w "${PWD}" --rm -t \
+	mptcp/mptcp-upstream-virtme-docker:latest \
+	cmd clangd "${@}"

vscode/settings-extra.json

@@ -0,0 +1,6 @@
+{
+  "clangd.arguments": [
+    "-compile-commands-dir=${workspaceFolder}/.virtme/build-clang"
+  ],
+  "clangd.path": "${workspaceFolder}/.vscode/mptcp-virtme-clangd"
+}