cstrans-df-run: propagate args of the RUN directive

... rather than passing them to a shell interpreter after the
transformation:
```
[8/9] STEP 7/13: RUN ["env", "COV_HOST=cspodman", "COVERITY_POSIX_SPAWN_FALLBACK=1", "/opt/cov-sa-2024.3/bin/cov-build",
"--dir=/cov", "--append-log", "sh", "-c", "--mount=type=bind,from=build,src=/workspace/dist,target=/workspace/dist
--mount=type=cache,target=/root/.cache/pip     pip install $(echo dist/*.whl)'[tensorizer]' --verbose"]
Coverity Build Capture (64-bit) version 2024.3.0 on Linux
6.9.5-100.fc39.x86_64 x86_64
Internal version numbers: 3b0d073f2b p-2024.3-push-30

sh: --: invalid option
```

Reported-by: Steve Grubb
Closes: #190

src/cstrans-df-run.cc

@@ -55,12 +55,15 @@ class DockerFileTransformer {
         /// match ... in RUN ...
         const RE reLineRun_     = RE("^RUN (.*)$");
 
-        /// match ... in RUN [...]
-        const RE reLineRunExec_ = RE("^RUN  *\\[(.*)\\] *$");
-
         /// match ... in ... BS-NL
         const RE reLineCont_    = RE("(^.*[^\\\\])\\\\ *$");
 
+        // split RUN directive with options from the actual command
+        const RE reLineRunOpts_ = RE("^(RUN +(?:--[A-Za-z0-9_]+=[^ ]+ +)*)(.*)$");
+
+        /// match ... in RUN [...]
+        const RE reLineRunExec_ = RE("^\\[(.*)\\] *$");
+
         /// match in-line comments
         const RE reComment_     = RE("^\\s*#.*$");
 };
@@ -148,10 +151,10 @@ std::string runQuoteArg(std::string arg)
     return arg;
 }
 
-std::string runLineFromExecList(const TStringList &execList)
+std::string runCmdFromExecList(const TStringList &execList)
 {
     // construct RUN ["cmd", "arg1", "arg2", ...] from execList
-    std::string runLine = "RUN [";
+    std::string runLine = "[";
     int i = 0;
     for (const std::string &arg : execList) {
         if (i++)
@@ -165,23 +168,26 @@ std::string runLineFromExecList(const TStringList &execList)
 
 void DockerFileTransformer::transformRunLine(std::string *pRunLine)
 {
+    // split RUN directive with options from the actual command
+    boost::smatch sm;
+    if (!boost::regex_match(*pRunLine, sm, reLineRunOpts_))
+        // should never happen
+        throw std::runtime_error("internal error");
+
+    std::string newRunLine = sm[1];
+    const std::string cmd = sm[2];
+
     // start with the prefix specified on cmd-line
     TStringList execList = prefixCmd_;
 
-    boost::smatch sm;
-    if (boost::regex_match(*pRunLine, sm, reLineRunExec_))
-        // RUN ["cmd", "arg1", "arg2", ...]
+    if (boost::regex_match(cmd, sm, reLineRunExec_))
+        // ["cmd", "arg1", "arg2", ...]
         appendExecArgs(&execList, sm[1]);
-
-    else if (boost::regex_match(*pRunLine, sm, reLineRun_))
-        // RUN arbitrary shell code...
-        appendShellExec(&execList, sm[1]);
-
     else
-        // should never happen
-        throw std::runtime_error("internal error");
+        // arbitrary shell code...
+        appendShellExec(&execList, cmd);
 
-    const std::string newRunLine = runLineFromExecList(execList);
+    newRunLine += runCmdFromExecList(execList);
     if (verbose_) {
         // diagnostic output printed with --verbose
         std::cerr << prog_name << " <<< " << *pRunLine << std::endl;

tests/cstrans-df-run/0011-stdin.txt

@@ -0,0 +1,38 @@
+## Python cuda base #################################################################
+FROM cuda-base AS python-cuda-base
+
+ENV VIRTUAL_ENV=/opt/vllm
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# install cuda and common dependencies
+RUN --mount=type=cache,target=/root/.cache/pip \
+    --mount=type=bind,source=requirements-common.txt,target=requirements-common.txt \
+    --mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt \
+    pip install \
+        -r requirements-cuda.txt
+
+## Development #################################################################
+FROM python-cuda-base AS dev
+
+# install build and runtime dependencies
+RUN --mount=type=cache,target=/root/.cache/pip \
+    --mount=type=bind,source=requirements-common.txt,target=requirements-common.txt \
+    --mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt \
+    --mount=type=bind,source=requirements-dev.txt,target=requirements-dev.txt \
+    --mount=type=bind,source=requirements-lint.txt,target=requirements-lint.txt \
+    --mount=type=bind,source=requirements-test.txt,target=requirements-test.txt \
+    pip3 install \
+        -r requirements-cuda.txt \
+        -r requirements-dev.txt
+
+## Builder #####################################################################
+FROM dev AS build
+
+# install build dependencies
+RUN --mount=type=cache,target=/root/.cache/pip \
+    --mount=type=bind,source=requirements-build.txt,target=requirements-build.txt \
+    pip install -r requirements-build.txt
+
+# install compiler cache to speed up compilation leveraging local or remote caching
+# git is required for the cutlass kernels
+RUN rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && rpm -ql epel-release && microdnf install -y git ccache && microdnf clean all

tests/cstrans-df-run/0011-stdout.txt

@@ -0,0 +1,24 @@
+## Python cuda base #################################################################
+FROM cuda-base AS python-cuda-base
+
+ENV VIRTUAL_ENV=/opt/vllm
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# install cuda and common dependencies
+RUN --mount=type=cache,target=/root/.cache/pip     --mount=type=bind,source=requirements-common.txt,target=requirements-common.txt     --mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt     ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "pip install         -r requirements-cuda.txt"]
+
+## Development #################################################################
+FROM python-cuda-base AS dev
+
+# install build and runtime dependencies
+RUN --mount=type=cache,target=/root/.cache/pip     --mount=type=bind,source=requirements-common.txt,target=requirements-common.txt     --mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt     --mount=type=bind,source=requirements-dev.txt,target=requirements-dev.txt     --mount=type=bind,source=requirements-lint.txt,target=requirements-lint.txt     --mount=type=bind,source=requirements-test.txt,target=requirements-test.txt     ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "pip3 install         -r requirements-cuda.txt         -r requirements-dev.txt"]
+
+## Builder #####################################################################
+FROM dev AS build
+
+# install build dependencies
+RUN --mount=type=cache,target=/root/.cache/pip     --mount=type=bind,source=requirements-build.txt,target=requirements-build.txt     ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "pip install -r requirements-build.txt"]
+
+# install compiler cache to speed up compilation leveraging local or remote caching
+# git is required for the cutlass kernels
+RUN ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && rpm -ql epel-release && microdnf install -y git ccache && microdnf clean all"]

tests/cstrans-df-run/CMakeLists.txt

@@ -36,3 +36,4 @@ tests_cstrans_df_run(0007)
 tests_cstrans_df_run(0008)
 tests_cstrans_df_run(0009)
 tests_cstrans_df_run(0010)
+tests_cstrans_df_run(0011)

tests/cstrans-df-run/sync.sh

@@ -0,0 +1,21 @@
+#!/bin/zsh
+set -exo pipefail
+
+TEST_ARGS=(-- /opt/cov-sa-2019.09/bin/cov-build --dir=/cov --append-log)
+
+# set path to project root
+PROJECT_ROOT="../.."
+
+if [[ $# -eq 0 ]]; then
+    tests=( *-stdin.txt )
+else
+    tests=( "$@" )
+fi
+
+for tst in "${tests[@]}"; do
+    tst=${tst%-stdin.txt}
+    ${PROJECT_ROOT}/csdiff_build/src/cstrans-df-run \
+        ${TEST_ARGS[@]} \
+        < ${tst}-stdin.txt \
+        > ${tst}-stdout.txt
+done