Skip to content

Commit

Permalink
cstrans-df-run: propagate args of the RUN directive
Browse files Browse the repository at this point in the history
... rather than passing them to a shell interpreter after the
transformation:
```
[8/9] STEP 7/13: RUN ["env", "COV_HOST=cspodman", "COVERITY_POSIX_SPAWN_FALLBACK=1", "/opt/cov-sa-2024.3/bin/cov-build",
"--dir=/cov", "--append-log", "sh", "-c", "--mount=type=bind,from=build,src=/workspace/dist,target=/workspace/dist
--mount=type=cache,target=/root/.cache/pip     pip install $(echo dist/*.whl)'[tensorizer]' --verbose"]
Coverity Build Capture (64-bit) version 2024.3.0 on Linux
6.9.5-100.fc39.x86_64 x86_64
Internal version numbers: 3b0d073f2b p-2024.3-push-30

sh: --: invalid option
```

Reported-by: Steve Grubb
  • Loading branch information
kdudka committed Jun 28, 2024
1 parent 34bc96f commit fbdac54
Show file tree
Hide file tree
Showing 5 changed files with 106 additions and 16 deletions.
38 changes: 22 additions & 16 deletions src/cstrans-df-run.cc
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,15 @@ class DockerFileTransformer {
/// match ... in RUN ...
const RE reLineRun_ = RE("^RUN (.*)$");

/// match ... in RUN [...]
const RE reLineRunExec_ = RE("^RUN *\\[(.*)\\] *$");

/// match ... in ... BS-NL
const RE reLineCont_ = RE("(^.*[^\\\\])\\\\ *$");

// split RUN directive with options from the actual command
const RE reLineRunOpts_ = RE("^(RUN +(?:--[A-Za-z0-9_]+=[^ ]+ +)*)(.*)$");

/// match ... in RUN [...]
const RE reLineRunExec_ = RE("^\\[(.*)\\] *$");

/// match in-line comments
const RE reComment_ = RE("^\\s*#.*$");
};
Expand Down Expand Up @@ -148,10 +151,10 @@ std::string runQuoteArg(std::string arg)
return arg;
}

std::string runLineFromExecList(const TStringList &execList)
std::string runCmdFromExecList(const TStringList &execList)
{
// construct RUN ["cmd", "arg1", "arg2", ...] from execList
std::string runLine = "RUN [";
std::string runLine = "[";
int i = 0;
for (const std::string &arg : execList) {
if (i++)
Expand All @@ -165,23 +168,26 @@ std::string runLineFromExecList(const TStringList &execList)

void DockerFileTransformer::transformRunLine(std::string *pRunLine)
{
// split RUN directive with options from the actual command
boost::smatch sm;
if (!boost::regex_match(*pRunLine, sm, reLineRunOpts_))
// should never happen
throw std::runtime_error("internal error");

std::string newRunLine = sm[1];
const std::string cmd = sm[2];

// start with the prefix specified on cmd-line
TStringList execList = prefixCmd_;

boost::smatch sm;
if (boost::regex_match(*pRunLine, sm, reLineRunExec_))
// RUN ["cmd", "arg1", "arg2", ...]
if (boost::regex_match(cmd, sm, reLineRunExec_))
// ["cmd", "arg1", "arg2", ...]
appendExecArgs(&execList, sm[1]);

else if (boost::regex_match(*pRunLine, sm, reLineRun_))
// RUN arbitrary shell code...
appendShellExec(&execList, sm[1]);

else
// should never happen
throw std::runtime_error("internal error");
// arbitrary shell code...
appendShellExec(&execList, cmd);

const std::string newRunLine = runLineFromExecList(execList);
newRunLine += runCmdFromExecList(execList);
if (verbose_) {
// diagnostic output printed with --verbose
std::cerr << prog_name << " <<< " << *pRunLine << std::endl;
Expand Down
38 changes: 38 additions & 0 deletions tests/cstrans-df-run/0011-stdin.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
## Python cuda base #################################################################
FROM cuda-base AS python-cuda-base

ENV VIRTUAL_ENV=/opt/vllm
ENV PATH="$VIRTUAL_ENV/bin:$PATH"

# install cuda and common dependencies
RUN --mount=type=cache,target=/root/.cache/pip \
--mount=type=bind,source=requirements-common.txt,target=requirements-common.txt \
--mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt \
pip install \
-r requirements-cuda.txt

## Development #################################################################
FROM python-cuda-base AS dev

# install build and runtime dependencies
RUN --mount=type=cache,target=/root/.cache/pip \
--mount=type=bind,source=requirements-common.txt,target=requirements-common.txt \
--mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt \
--mount=type=bind,source=requirements-dev.txt,target=requirements-dev.txt \
--mount=type=bind,source=requirements-lint.txt,target=requirements-lint.txt \
--mount=type=bind,source=requirements-test.txt,target=requirements-test.txt \
pip3 install \
-r requirements-cuda.txt \
-r requirements-dev.txt

## Builder #####################################################################
FROM dev AS build

# install build dependencies
RUN --mount=type=cache,target=/root/.cache/pip \
--mount=type=bind,source=requirements-build.txt,target=requirements-build.txt \
pip install -r requirements-build.txt

# install compiler cache to speed up compilation leveraging local or remote caching
# git is required for the cutlass kernels
RUN rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && rpm -ql epel-release && microdnf install -y git ccache && microdnf clean all
24 changes: 24 additions & 0 deletions tests/cstrans-df-run/0011-stdout.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
## Python cuda base #################################################################
FROM cuda-base AS python-cuda-base

ENV VIRTUAL_ENV=/opt/vllm
ENV PATH="$VIRTUAL_ENV/bin:$PATH"

# install cuda and common dependencies
RUN --mount=type=cache,target=/root/.cache/pip --mount=type=bind,source=requirements-common.txt,target=requirements-common.txt --mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "pip install -r requirements-cuda.txt"]

## Development #################################################################
FROM python-cuda-base AS dev

# install build and runtime dependencies
RUN --mount=type=cache,target=/root/.cache/pip --mount=type=bind,source=requirements-common.txt,target=requirements-common.txt --mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt --mount=type=bind,source=requirements-dev.txt,target=requirements-dev.txt --mount=type=bind,source=requirements-lint.txt,target=requirements-lint.txt --mount=type=bind,source=requirements-test.txt,target=requirements-test.txt ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "pip3 install -r requirements-cuda.txt -r requirements-dev.txt"]

## Builder #####################################################################
FROM dev AS build

# install build dependencies
RUN --mount=type=cache,target=/root/.cache/pip --mount=type=bind,source=requirements-build.txt,target=requirements-build.txt ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "pip install -r requirements-build.txt"]

# install compiler cache to speed up compilation leveraging local or remote caching
# git is required for the cutlass kernels
RUN ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && rpm -ql epel-release && microdnf install -y git ccache && microdnf clean all"]
1 change: 1 addition & 0 deletions tests/cstrans-df-run/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -36,3 +36,4 @@ tests_cstrans_df_run(0007)
tests_cstrans_df_run(0008)
tests_cstrans_df_run(0009)
tests_cstrans_df_run(0010)
tests_cstrans_df_run(0011)
21 changes: 21 additions & 0 deletions tests/cstrans-df-run/sync.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#!/bin/zsh

Check failure

Code scanning / shellcheck

SC1071 Error test

ShellCheck only supports sh/bash/dash/ksh scripts. Sorry!
set -exo pipefail

TEST_ARGS=(-- /opt/cov-sa-2019.09/bin/cov-build --dir=/cov --append-log)

# set path to project root
PROJECT_ROOT="../.."

if [[ $# -eq 0 ]]; then
tests=( *-stdin.txt )
else
tests=( "$@" )
fi

for tst in "${tests[@]}"; do
tst=${tst%-stdin.txt}
${PROJECT_ROOT}/csdiff_build/src/cstrans-df-run \
${TEST_ARGS[@]} \
< ${tst}-stdin.txt \
> ${tst}-stdout.txt
done

0 comments on commit fbdac54

Please sign in to comment.