PSMDB 8.0 (#154)

* PSMDB build image with poetry * PSMDB 8.0 build image for PBM * PSMDB do not depend on upstream binary * PSMDB 8.0 suites * PSMDB 8.0 update suites * PSMDB comment stucked suites * PSMDB 8.0 filter inMemory suites * PSMDB 8.0 remove stucked suites * Fix for sdam_json_test and others * Disable ocsp suite due to issues with oscrypto package * Finalize 8.0 suites
Percona-QA · Jul 5, 2024 · a1a518e · a1a518e
1 parent b7a2faa
commit a1a518e
Show file tree

Hide file tree

Showing 4 changed files with 1,113 additions and 3 deletions.
diff --git a/pbm-functional/pytest/Dockerfile b/pbm-functional/pytest/Dockerfile
@@ -46,9 +46,9 @@ RUN yum install -y epel-release && yum install -y cyrus-sasl-gssapi && yum insta
     mkdir -p /pbm-old && mkdir -p /pbm-new && mkdir /etc/pykmip && mkdir /etc/vault && mkdir /etc/x509 && \
     VERSION=$(mongod --version | tail -n +2 | sed 's|Build Info: ||' | jq -r .version | sed -E 's|-.+||') && \
     ARCH=$(mongod --version | tail -n +2 | sed 's/Build Info: //' | jq -r .environment.distarch)  && \
-    if [ $ARCH == x86_64 ]; then curl -Lf -o mongo-ce.tgz https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-$VERSION.tgz ; fi && \
-    if [ $ARCH == aarch64 ]; then curl -Lf -o mongo-ce.tgz https://fastdl.mongodb.org/linux/mongodb-linux-aarch64-rhel82-$VERSION.tgz ; fi && \
-    mkdir -p /opt/mongodb && tar -C /opt/mongodb --transform='s,^/*[^/]*,,S' -xzvf mongo-ce.tgz && rm -rf mongo-ce.tgz
+    if [ $ARCH == x86_64 ]; then curl -Lf -o mongo-ce.tgz https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-$VERSION.tgz || true; fi && \
+    if [ $ARCH == aarch64 ]; then curl -Lf -o mongo-ce.tgz https://fastdl.mongodb.org/linux/mongodb-linux-aarch64-rhel82-$VERSION.tgz || true ; fi && \
+    mkdir -p /opt/mongodb && tar -C /opt/mongodb --transform='s,^/*[^/]*,,S' -xzvf mongo-ce.tgz || true && rm -rf mongo-ce.tgz || true
 COPY --from=builder /pbm-new/ /pbm-new/
 COPY --from=builder /pbm-old/ /pbm-old/
 COPY --from=pykmip /etc/mongod.pem /etc/pykmip/mongod.pem
@@ -60,6 +60,7 @@ COPY conf/krb5.conf /etc/krb5.conf
 COPY keyfile /etc/keyfile
 COPY mongodb-keyfile /etc/mongodb-keyfile
 RUN echo -e 'vaulttoken' > /etc/vault/token && \
+    mkdir -p /var/lib/mongo && \
     chown -R mongodb /var/lib/mongo && \
     chown mongodb /etc/keyfile && chmod 400 /etc/keyfile && \
     chown mongodb /etc/mongodb-keyfile && chmod 400 /etc/mongodb-keyfile && \

diff --git a/regression-tests/build_image/Dockerfile_debian12_gcc_8.0 b/regression-tests/build_image/Dockerfile_debian12_gcc_8.0
@@ -0,0 +1,93 @@
+FROM debian as builder     
+ENV CMAKE_C_FLAGS=" -Wno-error=uninitialized "
+ENV CMAKE_CXX_FLAGS=" -Wno-error=deprecated-declarations -Wno-error=uninitialized "
+ENV AWS_LIBS=/tmp/lib/aws
+ENV VIRTUAL_ENV=/opt/venv
+ARG psm_ver=8.0.0
+ARG psm_release=0
+ARG branch=v8.0
+ARG AWS_VERSION=1.9.379
+ARG mongo_tools_tag=100.4.1
+ARG jemalloc_tag=psmdb-3.2.11-3.1
+ARG repo=https://github.com/percona/percona-server-mongodb.git
+ARG special_targets="dbtest mongo"
+WORKDIR /opt
+RUN apt update && apt install -y python3 python3-dev python3-venv python3-pip cmake curl libssl-dev libldap2-dev libkrb5-dev \
+                                 libcurl4-openssl-dev libsasl2-dev liblz4-dev libpcap-dev libbz2-dev libsnappy-dev \
+                                 zlib1g-dev libzlcore-dev libsasl2-dev liblzma-dev libext2fs-dev e2fslibs-dev bear \
+                                 libc-devtools gcc g++ git wget lsb-release iproute2 net-tools tzdata python-is-python3 \
+                                 clang lld libclang-dev libc++-dev libc++abi-dev 
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN git clone https://github.com/percona/percona-server-mongodb.git && \
+    cd percona-server-mongodb/ && \
+    git checkout $branch && git pull && \
+    pip install --upgrade pip && \
+    pip install 'poetry==1.5.1' 'pyproject-hooks==1.0.0' && \
+    poetry install --sync --no-root && \
+    rm -rf /opt/percona-server-mongodb/.git && mkdir -p /data/db
+RUN cd /opt && git clone https://github.com/aws/aws-sdk-cpp.git && \
+    mkdir -p /tmp/lib/aws && \
+    cd aws-sdk-cpp && git checkout $AWS_VERSION && git submodule update --init --recursive && \
+    mkdir build && cd build && \
+    cmake .. -DCMAKE_C_FLAGS="$CMAKE_C_FLAGS" -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS" -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DBUILD_SHARED_LIBS=OFF -DMINIMIZE_SIZE=ON -DCMAKE_INSTALL_PREFIX="$AWS_LIBS" && \
+    make install && cd /opt && rm -rf aws-sdk-cpp
+RUN cd /opt/percona-server-mongodb/ && \
+    buildscripts/scons.py MONGO_VERSION=$psm_ver -j8 --disable-warnings-as-errors --ssl --opt=on --use-sasl-client \
+                          --wiredtiger --audit --inmemory --hotbackup CPPPATH="$AWS_LIBS/include" LIBPATH="$AWS_LIBS/lib" \
+                          install-mongod install-mongos install-mongo install-dbtest build/install/bin/mongobridge build/install/bin/wt 
+RUN strip --strip-debug /opt/percona-server-mongodb/build/install/bin/mongo && \
+    strip --strip-debug /opt/percona-server-mongodb/build/install/bin/mongod && \
+    strip --strip-debug /opt/percona-server-mongodb/build/install/bin/mongos && \
+    strip --strip-debug /opt/percona-server-mongodb/build/install/bin/mongobridge && \
+    strip --strip-debug /opt/percona-server-mongodb/build/install/bin/dbtest    
+
+FROM debian
+ENV CMAKE_C_FLAGS=" -Wno-error=uninitialized "
+ENV CMAKE_CXX_FLAGS=" -Wno-error=deprecated-declarations -Wno-error=uninitialized "
+ENV AWS_LIBS=/tmp/lib/aws
+ENV VIRTUAL_ENV=/opt/venv
+ARG psm_ver=8.0.0
+ARG psm_release=0
+ARG branch=v8.0
+ARG AWS_VERSION=1.9.379
+ARG mongo_tools_tag=100.4.1
+ARG jemalloc_tag=psmdb-3.2.11-3.1
+ARG repo=https://github.com/percona/percona-server-mongodb.git
+ARG special_targets="dbtest mongo"
+WORKDIR /opt
+RUN apt update && apt install -y python3 python3-dev python3-venv python3-pip cmake curl libssl-dev libldap2-dev libkrb5-dev \
+                                 libcurl4-openssl-dev libsasl2-dev liblz4-dev libpcap-dev libbz2-dev libsnappy-dev \
+                                 zlib1g-dev libzlcore-dev libsasl2-dev liblzma-dev libext2fs-dev e2fslibs-dev bear \
+                                 libc-devtools gcc g++ git wget lsb-release iproute2 net-tools tzdata python-is-python3 \
+                                 clang lld libclang-dev libc++-dev libc++abi-dev 
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN git clone https://github.com/percona/percona-server-mongodb.git && \
+    cd percona-server-mongodb/ && \
+    git checkout $branch && git pull && \
+    pip install --upgrade pip && \
+    pip install 'poetry==1.5.1' 'pyproject-hooks==1.0.0' && \
+    poetry install --sync --no-root && \
+    rm -rf /opt/percona-server-mongodb/.git && mkdir -p /data/db
+RUN cd /opt && git clone https://github.com/aws/aws-sdk-cpp.git && \
+    mkdir -p /tmp/lib/aws && \
+    cd aws-sdk-cpp && git checkout $AWS_VERSION && git submodule update --init --recursive && \
+    mkdir build && cd build && \
+    cmake .. -DCMAKE_C_FLAGS="$CMAKE_C_FLAGS" -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS" -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DBUILD_SHARED_LIBS=OFF -DMINIMIZE_SIZE=ON -DCMAKE_INSTALL_PREFIX="$AWS_LIBS" && \
+    make install && cd /opt && rm -rf aws-sdk-cpp
+COPY --from=builder /opt/percona-server-mongodb/build/install/bin/* /usr/bin/
+WORKDIR /opt/percona-server-mongodb
+ADD https://raw.githubusercontent.com/Percona-QA/psmdb-testing/main/regression-tests/resmoke2junit.py .
+RUN find jstests -type f | xargs chmod 400 && \
+    echo "mongo_version: \"$psm_ver-$psm_release\"" > .resmoke_mongo_version.yml && \
+    pip3 install --upgrade wrapt && \
+    cp /usr/bin/dbtest . && \
+    support-files/ldap-sasl/deploy_ldap_and_sasl.sh || true && \
+    echo "START=yes" >> /etc/default/saslauthd    
+ENV PATH="${PATH}:/data/multiversion"
+ENV PYTHONPATH="/opt/percona-server-mongodb:/opt/percona-server-mongodb/src"
+ENV TZ=America/New_York
+ENV PORTABLE=1
+ENV USE_SSE=1
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone    
diff --git a/regression-tests/build_image/Dockerfile_el9_gcc_8.0 b/regression-tests/build_image/Dockerfile_el9_gcc_8.0
@@ -0,0 +1,54 @@
+FROM oraclelinux:9 as builder     
+ENV CMAKE_C_FLAGS=" -Wno-error=uninitialized "
+ENV CMAKE_CXX_FLAGS=" -Wno-error=deprecated-declarations -Wno-error=uninitialized "
+ENV AWS_LIBS=/tmp/lib/aws
+ENV VIRTUAL_ENV=/opt/venv
+ARG psm_ver=8.0.0
+ARG psm_release=0
+ARG branch=v8.0
+ARG AWS_VERSION=1.9.379
+ARG mongo_tools_tag=100.4.1
+ARG jemalloc_tag=psmdb-3.2.11-3.1
+ARG repo=https://github.com/percona/percona-server-mongodb.git
+ARG special_targets="dbtest mongo"
+WORKDIR /opt
+RUN yum -y install gcc-toolset-12 python3 python3-devel lld python-unversioned-command git cmake curl \
+                   python3.11 python3.11-pip python3.11-devel \
+                   binutils-devel openssl-devel openldap-devel krb5-devel libcurl-devel \
+                   cyrus-sasl-devel bzip2-devel zlib-devel lz4-devel xz-devel e2fsprogs-devel
+RUN python3.11 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN source /opt/rh/gcc-toolset-12/enable && \
+    git clone https://github.com/percona/percona-server-mongodb.git && \
+    cd percona-server-mongodb/ && \
+    git checkout $branch && git pull && \
+    pip install --upgrade pip && \
+    pip install 'poetry==1.5.1' 'pyproject-hooks==1.0.0' && \
+    poetry install --sync --no-root && \
+    rm -rf /opt/percona-server-mongodb/.git && mkdir -p /data/db
+RUN source /opt/rh/gcc-toolset-12/enable && \
+    cd /opt && git clone https://github.com/aws/aws-sdk-cpp.git && \
+    mkdir -p /tmp/lib/aws && \
+    cd aws-sdk-cpp && git checkout $AWS_VERSION && git submodule update --init --recursive && \
+    mkdir build && cd build && \
+    cmake .. -DCMAKE_C_FLAGS="$CMAKE_C_FLAGS" -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS" -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DBUILD_SHARED_LIBS=OFF -DMINIMIZE_SIZE=ON -DCMAKE_INSTALL_PREFIX="$AWS_LIBS" && \
+    make install && cd /opt && rm -rf aws-sdk-cpp
+RUN source /opt/rh/gcc-toolset-12/enable && \
+    cd /opt/percona-server-mongodb/ && \
+    buildscripts/scons.py MONGO_VERSION=$psm_ver CC=/opt/rh/gcc-toolset-12/root/usr/bin/gcc CXX=/opt/rh/gcc-toolset-12/root/usr/bin/g++ \
+                          -j8 --disable-warnings-as-errors --ssl --opt=on --use-sasl-client \
+                          --wiredtiger --audit --inmemory --hotbackup CPPPATH="$AWS_LIBS/include" LIBPATH="$AWS_LIBS/lib64" \
+                          install-mongod install-mongos install-mongo install-dbtest build/install/bin/mongobridge build/install/bin/wt 
+RUN source /opt/rh/gcc-toolset-12/enable && \
+    strip --strip-debug /opt/percona-server-mongodb/build/install/bin/mongo && \
+    strip --strip-debug /opt/percona-server-mongodb/build/install/bin/mongod && \
+    strip --strip-debug /opt/percona-server-mongodb/build/install/bin/mongos && \
+    strip --strip-debug /opt/percona-server-mongodb/build/install/bin/mongobridge && \
+    strip --strip-debug /opt/percona-server-mongodb/build/install/bin/dbtest    
+
+FROM oraclelinux:9
+RUN yum -y install python3 python-unversioned-command git curl openssl-devel openldap-devel krb5-devel cyrus-sasl-devel && \
+    useradd -u 1001 -r -g 0 -m -s /sbin/nologin \
+            -c "Default Application User" mongodb
+COPY --from=builder /opt/percona-server-mongodb/build/install/bin/* /usr/bin/
+EXPOSE 27017