Possible Extensions: discussion-only, do not merge

docker/fc33/basic-setup.sh

@@ -69,4 +69,4 @@ cat <<EOF > /etc/sysctl.d/hugepages.conf
 vm.nr_hugepages=2
 EOF
 
-rpm -U /opt/rpms/*.rpm
+rpm -U --force /opt/rpms/*.rpm

docker/fc33/kvm.Dockerfile

@@ -2,12 +2,7 @@ FROM fedora:33 as rpms
 
 COPY --from=local_mkt/support_rdma_core:fc33 /root/rpmbuild/RPMS/x86_64/*.rpm /opt/rpms/
 COPY --from=local_mkt/support_simx:fc33 /root/rpmbuild/RPMS/x86_64/*.rpm /opt/rpms/
-
-RUN rm -f \
-   /opt/rpms/*debug*.rpm \
-   /opt/rpms/*ibacm*.rpm \
-   /opt/rpms/*devel*.rpm \
-   /opt/rpms/*iwpmd*.rpm
+COPY --from=local_mkt/support_qemu:fc33 /root/rpmbuild/RPMS/x86_64/*.rpm /opt/rpms/
 
 FROM fedora:33
 
@@ -58,10 +53,44 @@ RUN \
     strace \
     sudo \
     tcpdump \
+    passwd \
+    nmap-ncat \
+    e2fsprogs \
     udev \
     unzip \
     valgrind \
     wget \
+    bc \
+    binutils \
+    bison \
+    ccache \
+    cmake \
+    ctags \
+    elfutils-devel \
+    elfutils-libelf-devel \
+    findutils \
+    flex \
+    gcc \
+    gcc-c++ \
+    git-core \
+    glib2-devel \
+    hostname \
+    iptables-devel \
+    libaio-devel \
+    linux-atm-libs-devel \
+    libattr-devel \
+    libcap-devel \
+    libcap-ng-devel \
+    libdb-devel \
+    libfdt-devel \
+    libmnl-devel \
+    libnl3-cli \
+    libnl3-devel \
+    libseccomp-devel \
+    libudev-devel \
+    libusb-devel \
+    libxml2-devel \
+    llvm-devel \
     && dnf clean dbcache packages
 
 COPY --from=rpms /opt/rpms /opt/rpms
@@ -70,4 +99,12 @@ ADD sshd_config ssh_host_rsa_key /etc/ssh/
 
 ADD basic-setup.sh kvm-setup.sh /root/
 
-RUN /root/basic-setup.sh && /root/kvm-setup.sh
+RUN /root/basic-setup.sh
+#RUN /root/basic-setup_rpm.sh
+RUN /root/kvm-setup.sh
+
+RUN ls /opt
+COPY --from=local_mkt/support_simx:fc33 /opt/simx-src.tar.gz /opt/
+RUN ls /opt
+RUN cd /opt && tar -xzvf simx-src.tar.gz
+RUN ls /opt

docker/fc33/support-qemu.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+# ---
+# git_url: [email protected]:artpol84/qemu.git
+# bkp_git_commit: 0261e73224a48e99b2dabb065437ae21b3e08944
+# git_commit: 16d97a001da
+# git_modules: yes
+
+cat <<EOF > mlx-qemu.spec
+%global debug_package %{nil}
+
+Name:           mlx-qemu
+Version:        1
+Release:        1%{?dist}
+Summary:	    QEMU with Suspend/Resume (SR) capability for VFIO
+License:        Proprietary
+
+%description
+
+
+%build
+./configure \
+        --prefix=/opt/qemu-sr \
+        --disable-strip \
+        --disable-qom-cast-debug \
+        --extra-ldflags="-pie -Wl,-z,relro -Wl,-z,now" \
+        --extra-cflags="-fPIE -DPIE" \
+        --enable-trace-backends=simple,dtrace \
+        --with-trace-file=/var/log/traces \
+        --enable-werror \
+        --disable-xen \
+        --disable-virtfs \
+        --enable-kvm \
+        --enable-libusb \
+        --enable-spice \
+        --enable-seccomp \
+        --disable-fdt \
+        --disable-docs \
+        --disable-sdl \
+        --disable-debug-tcg \
+        --disable-sparse \
+        --disable-brlapi \
+        --disable-vde \
+        --disable-curses \
+        --disable-curl \
+        --enable-vnc-sasl \
+        --enable-linux-aio \
+        --enable-lzo \
+        --enable-snappy \
+        --enable-usb-redir \
+        --enable-vnc-png \
+        --disable-vnc-jpeg \
+        --disable-vhost-scsi \
+        --target-list=x86_64-softmmu \
+        --block-drv-rw-whitelist=qcow2,raw,file,host_device,blkdebug,nbd,iscsi,gluster,rbd \
+        --block-drv-ro-whitelist=vmdk,vhdx,vpc \
+        --enable-debug-info \
+
+
+#        --enable-debug
+
+#        --audio-drv-list=alsa \
+#        --enable-rbd \
+#        --disable-bluez \
+#        --enable-glusterfs \
+
+./scripts/git-submodule.sh update
+
+make %{?_smp_mflags}
+
+#%install
+make DESTDIR=%{buildroot} install
+
+%files
+/opt/qemu-sr/*
+EOF
+
+rpmbuild --build-in-place -bb mlx-qemu.spec

docker/fc33/support-simx.sh

@@ -1,7 +1,9 @@
 #!/bin/bash
 # ---
-# git_url: http://webdev01.mtl.labs.mlnx:8080/git/simx.git
-# git_commit: 9f601e894e3ac74795a8990c0349ea36059004c9
+# git_url: /global/home/users/artemp/scratch/SOFTWARE/simx.git
+# git_commit: 022bedab1
+# old_git_commit: 7032d67e5
+# old_git_commit: 9f601e894e3ac74795a8990c0349ea36059004c9
 
 cat <<EOF > mlx-simx.spec
 %global debug_package %{nil}
@@ -16,7 +18,9 @@ License:        Proprietary
 From simx.git
 
 %build
-./mlnx_infra/config.status.mlnx --target=x86 --prefix=/opt/simx
+cat ./mlnx_infra/config.status.mlnx | grep -v '\-\-disable\-tools' > ./mlnx_infra/config.status.mlnx.new
+chmod +x ./mlnx_infra/config.status.mlnx.new
+./mlnx_infra/config.status.mlnx.new --target=x86 --prefix=/opt/simx
 make %{?_smp_mflags}
 make %{?_smp_mflags} -C mellanox/
 
@@ -26,10 +30,14 @@ mkdir -p %{buildroot}/etc/qemu-kvm/
 ln -s /etc/qemu/bridge.conf %{buildroot}/etc/qemu-kvm/bridge.conf
 mkdir -p %{buildroot}/opt/simx/lib/
 cp mellanox/libml*.so %{buildroot}/opt/simx/lib/
+cp ./mlnx_infra/bridge-{start,stop}.sh  %{buildroot}/opt/simx/bin/
 
 %files
 /opt/simx/*
 /etc/qemu-kvm/bridge.conf
 EOF
 
 rpmbuild --build-in-place -bb mlx-simx.spec
+
+cd /opt/
+tar -czvf simx-src.tar.gz src

docker/fc33/support-smatch.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 # ---
-# git_url: git://repo.or.cz/smatch.git
+# git_url: https://repo.or.cz/smatch.git
 # git_commit: 72c21a144a812cadbe349801da1b24bc331af256
 
 cat <<EOF > smatch.spec

docker/fc33/support-sparse.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 # ---
-# git_url: git://git.kernel.org/pub/scm/devel/sparse/sparse.git
+# git_url: https://git.kernel.org/pub/scm/devel/sparse/sparse.git
 # git_commit: 49c98aa3ed1b315ed2f4fbe44271ecd5bdd9cbc7
 
 cat <<EOF > sparse.spec

docker/fc33/support.Dockerfile

@@ -57,4 +57,8 @@ RUN dnf install -y \
     uuid-devel \
     valgrind-devel \
     zlib-devel \
+    lzo-devel \
+    snappy-devel \
+    cyrus-sasl-devel \
+    libpng-devel \
     && dnf clean dbcache packages

plugins/cmd_images.py

@@ -78,6 +78,9 @@ def __init__(self, osname, script):
                 raise ValueError("Bad script %r, mising trailer" % (script))
         import yaml
         self.spec = yaml.safe_load(b"".join(text))
+        self.git_modules = False
+        if ("git_modules" in self.spec):
+            self.git_modules = True
 
     def _fetch_git(self):
         """Make sure that the required commit ID is available under ~/.cache/ for
@@ -95,17 +98,27 @@ def _fetch_git(self):
             self.git_dir = self.git_dir + ".git"
         if not os.path.exists(self.git_dir):
             try:
-                git_call(["clone", "--bare", git_url, self.git_dir])
+                cmd = ["clone"]
+                if not self.git_modules:
+                    cmd += ["--bare"]
+                git_call(cmd + [git_url, self.git_dir])
             except subprocess.CalledProcessError:
                 shutil.rmtree(self.git_dir)
 
-        with in_directory(self.git_dir):
-            self.git_id = git_commit_id(git_ref, fail_is_none=True)
-            if self.git_id is not None:
-                return
+        if not self.git_modules:
+            with in_directory(self.git_dir):
+                print(self.git_dir)
+                self.git_id = git_commit_id(git_ref, fail_is_none=True)
+                if self.git_id is None:
+                    git_call(["fetch", "--tags", git_url])
+                    self.git_id = git_commit_id(git_ref)
+        else:
+            with in_directory(self.git_dir):
+                git_call(["fetch"])
+                self.git_id = git_commit_id(git_ref)
+                git_call(["checkout", self.git_id ])
+                git_call(["submodule", "update", "--init", "--recursive"])
 
-            git_call(["fetch", "--tags", git_url])
-            self.git_id = git_commit_id(git_ref)
 
     def _fetch_nfs(self):
         """This is very annoying but we need some files from NFS. If the system is on
@@ -163,10 +176,16 @@ def _make_docker_file(self, dockerfn):
 
     def _setup_git(self, dfn):
         with in_directory(self.git_dir):
-            git_call([
-                "archive", "--format=tar", "--prefix", "opt/src/", "-o",
-                self.tarfn, self.git_id
-            ])
+            if not self.git_modules:
+                git_call([
+                    "archive", "--format=tar", "--prefix", "opt/src/", "-o",
+                    self.tarfn, self.git_id
+                ])
+            else:
+                subprocess.check_call([
+                    "tar", "--create", "--mtime=Jan 1 2010", "--file", self.tarfn,
+                    "--transform", "s|^|opt/src/|", "."])
+#                    "--exclude", ".git", "--transform", "s|^|opt/src/|", "."])
 
     def _setup_nfs(self, dfn):
         for nfs_fn, inf in self.spec.get("nfs_files", {}).items():

plugins/cmd_run.py

@@ -233,7 +233,8 @@ def get_pickle(args, vm_addr):
         mem += len(p["simx"])
     if args.virt:
         p["virt"] = sorted(args.virt)
-    p["mem"] = str(mem) + 'G'
+#    p["mem"] = str(mem) + '8G'
+    p["mem"] = '8G'
 
     if args.boot_script:
         p["boot_script"] = args.boot_script
@@ -407,11 +408,13 @@ def cmd_run(args):
         mapdirs.add(os.path.dirname(args.kernel_rpm))
         args.kernel = None
     else:
+        print (args.kernel)
         args.kernel = os.path.realpath(args.kernel)
         if not os.path.isdir(args.kernel):
             raise ValueError("Kernel path %r is not a directory/does not exist"
                              % (args.kernel))
         mapdirs.add(args.kernel)
+        print (mapdirs)
 
     if args.image:
         try:
@@ -449,6 +452,7 @@ def cmd_run(args):
         do_kvm_args = ["/bin/bash"]
     else:
         do_kvm_args = ["python3", "/plugins/do-kvm.py"]
+#        do_kvm_args = ["python3", "/plugins/do-test.py"]
         if vm_addr.ip:
             # Open network for QEMU, relevant for bridged mode only
             iprule = ["FORWARD", "-m", "physdev", "--physdev-is-bridged", "-j", "ACCEPT"]
@@ -483,6 +487,7 @@ def cmd_run(args):
         subprocess.call(["ssh", "root@%s" % (get_host_name(cname))])
     else:
         cname = get_container_name(vm_addr)
+        print (mapdirs.as_docker_bind())
         docker_exec(["run"] + mapdirs.as_docker_bind() + [
             "-v",
             "%s:/plugins:ro,Z" % (src_dir),

plugins/cmd_setup.py

@@ -74,6 +74,7 @@ def cmd_setup(args):
             'ubuntu' : '16',
             'rhel' : '8',
             'redhat' : '8',
+            'centos' : '8',
     }
 
     # Python API stability is dissaster