Restructure RPM Build process

Signed-off-by: Utkarsh Srivastava <[email protected]>

add support for s3select config

Signed-off-by: Utkarsh Srivastava <[email protected]>

add support for srpm only config

Signed-off-by: Utkarsh Srivastava <[email protected]>

replace hardcoded paths with RPM macros

Signed-off-by: Utkarsh Srivastava <[email protected]>

final cleanup

Signed-off-by: Utkarsh Srivastava <[email protected]>

avoid symlinking on host machine

Signed-off-by: Utkarsh Srivastava <[email protected]>

address PR comments

Co-authored-by: Boris Ranto <[email protected]>
Signed-off-by: Utkarsh Srivastava <[email protected]>

address PR comments

Signed-off-by: Utkarsh Srivastava <[email protected]>

address PR comments

Signed-off-by: Utkarsh Srivastava <[email protected]>

revert nasm installation to curl based

Signed-off-by: Utkarsh Srivastava <[email protected]>

fix RPM cleanup error

Signed-off-by: Utkarsh Srivastava <[email protected]>

Author: tangledbytes

Makefile

@@ -165,10 +165,10 @@ nbdev:
 	@echo ""
 .PHONY: nbdev
 
-rpm: base
+rpm: builder
 	echo "\033[1;34mStarting RPM build for $${CONTAINER_PLATFORM}.\033[0m"
 	mkdir -p build/rpm
-	$(CONTAINER_ENGINE) build $(CONTAINER_PLATFORM_FLAG) $(CPUSET) -f src/deploy/RPM_build/RPM.Dockerfile $(CACHE_FLAG) -t $(NOOBAA_RPM_TAG) --build-arg GIT_COMMIT=$(GIT_COMMIT) . $(REDIRECT_STDOUT)
+	$(CONTAINER_ENGINE) build $(CONTAINER_PLATFORM_FLAG) $(CPUSET) -f src/deploy/RPM_build/RPM.Dockerfile $(CACHE_FLAG) -t $(NOOBAA_RPM_TAG) --build-arg CENTOS_VER=$(CENTOS_VER) --build-arg BUILD_S3SELECT=$(BUILD_S3SELECT) --build-arg BUILD_S3SELECT_PARQUET=$(BUILD_S3SELECT_PARQUET) --build-arg SRPM_ONLY=$(SRPM_ONLY) --build-arg GIT_COMMIT=$(GIT_COMMIT) . $(REDIRECT_STDOUT)
 	echo "\033[1;32mImage \"$(NOOBAA_RPM_TAG)\" is ready.\033[0m"
 	echo "Generating RPM..."
 	$(CONTAINER_ENGINE) run --rm -v $(PWD)/build/rpm:/export:z -t $(NOOBAA_RPM_TAG)

src/deploy/RPM_build/RPM.Dockerfile

@@ -1,6 +1,9 @@
-FROM noobaa-base
+FROM noobaa-builder
 ARG TARGETARCH
+
+ARG CENTOS_VER=9
 ARG BUILD_S3SELECT=0
+ARG BUILD_S3SELECT_PARQUET=0
 
 RUN mkdir -p /etc/logrotate.d/noobaa/
 RUN mkdir -p /etc/noobaa.conf.d/
@@ -9,8 +12,14 @@ COPY ./src/agent ./src/agent
 COPY ./src/api ./src/api
 COPY ./src/cmd ./src/cmd
 COPY ./src/deploy/spectrum_archive ./src/deploy/spectrum_archive
+COPY ./src/deploy/noobaa_nsfs.service ./src/deploy/
+COPY ./src/deploy/nsfs_env.env ./src/deploy/
+COPY ./src/deploy/NVA_build/clone_submodule.sh ./src/deploy/NVA_build/
+COPY ./src/deploy/NVA_build/clone_s3select_submodules.sh ./src/deploy/NVA_build/
+COPY ./src/deploy/NVA_build/install_nodejs.sh ./src/deploy/NVA_build/
 COPY ./src/endpoint ./src/endpoint
 COPY ./src/hosted_agents ./src/hosted_agents
+COPY ./src/native ./src/native/
 COPY ./src/rpc ./src/rpc
 COPY ./src/s3 ./src/s3
 COPY ./src/sdk ./src/sdk
@@ -22,17 +31,25 @@ COPY ./config.js ./
 COPY ./platform_restrictions.json ./
 COPY ./Makefile ./
 COPY ./package*.json ./
+COPY ./binding.gyp .
 COPY ./src/deploy/standalone/noobaa_rsyslog.conf ./src/deploy/standalone/noobaa_rsyslog.conf
 COPY ./src/deploy/standalone/noobaa_syslog.conf ./src/deploy/standalone/noobaa_syslog.conf
 COPY ./src/deploy/standalone/logrotate_noobaa.conf ./src/deploy/standalone/logrotate_noobaa.conf
 COPY ./src/manage_nsfs ./src/manage_nsfs
 
 WORKDIR /build
 
-RUN tar -czvf noobaa-core.tar.gz /noobaa
-
 COPY ./src/deploy/RPM_build/* ./
 COPY ./package.json ./
+RUN bash ./preparesrc.sh /noobaa
 RUN chmod +x ./packagerpm.sh
 
+# These envs are used by the packagerpm.sh either directly or by
+# performing subsitutions in the noobaa.spec file
+ARG SRPM_ONLY=false
+
+ENV BUILD_S3SELECT=${BUILD_S3SELECT}
+ENV BUILD_S3SELECT_PARQUET=${BUILD_S3SELECT_PARQUET}
+ENV CENTOS_VER=${CENTOS_VER}
+ENV SRPM_ONLY=${SRPM_ONLY}
 CMD ./packagerpm.sh /export /build

src/deploy/RPM_build/noobaa.spec

@@ -1,62 +1,88 @@
+# Disable debuginfo -- it does not work well with nodejs apps
+%global debug_package %{nil}
+# This speeds up the build a lot and it is not really required
+%global __os_install_post %{nil}
+
 %define revision null
 %define noobaaver null
 %define nodever null
 %define releasedate null
 %define changelogdata null
+%define BUILD_S3SELECT null
+%define BUILD_S3SELECT_PARQUET null
+%define CENTOS_VER null
+%define _build_id_links none
 
 %define noobaatar %{name}-%{version}-%{revision}.tar.gz
-%define nodetar node-%{nodever}.tar.xz
 %define buildroot %{_tmppath}/%{name}-%{version}-%{release}
 
-Name:		noobaa-core
-Version:	%{noobaaver}
-Release:	%{revision}%{?dist}
-Summary:	NooBaa RPM
+Name: noobaa-core
+Version:  %{noobaaver}
+Release:  %{revision}%{?dist}
+Summary:  NooBaa RPM
 
-License:	Apache-2.0
-URL:        https://www.noobaa.io/
-Source0:	%{noobaatar}
-Source1:    %{nodetar}
+License:  Apache-2.0
+URL:  https://www.noobaa.io/
+Source0:  %{noobaatar}
 
-BuildRequires: systemd
+BuildRequires:  systemd
+BuildRequires:  python3
+BuildRequires:  make
+BuildRequires:  gcc-c++
+BuildRequires:  boost-devel
 
-Recommends: jemalloc
-
-%global __os_install_post %{nil}
+Recommends:     jemalloc
 
 %global __requires_exclude ^/usr/bin/python$
 
 %description
 NooBaa is a data service for cloud environments, providing S3 object-store interface with flexible tiering, mirroring, and spread placement policies, over any storage resource that allows GET/PUT including S3, GCS, Azure Blob, Filesystems, etc.
 
 %prep
-mkdir noobaa-core-%{version}-%{revision}
-mkdir node-%{nodever}
-tar -xzf %{SOURCE0} -C noobaa-core-%{version}-%{revision}/
-tar -xJf %{SOURCE1} -C node-%{nodever}/
+%setup -n noobaa -q
+
+%build
+NODEJS_VERSION="%{nodever}"
+SKIP_NODE_INSTALL=1 source src/deploy/NVA_build/install_nodejs.sh $NODEJS_VERSION
+
+mkdir -p ../node/
+
+nodepath=$(download_node)
+tar -xJf ${nodepath} -C ../node/
 
-%clean
-[ ${RPM_BUILD_ROOT} != "/" ] && rm -rf ${RPM_BUILD_ROOT}
+PATH=$PATH:%{_builddir}/node/node-v$NODEJS_VERSION-linux-$(get_arch)/bin
+
+npm install --omit=dev && npm cache clean --force
+
+./src/deploy/NVA_build/clone_s3select_submodules.sh
+
+if [[ "%{CENTOS_VER}" = "8" ]]
+then
+  sed -i 's/\/lib64\/libboost_thread.so.1.75.0/\/lib64\/libboost_thread.so.1.66.0/g' ./src/native/s3select/s3select.gyp
+  echo "Using libboost 1.66 for S3 Select"
+fi
+
+GYP_DEFINES="BUILD_S3SELECT=%{BUILD_S3SELECT} BUILD_S3SELECT_PARQUET=%{BUILD_S3SELECT_PARQUET}" npm run build
 
 %install
 rm -rf $RPM_BUILD_ROOT
-mkdir -p $RPM_BUILD_ROOT/usr/local/
 
-cp -R %{_builddir}/%{name}-%{version}-%{revision}/noobaa $RPM_BUILD_ROOT/usr/local/noobaa-core
-cp -R %{_builddir}/node-%{nodever}/* $RPM_BUILD_ROOT/usr/local/noobaa-core/node
+mkdir -p $RPM_BUILD_ROOT/usr/local/
+cp -R %{_builddir}/noobaa $RPM_BUILD_ROOT/usr/local/noobaa-core
+mv %{_builddir}/node/* $RPM_BUILD_ROOT/usr/local/noobaa-core/node
 
 mkdir -p $RPM_BUILD_ROOT/usr/local/noobaa-core/bin
 ln -s /usr/local/noobaa-core/node/bin/node $RPM_BUILD_ROOT/usr/local/noobaa-core/bin/node
 ln -s /usr/local/noobaa-core/node/bin/npm $RPM_BUILD_ROOT/usr/local/noobaa-core/bin/npm
 ln -s /usr/local/noobaa-core/node/bin/npx $RPM_BUILD_ROOT/usr/local/noobaa-core/bin/npx
 
 mkdir -p $RPM_BUILD_ROOT%{_unitdir}/
-cp -R %{_builddir}/%{name}-%{version}-%{revision}/noobaa/src/deploy/noobaa_nsfs.service $RPM_BUILD_ROOT%{_unitdir}/noobaa_nsfs.service
+mv $RPM_BUILD_ROOT/usr/local/noobaa-core/src/deploy/noobaa_nsfs.service $RPM_BUILD_ROOT%{_unitdir}/noobaa_nsfs.service
 ln -s /usr/local/noobaa-core/src/deploy/nsfs_env.env $RPM_BUILD_ROOT/usr/local/noobaa-core/nsfs_env.env
 mkdir -p $RPM_BUILD_ROOT/etc/noobaa.conf.d/
 
 mkdir -p $RPM_BUILD_ROOT/etc/rsyslog.d/
-cp -R $RPM_BUILD_ROOT/usr/local/noobaa-core/src/deploy/standalone/noobaa_syslog.conf $RPM_BUILD_ROOT/etc/rsyslog.d/noobaa_syslog.conf
+mv $RPM_BUILD_ROOT/usr/local/noobaa-core/src/deploy/standalone/noobaa_syslog.conf $RPM_BUILD_ROOT/etc/rsyslog.d/noobaa_syslog.conf
 
 mkdir -p $RPM_BUILD_ROOT/etc/logrotate.d/noobaa
 ln -s /usr/local/noobaa-core/src/deploy/standalone/logrotate_noobaa.conf $RPM_BUILD_ROOT/etc/logrotate.d/noobaa/logrotate_noobaa.conf

src/deploy/RPM_build/packagerpm.sh

@@ -10,8 +10,8 @@ NODE_PATH="${NODE_PATH:-/usr/local/node}"
 
 noobaaver="$(npm pkg get version | tr -d '"')"
 releasedate=$(date '+%a %b %d %Y')
-revision="$(date -u '+%Y%m%d')"
 nodever=$(node --version | tr -d 'v')
+revision="$(date -u '+%Y%m%d')"
 changelogdata="Initial release of NooBaa ${noobaaver}"
 ARCHITECTURE=$(uname -m)
 
@@ -33,20 +33,6 @@ function move_builds() {
     cp ${TAR_DIR}/noobaa-core-${noobaaver}-${revision}.tar.gz ~/rpmbuild/SOURCES/
 }
 
-function get_node_tar() {
-    local arch=$(get_arch)
-    local filename="node-v${nodever}-linux-${arch}.tar.xz"
-
-    pushd ~/rpmbuild/SOURCES/
-    if [ -f ${NODE_PATH}/${filename} ]
-    then
-        mv ${NODE_PATH}/${filename} node-${nodever}.tar.xz
-    else 
-        local path=$(NODEJS_VERSION=${nodever} download_node)
-        mv ${path} node-${nodever}.tar.xz
-    fi
-    popd
-}
 
 function generate_spec_from_template() {
     while IFS= read -r line; do
@@ -73,9 +59,6 @@ mkdir -p ~/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
 # Move the binary into the SOURCES directory
 move_builds
 
-# Download nodejs tarball to SOURCES directory
-get_node_tar
-
 # Set the changelog
 set_changelog
 
@@ -89,8 +72,14 @@ generate_spec_from_template
 mv ${OUTPUT_FILE} ~/rpmbuild/SPECS/
 
 # Build the RPM package
-rpmbuild -ba ~/rpmbuild/SPECS/noobaa.final.spec
+if [[ "$SRPM_ONLY" = "true" ]]; then
+    rpmbuild -bs ~/rpmbuild/SPECS/noobaa.final.spec
+else
+    rpmbuild -ba ~/rpmbuild/SPECS/noobaa.final.spec
+
+    # Move the RPM package to the target directory
+    mv ~/rpmbuild/RPMS/${ARCHITECTURE}/noobaa-core-${noobaaver}-${revision}.*.rpm ${TARGET_DIR}
+fi
 
-# Move the RPM and SRPM package to the target directory
-mv ~/rpmbuild/RPMS/${ARCHITECTURE}/noobaa-core-${noobaaver}-${revision}.*.rpm ${TARGET_DIR}
+# Move the SRPM package to the target directory
 mv ~/rpmbuild/SRPMS/noobaa-core-${noobaaver}-${revision}.*.rpm ${TARGET_DIR}

src/deploy/RPM_build/preparesrc.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -eou pipefail
+set -x
+
+TARGET="noobaa-core.tar.gz"
+
+SRC_DIR=$(realpath "${1:-$(pwd)}")
+
+function prepare_excludes() {
+	local exclude_flag_gitignore="$(grep -v '^#' $SRC_DIR/.gitignore | grep '[^[:blank:]]' | xargs -I{} printf '%s=%s ' '--exclude' '{}')"
+	local exclude_flag_local="--exclude ./build --exclude ./images --exclude ./tools --exclude ./submodules --exclude .github --exclude .travis --exclude .jenkins"
+
+	echo "${exclude_flag_gitignore} ${exclude_flag_local}"
+}
+
+function archive() {
+	local tar="$1"
+	local exclude_flag="$(prepare_excludes)"
+
+	# No quotes here is intentional
+	tar $exclude_flag --exclude-vcs -czvf "$tar" "$SRC_DIR"
+}
+
+echo "Using source: $SRC_DIR"
+if [ "$SRC_DIR" = "$(pwd)" ]; then
+	pushd ..
+	tpath="$TMPDIR$TARGET"
+	archive "$tpath"
+	popd
+
+	mv "$tpath" .
+else
+	tpath="./$TARGET"
+	archive "$tpath"
+fi