Add file integrity checks before service installation in dw-quickstart (

#2331) * Change dist URIs to https & introduce tarball integrity checks * Add already installed checks & update NiFi dist URI & remove unneeded code * add missing comment * change checksums based on feedback --------- Co-authored-by: Ivan Bella <[email protected]> Co-authored-by: Keith Ratcliffe <[email protected]>
NationalSecurityAgency · Apr 10, 2024 · 387b93b · 387b93b
1 parent 6730346
commit 387b93b
Show file tree

Hide file tree

Showing 9 changed files with 59 additions and 6 deletions.
diff --git a/contrib/datawave-quickstart/bin/common.sh b/contrib/datawave-quickstart/bin/common.sh
@@ -80,6 +80,24 @@ function askYesNo() {
    done
 }
 
+function verifyChecksum() {
+  # $1 - distribution URI
+  # $2 - service directory
+  # $3 - tarball sha512 checksum
+  local tarballName="$(basename "$1")"
+  if [[ -f "$2/${tarballName}" ]]; then
+      local calculatedChecksum="$( cd $2 && sha512sum ${tarballName} )"
+      if [[ "${calculatedChecksum}" = "$3  ${tarballName}" ]] ; then
+          info "Checksum verification success... [${tarballName}]"
+      else
+          error "------------------------------------------------------------------------"
+          error "$(printRed "CHECKSUM MISMATCH") - Could not verify integrity of: ${tarballName}"
+          error "------------------------------------------------------------------------"
+          kill -INT $$
+      fi
+  fi
+}
+
 function downloadTarball() {
    # Downloads the specified tarball, if it doesn't already exist.
    # If you want to utilize a tarball from the local file system, simply use
@@ -90,7 +108,13 @@ function downloadTarball() {
    if [ ! -f "${tarballdir}/${tarball}" ] ; then
       if [[ ${uri} == file://* ]] ; then
           $( cd "${tarballdir}" && cp  "${uri:7}" ./${tarball} ) || error "File copy failed for ${uri:7}"
-      else
+      elif [[ ${uri} == http://* ]] ; then
+          if ! askYesNo "Are you sure you want to download ${tarball} using HTTP? $( printRed "This can potentially be insecure." )" ; then
+            kill -INT $$
+          else
+            $( cd "${tarballdir}" && wget ${DW_WGET_OPTS} "${uri}" )
+          fi
+      elif [[ ${uri} == https://* ]] ; then
           $( cd "${tarballdir}" && wget ${DW_WGET_OPTS} "${uri}" )
       fi
    fi

diff --git a/contrib/datawave-quickstart/bin/services/accumulo/bootstrap.sh b/contrib/datawave-quickstart/bin/services/accumulo/bootstrap.sh
@@ -15,7 +15,10 @@ DW_ACCUMULO_SERVICE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 # Zookeeper config
 
 # You may override DW_ZOOKEEPER_DIST_URI in your env ahead of time, and set as file:///path/to/file.tar.gz for local tarball, if needed
+# DW_ZOOKEEPER_DIST_URI should, if possible, be using https. There are potential security risks by using http.
 DW_ZOOKEEPER_DIST_URI="${DW_ZOOKEEPER_DIST_URI:-https://archive.apache.org/dist/zookeeper/zookeeper-3.7.1/apache-zookeeper-3.7.1-bin.tar.gz}"
+# The sha512 checksum for the tarball. Value should be the hash value only and does not include the file name. Cannot be left blank.
+DW_ZOOKEEPER_DIST_SHA512_CHECKSUM="${DW_ZOOKEEPER_DIST_SHA512_CHECKSUM:-9103628a50745fa1a289bca666fda4a9c08ec17c55cf13e66887e7ba76e93dbae60a1f1ffd6c10798be3a16069344ecbc00cebb29bf03d9cd7096ccd098ed011}"
 # shellcheck disable=SC2154
 # shellcheck disable=SC2034
 DW_ZOOKEEPER_DIST="$( downloadTarball "${DW_ZOOKEEPER_DIST_URI}" "${DW_ACCUMULO_SERVICE_DIR}" && echo "${tarball}" )"
@@ -45,7 +48,10 @@ admin.enableServer=false"
 # Accumulo config
 
 # You may override DW_ACCUMULO_DIST_URI in your env ahead of time, and set as file:///path/to/file.tar.gz for local tarball, if needed
-DW_ACCUMULO_DIST_URI="${DW_ACCUMULO_DIST_URI:-http://archive.apache.org/dist/accumulo/2.1.1/accumulo-2.1.1-bin.tar.gz}"
+# DW_ACCUMULO_DIST_URI should, if possible, be using https. There are potential security risks by using http.
+DW_ACCUMULO_DIST_URI="${DW_ACCUMULO_DIST_URI:-https://archive.apache.org/dist/accumulo/2.1.1/accumulo-2.1.1-bin.tar.gz}"
+# The sha512 checksum for the tarball. Value should be the hash value only and does not include the file name. Cannot be left blank.
+DW_ACCUMULO_DIST_SHA512_CHECKSUM="${DW_ACCUMULO_DIST_SHA512_CHECKSUM:-adb23e56362c2e3e813d07791389b8ca2d5976df8b00a29b607e6ae05ea465eff80ada6d1ec9a9c596df8b4066c51078cd5a4006dc78568ac38f638a1d3895be}"
 # shellcheck disable=SC2034
 DW_ACCUMULO_DIST="$( downloadTarball "${DW_ACCUMULO_DIST_URI}" "${DW_ACCUMULO_SERVICE_DIR}" && echo "${tarball}" )"
 DW_ACCUMULO_BASEDIR="accumulo-install"
@@ -242,7 +248,7 @@ function accumuloUninstall() {
 }
 
 function accumuloInstall() {
-    "${DW_ACCUMULO_SERVICE_DIR}/install.sh"
+  "${DW_ACCUMULO_SERVICE_DIR}/install.sh"
 }
 
 function zookeeperIsInstalled() {

diff --git a/contrib/datawave-quickstart/bin/services/accumulo/install.sh b/contrib/datawave-quickstart/bin/services/accumulo/install.sh
@@ -14,6 +14,11 @@ source "${SERVICES_DIR}/hadoop/bootstrap.sh"
 
 hadoopIsInstalled || fatal "Accumulo requires that Hadoop be installed"
 
+# If Accumulo is not installed, verify that the two checksums match before installing.
+accumuloIsInstalled || verifyChecksum "${DW_ACCUMULO_DIST_URI}" "${DW_ACCUMULO_SERVICE_DIR}" "${DW_ACCUMULO_DIST_SHA512_CHECKSUM}"
+# If Zookeeper is not installed, verify that the two checksums match before installing.
+zookeeperIsInstalled || verifyChecksum "${DW_ZOOKEEPER_DIST_URI}" "${DW_ACCUMULO_SERVICE_DIR}" "${DW_ZOOKEEPER_DIST_SHA512_CHECKSUM}"
+
 if zookeeperIsInstalled ; then
    info "ZooKeeper is already installed"
 else

diff --git a/contrib/datawave-quickstart/bin/services/datawave/bootstrap-web.sh b/contrib/datawave-quickstart/bin/services/datawave/bootstrap-web.sh
@@ -1,6 +1,9 @@
 
 # You may override DW_WILDFLY_DIST_URI in your env ahead of time, and set as file:///path/to/file.tar.gz for local tarball, if needed
+# DW_WILDFLY_DIST_URI should, if possible, be using https. There are potential security risks by using http.
 DW_WILDFLY_DIST_URI="${DW_WILDFLY_DIST_URI:-https://download.jboss.org/wildfly/17.0.1.Final/wildfly-17.0.1.Final.tar.gz}"
+# The sha512 checksum for the tarball. Value should be the hash value only and does not include the file name. Cannot be left blank.
+DW_WILDFLY_DIST_SHA512_CHECKSUM="${DW_WILDFLY_DIST_SHA512_CHECKSUM:-fcbdff4bc275f478c3bf5f665a83e62468a920e58fcddeaa2710272dd0f1ce3154cdc371d5011763a6be24ae1a5e0bca0218cceea63543edb4b5cf22de60b485}"
 DW_WILDFLY_DIST="$( downloadTarball "${DW_WILDFLY_DIST_URI}" "${DW_DATAWAVE_SERVICE_DIR}" && echo "${tarball}" )"
 DW_WILDFLY_BASEDIR="wildfly-install"
 DW_WILDFLY_SYMLINK="wildfly"

diff --git a/contrib/datawave-quickstart/bin/services/datawave/install-web.sh b/contrib/datawave-quickstart/bin/services/datawave/install-web.sh
@@ -10,6 +10,9 @@ source "${THIS_DIR}/bootstrap.sh"
 source "${SERVICES_DIR}/hadoop/bootstrap.sh"
 source "${SERVICES_DIR}/accumulo/bootstrap.sh"
 
+# If Wildfly is not installed, verify that the two checksums match before installing.
+datawaveWebIsInstalled || verifyChecksum "${DW_WILDFLY_DIST_URI}" "${DW_DATAWAVE_SERVICE_DIR}" "${DW_WILDFLY_DIST_SHA512_CHECKSUM}"
+
 accumuloIsInstalled || fatal "DataWave Web requires that Accumulo be installed"
 
 datawaveWebIsInstalled && info "DataWave Web is already installed" && exit 1

diff --git a/contrib/datawave-quickstart/bin/services/hadoop/bootstrap.sh b/contrib/datawave-quickstart/bin/services/hadoop/bootstrap.sh
@@ -3,7 +3,10 @@
 DW_HADOOP_SERVICE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 # You may override DW_HADOOP_DIST_URI in your env ahead of time, and set as file:///path/to/file.tar.gz for local tarball, if needed
-DW_HADOOP_DIST_URI="${DW_HADOOP_DIST_URI:-http://archive.apache.org/dist/hadoop/common/hadoop-3.3.4/hadoop-3.3.4.tar.gz}"
+# DW_HADOOP_DIST_URI should, if possible, be using https. There are potential security risks by using http.
+DW_HADOOP_DIST_URI="${DW_HADOOP_DIST_URI:-https://archive.apache.org/dist/hadoop/common/hadoop-3.3.4/hadoop-3.3.4.tar.gz}"
+# The sha512 checksum for the tarball. Value should be the hash value only and does not include the file name. Cannot be left blank.
+DW_HADOOP_DIST_SHA512_CHECKSUM="${DW_HADOOP_DIST_SHA512_CHECKSUM:-ca5e12625679ca95b8fd7bb7babc2a8dcb2605979b901df9ad137178718821097b67555115fafc6dbf6bb32b61864ccb6786dbc555e589694a22bf69147780b4}"
 DW_HADOOP_DIST="$( downloadTarball "${DW_HADOOP_DIST_URI}" "${DW_HADOOP_SERVICE_DIR}" && echo "${tarball}" )"
 DW_HADOOP_BASEDIR="hadoop-install"
 DW_HADOOP_SYMLINK="hadoop"
@@ -202,7 +205,7 @@ function hadoopUninstall() {
 }
 
 function hadoopInstall() {
-   "${DW_HADOOP_SERVICE_DIR}"/install.sh
+  "${DW_HADOOP_SERVICE_DIR}"/install.sh
 }
 
 function hadoopPrintenv() {

diff --git a/contrib/datawave-quickstart/bin/services/hadoop/install.sh b/contrib/datawave-quickstart/bin/services/hadoop/install.sh
@@ -36,6 +36,9 @@ Generate the password-less ssh key now?"
 source "${BIN_DIR}/env.sh"
 source "${THIS_DIR}/bootstrap.sh"
 
+# If Hadoop is not installed, verify that the two checksums match before installing.
+hadoopIsInstalled || verifyChecksum "${DW_HADOOP_DIST_URI}" "${DW_HADOOP_SERVICE_DIR}" "${DW_HADOOP_DIST_SHA512_CHECKSUM}"
+
 hadoopIsInstalled && info "Hadoop is already installed" && exit 1
 
 [ -f "${DW_HADOOP_SERVICE_DIR}/${DW_HADOOP_DIST}" ] || fatal "Hadoop tarball not found"

diff --git a/contrib/datawave-quickstart/bin/services/nifi/bootstrap.sh b/contrib/datawave-quickstart/bin/services/nifi/bootstrap.sh
@@ -3,7 +3,10 @@
 DW_NIFI_SERVICE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 # You may override DW_NIFI_DIST_URI in your env ahead of time, and set as file:///path/to/file.tar.gz for local tarball, if needed
-DW_NIFI_DIST_URI="${DW_NIFI_DIST_URI:-http://apache.claz.org/nifi/1.1.1/nifi-1.1.1-bin.tar.gz}"
+# DW_NIFI_DIST_URI should, if possible, be using https. There are potential security risks by using http.
+DW_NIFI_DIST_URI="${DW_NIFI_DIST_URI:-https://archive.apache.org/dist/nifi/1.1.1/nifi-1.1.1-bin.tar.gz}"
+# The sha512 checksum for the tarball. Value should be the hash value only and does not include the file name. Cannot be left blank.
+DW_NIFI_DIST_SHA512_CHECKSUM="${DW_NIFI_DIST_SHA512_CHECKSUM:-2d37810985bda230180aac82f9fcf7a23a2e7a2257cf8b3c31d18280a2c67ba165061ba801389d6bb12cdb261d4a9aeef29e931e434144bc3ee5a3d6cfd6cdfa}"
 DW_NIFI_DIST="$( downloadTarball "${DW_NIFI_DIST_URI}" "${DW_NIFI_SERVICE_DIR}" && echo "${tarball}" )"
 DW_NIFI_BASEDIR="nifi-install"
 DW_NIFI_SYMLINK="nifi"

diff --git a/contrib/datawave-quickstart/bin/services/nifi/install.sh b/contrib/datawave-quickstart/bin/services/nifi/install.sh
@@ -8,6 +8,9 @@ BIN_DIR="$( dirname "${SERVICES_DIR}" )"
 source "${BIN_DIR}/env.sh"
 source "${THIS_DIR}/bootstrap.sh"
 
+# If NiFi is not installed, verify that the two checksums match before installing.
+nifiIsInstalled || verifyChecksum "${DW_NIFI_DIST_URI}" "${DW_NIFI_SERVICE_DIR}" "${DW_NIFI_DIST_SHA512_CHECKSUM}"
+
 nifiIsInstalled && info "NiFi is already installed" && exit 1
 
 [ ! -f "${DW_NIFI_SERVICE_DIR}/${DW_NIFI_DIST}" ] && fatal "NiFi tarball not found"