added scripts and wekatester

install/scripts.d/ta/195_ofed.sh

@@ -2,23 +2,24 @@
 
 DESCRIPTION="Check if Mellanox OFED is installed"
 SCRIPT_TYPE="parallel"
+RETURN_CODE=0
 
 # fail immediately if no OFED installed
 if ! ofed_info -n &> /dev/null; then
-    echo "OFED not installed"
+    echo "WARN: OFED not installed"
     exit 254
 fi
 
 # is it a supported OFED version?
 OFEDVER=$(ofed_info -n)
 
-case "$OFEDVER" in 
+case "$OFEDVER" in
     5.1-2.5.8.0 | 5.1-2.6.2.0 | 5.4-3.4.0.0 | 5.4-3.5.8.0 | 5.6-1.0.3.3 | 5.6-2.0.9.0 | 5.7-1.0.2.0 | 5.8-1.1.2.1 | 5.8-3.0.7.0 | 5.9-0.5.6.0 | 23.04-1.1.3.0 | 23.10-0.5.5.0 )
         #continue
         ;;
     *)
-        echo "Unsupported OFED version $OFEDVER"
-        exit 254
+        echo "WARN: Unsupported OFED version $OFEDVER"
+        RETURN_CODE=254
         ;;
 esac
 
@@ -28,17 +29,20 @@ esac
 if modinfo mlx5_core &> /dev/null; then
     MLX5_VER=$(modinfo mlx5_core | awk '/^version:/{ print $2 }')
 else
-    echo "No mlx5_core kernel module loaded"
-    exit 254
+    echo "WARN: No mlx5_core kernel module loaded"
+    RETURN_CODE=254
 fi
 
 # make sure loaded drivers match the installed OFED
 if [[ -n "$MLX5_VER" ]]; then
     if [[ "$MLX5_VER" != "${OFEDVER:0:9}" ]]; then
-        echo "Loaded Mellanox driver $MLX5_VER does not match OFED version $OFEDVER!"
-        exit 254
+        echo "WARN: Loaded Mellanox driver $MLX5_VER does not match OFED version $OFEDVER!"
+        RETURN_CODE=254
     fi
 fi
 
-echo "Valid OFED configuration observed"
-exit 0
+if [[ ${RETURN_CODE} -eq 0 ]]; then
+  echo "Valid OFED configuration observed"
+fi
+
+exit ${RETURN_CODE}

install/scripts.d/ta/270_weka_local_resources_gateways.sh

@@ -8,34 +8,42 @@ SCRIPT_TYPE="parallel"
 
 RETURN_CODE=0
 
-for WEKA_CONTAINER in $(sudo weka local ps --output name --no-header); do
-    if [[ ( ${WEKA_CONTAINER} == "ganesha" ) || \
-          ( ${WEKA_CONTAINER} == "samba" )   || \
-          ( ${WEKA_CONTAINER} == "smb"   ) ]] ; then
-        continue
-    fi
-    # Look for network devices with no gateway
-    RESOURCES=$(sudo weka local resources --container ${WEKA_CONTAINER} --json)
-    NUMBER_OF_DEVICES_WITH_NO_GATEWAY=$(echo ${RESOURCES} | python3 -c 'import sys, json; data = json.load(sys.stdin); print(len([device for device in data["net_devices"] if device["gateway"] == ""]))')
-
-
-    if [[ ${NUMBER_OF_DEVICES_WITH_NO_GATEWAY} -ne 0 ]] ; then
-        DEVICES_WITH_NO_GATEWAY=$(echo ${RESOURCES} | python3 -c 'import sys, json; data = json.load(sys.stdin); print("\n".join([device["name"] for device in data["net_devices"] if device["gateway"] == ""]))')
-        echo "The container ${WEKA_CONTAINER} has the following network devices defined without an IP"
-        echo "gateway - this might not be a mistake but means Weka POSIX traffic will not"
-        echo "leave this subnet"
-        echo ""
-        echo ${DEVICES_WITH_NO_GATEWAY}
-        echo ""
+for WEKA_CONTAINER in $(sudo weka local ps --output name --no-header | grep -vw -e envoy -e ganesha -e samba -e smbw -e s3); do
+    DEVICES_WITH_NO_GATEWAY=""
+    NET_DEVICE=""
+    NET_GATEWAY=""
+
+    while read NET_ENTRY; do
+        if [[ ${NET_ENTRY} =~ "gateway:"(.*)"name:"(.*) ]]; then
+            NET_GATEWAY=${BASH_REMATCH[1]}
+            NET_DEVICE=${BASH_REMATCH[2]}
+        fi
+
+        if [[ -n ${NET_DEVICE} ]]; then
+            if [[ -d /sys/class/net/${NET_DEVICE} ]]; then
+                NET_TYPE=$(cat /sys/class/net/${NET_DEVICE}/type)
+                if [[ -n ${NET_TYPE} && ${NET_TYPE} == "1" ]]; then # Only check ethernet devices
+                    if [[ -z ${NET_GATEWAY} ]]; then
+                        DEVICES_WITH_NO_GATEWAY="${DEVICES_WITH_NO_GATEWAY}${NET_DEVICE} "
+                    fi
+                fi
+            fi
+        fi
+    done < <(weka local resources -C ${WEKA_CONTAINER} net --stable -J | grep -w -e gateway -e name | paste - - | tr -d \"\,[:blank:])
+
+    if [[ -n ${DEVICES_WITH_NO_GATEWAY} ]]; then
+        echo "The container ${WEKA_CONTAINER} has the network device(s) ${DEVICES_WITH_NO_GATEWAY}"
+        echo "defined without an IP gateway - this might not be a mistake but means Weka"
+        echo "POSIX traffic will not leave this subnet."
         echo "The likely fix for this is to do weka local resources net remove for each device,"
         echo "then add back in with weka local resource net add <DEVNAME> --gateway ... --netmask .."
-        #RETURN_CODE=254
-        exit 254
-    fi 
+        echo
+        RETURN_CODE=254
+    fi
 done
+
 if [[ ${RETURN_CODE} -eq 0 ]]; then
     echo "All Weka containers have network devices with gateways"
 fi
-exit ${RETURN_CODE}
-
 
+exit ${RETURN_CODE}

install/scripts.d/ta/290_check_traces_free_space.sh

@@ -52,7 +52,10 @@ if (( ${WEKA_ENSURE_FREE} > ${TRACES_FS_SIZE})) ; then
     echo "Weka is currently set to ensure that ${WEKA_ENSURE_FREE} bytes are free"
     echo "on ${WEKA_TRACES_DIR}, but this filesystem is only ${TRACES_FS_SIZE} bytes" 
     echo "in size. These conditions cannot co-exist, so the outcome is that no"
-    echo "traces will be stored"
+    echo "traces will be stored."
+    echo "Recommended options:"
+    echo "   . Increase the size of ${WEKA_TRACES_DIR}"
+    echo "   . Reduce the size of traces with \"weka debug traces retention set --server-ensure-free XXXX\""
     RETURN_CODE=1
 fi 
 if [[ ${RETURN_CODE} -eq 0 ]]; then

install/scripts.d/ta/390_data_folder.sh

@@ -29,6 +29,7 @@ if [ -d "/data" ] ; then
         else
             echo "to ${JIRA_REFERENCE}, SFDC ${KB_REFERENCE}"                                                                                                   
         fi
+        echo "The recommend fix is to upgrade your version of Weka"
         RETURN_CODE=1
     fi
 fi

install/scripts.d/ta/400_s3_using_etcd.sh

@@ -40,12 +40,14 @@ if [ ${WEKA_S3_RUNNING} -ge 1 ] ; then
     if verlte ${MIN_VERSION} ${WEKA_VERSION} && verlte ${WEKA_VERSION} ${MAX_VERSION} ; then
         WEKA_ETCD_HOSTS=$(weka s3 cluster --json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(len(data["etcd_cluster_hosts"]))')
         if [ ${WEKA_ETCD_HOSTS} -gt 0 ] ; then
-            echo "S3 cluster is running, and this version of Weka requires migration"
+            echo "S3 cluster is running, and this version of Weka requires a configuration change."
             if [[ ! -z "${WTA_REFERENCE}" ]]; then
-                echo "to ${JIRA_REFERENCE}, discussed in ${WTA_REFERENCE}, SFDC ${KB_REFERENCE}"
+                echo "Refer to ${JIRA_REFERENCE}, discussed in ${WTA_REFERENCE}, SFDC ${KB_REFERENCE}"
             else
-                echo "to ${JIRA_REFERENCE}, SFDC ${KB_REFERENCE}"                                                                                                   
+                echo "Refer to ${JIRA_REFERENCE}, SFDC ${KB_REFERENCE}"                                                                                                   
             fi
+            echo "If you require the S3 service, please contact Customer Success indicating"
+            echo " you need to move the S3 service from ETCD to KWAS, as indicated in KB 1181"
             RETURN_CODE=254
         fi
     fi

install/scripts.d/ta/420_check_cross-numa_zone_memory_balance.sh

@@ -37,6 +37,8 @@ if [[ ${RATIO_SEEN} -gt ${MAX_ALLOWED_RATIO} ]]; then
     echo "from starting due to lack of NUMA zone-local memory"
     echo "The ratio is ${RATIO_SEEN}% and the maximum allowed ratio is ${MAX_ALLOWED_RATIO}%"
     echo "The memory in the highest zone is ${MAX_MEMORY_SEEN_KB} and in the lowest zone is ${MIN_MEMORY_SEEN_KB}"
+    echo "One recommend resolution is to balance the memory between NUMA zones by physically"
+    echo "moving memory, or by adding more to the smaller NUMA zone"
 fi
 
 if [[ ${RETURN_CODE} -eq 0 ]]; then

install/scripts.d/ta/430_nvme_used_capacity_vs_maximum.sh

@@ -27,6 +27,13 @@ echo ${WEKA_SSD_USED_BYTES}
 
 # if we've allocated more than half the maximum theoretical SSD space, warn
 if [[ $((${WEKA_SSD_USED_BYTES}*2)) -gt ${WEKA_THEORETICAL_MAX_SSD_BYTES} ]] ; then 
+    echo "You have used a significant proportion of the theoretical maximum"
+    echo "NVME capacity of the cluster which is decided at first install time."
+    echo "Please contact customer success to discuss options. Possible actions include:"
+    echo " . Adding an Object Store to expand data storage while keeping NVME capacity down"
+    echo " . In-place cluster resizing and migration (perhaps via snap2obj for fast backup/restore)"
+    echo " . Migrating to a different, larger cluster"
+    echo " . Pruning unnecessary data"
     RETURN_CODE=254
 fi
 if [[ ${RETURN_CODE} -eq 0 ]]; then

install/scripts.d/ta/440_hostnames_rfc952.sh

@@ -23,6 +23,7 @@ GREP_RESULT=$(echo ${SHORT_HOSTNAME} | grep "[^-a-z0-9.]")
 if [[ $? -eq 0 ]]; then
     echo "The hostname ${SHORT_HOSTNAME} appears to contain a character other than [a-z], -, and [0-9]."
     echo "Refer to RFC 952 for more information"
+    echo "Recommended resolution: change the hostname to include only alphanumerics and underscores"
     RETURN_CODE=254
 fi
 

install/scripts.d/ta/450_custom_ca_certs.sh

@@ -15,6 +15,8 @@ grep -q SSL_CERT_FILE /opt/weka/dist/release/${WEKA_VERSION}.spec 2>/dev/null
 
 if [[ $? -eq 0 ]] ; then 
         echo "This version of weka appears to use custom CA certificates. Care will be needed for upgrading"
+        echo "Recommended resolution: remove custom CA specification, and upgrade to a more recent"
+        echo "version that natively supports additional CA bundles"
         RETURN_CODE=254
 fi
 if [[ ${RETURN_CODE} -eq 0 ]]; then

install/scripts.d/ta/460_ip_source-based_routing.sh

@@ -78,6 +78,9 @@ if [[ ${SOURCE_BASED_ROUTING_RECOMMENDED} -ge "1" ]] ; then
         echo "Warning: Not every interface appears to have arp_filter=1 set. This could lead to communication problems"
         RETURN_CODE="254"
     fi
+    echo "Recommended resolution: Although networking is typically site- and hardware-dependent,"
+    echo " some example configurations for the common dual NIC setup are noted on the WEKA"
+    echo " documentation site: https://docs.weka.io/planning-and-installation/bare-metal/setting-up-the-hosts#configure-the-ha-networking"
 fi
 
 if [[ ${RETURN_CODE} -eq 0 ]]; then

install/scripts.d/ta/470_number_of_numa_domains.sh

@@ -36,6 +36,8 @@ echo -n "Detected $NUMBER_OF_NUMA_DOMAINS NUMA domains - "
 if [[ $NUMBER_OF_NUMA_DOMAINS -gt $MAXIMUM_NUMA_DOMAINS ]]; then
     RETURN_CODE=254
     echo "Weka currenty only supports a maximum of 32 NUMA domains (4.2.11+)."
+    echo " Recommended resolution: reduce the number of NUMA domains, perhaps by reducing"
+    echo " the NUMAs per socket setting in the machine's firmware"
 
 # 8 or fewer NUMAs is always supported
 elif [[ $NUMBER_OF_NUMA_DOMAINS -le 8 ]]; then
@@ -45,16 +47,25 @@ elif [[ $NUMBER_OF_NUMA_DOMAINS -le 8 ]]; then
 elif vergte $WEKA_VERSION "4.3.0" && verlt $WEKA_VERSION "4.3.2" && [[ $NUMBER_OF_NUMA_DOMAINS -gt 16 ]]; then
     RETURN_CODE=254
     echo "Weka only supports more than 16 NUMA domains in 4.3.2 and higher."
+    echo " Recommended resolutions: either"
+    echo "  . Reduce the number of NUMA domains, perhaps by reducing the NUMAs per socket setting in the machine's firmware"
+    echo "  . Upgrade Weka to a more recent version"
 
 # More than 16 NUMAs only supported in 4.2.11+
 elif vergt $WEKA_VERSION "4.2.6" && verlt $WEKA_VERSION "4.2.11" && [[ $NUMBER_OF_NUMA_DOMAINS -gt 16 ]]; then
     RETURN_CODE=254
     echo "Weka only supports more than 16 NUMA domains in (4.2.11+, 4.3.2+)."
+    echo " Recommended resolutions: either"
+    echo "  . Reduce the number of NUMA domains, perhaps by reducing the NUMAs per socket setting in the machine's firmware"
+    echo "  . Upgrade Weka to a more recent version"
 
 # More than 8 NUMAs only supported in 4.2.7+
 elif verlt $WEKA_VERSION "4.2.7" && [[ $NUMBER_OF_NUMA_DOMAINS -gt 8 ]]; then
     RETURN_CODE=254
     echo "Weka only supports more than 8 NUMA domains in 4.2.7 and higher."
+    echo " Recommended resolutions: either"
+    echo "  . Reduce the number of NUMA domains, perhaps by reducing the NUMAs per socket setting in the machine's firmware"
+    echo "  . Upgrade Weka to a more recent version"
 else
     echo "Number of NUMA domains is within supported limits."
 fi

install/scripts.d/ta/480_check_weka_agent.sh

@@ -17,11 +17,15 @@ if [[ $? -ne "0" ]] ; then
     RETURN_CODE=254
     echo "The service weka-agent is not reported as enabled by systemd"
     echo "This may cause weka to fail to start"
+    echo " Recommended Resolution: enable the service with systemctl enable weka-agent"
 
-
     if [[ ! -L /etc/init.d ]]; then
         echo "/etc/init.d is expected to be a symlink to /etc/rc.d/init.d"
         echo "Without this systemd is unable to find and thus start the weka-agent sysV init script"
+        echo " Recommended Resolution: on RHEL-based OSes move any scripts to /etc/rc.d/init.d, remove"
+        echo " the /etc/init.d directory, and re-create it as a link. The following commands are"
+        echo " one way to achieve this"
+        echo " mv /etc/init.d/* /etc/rc.d/init.d/ && rmdir /etc/init.d && ln -s /etc/rc.d/init.d /etc/init.d"
     fi
 fi
 

install/scripts.d/ta/490_ip_route_metrics.sh

@@ -46,6 +46,8 @@ if [[ ${NUMBER_OF_OVERLAPPING_ROUTES_WITH_METRICS} -gt "1" ]]; then
     echo "that these entries will negatively affect the performance of e.g. floating IP"
     echo "addresses. In any case it is unlikely that preferential IP routes are of"
     echo "benefit in a high-performance local network"
+    echo "Recommended Resolution: review the output of \"ip route\" and rationalize the routes,"
+    echo " likely by removing or coalescing the overlapping routes into larger ranges"
 fi
 
 if [[ ${RETURN_CODE} -eq 0 ]]; then

install/scripts.d/ta/500_sysctl_rp_filter.sh

@@ -35,16 +35,18 @@ if [[ $RP_FILTER_VALUE_ALL != "2" ]]; then
       echo "The value for net.ipv4.conf.${INTERFACE}.rp_filter is set to ${RP_FILTER_VALUE}."
       echo "This can disrupt floating IP addresses for protocols."
       echo "It is recommended to set net.ipv4.conf.${INTERFACE}.rp_filter to 2."
+      echo "Recommended resolution: set this value in e.g. /etc/sysctl.d/99-weka-nics.conf"
     elif [[ $RP_FILTER_VALUE_ALL == "1" && $RP_FILTER_VALUE == "0" ]]; then
       RETURN_CODE="254"
       echo "The value for net.ipv4.conf.${INTERFACE}.rp_filter is set to ${RP_FILTER_VALUE}."
       echo "The value for net.ipv4.conf.all.rp_filter is set to ${RP_FILTER_VALUE_ALL} and takes precedence."
       echo "This can disrupt floating IP addresses for protocols."
       echo "It is recommended to set net.ipv4.conf.${INTERFACE}.rp_filter or net.ipv4.conf.all.rp_filter to 2."
+      echo "Recommended resolution: set this value in e.g. /etc/sysctl.d/99-weka-nics.conf"
     fi
   done
 else
   echo "net.ipv4.conf.all.rp_filter is set to 2, no further testing necessary."
 fi
 
-exit ${RETURN_CODE}
+exit ${RETURN_CODE}

install/scripts.d/ta/510_check_for_noprefixroute.sh

@@ -28,7 +28,10 @@ if [[ "${NOPREFIXROUTE_COUNT}" != "0" ]]; then
     echo "Certain IP addresses are configured with noprefixroute. This will inhibit the ability"
     echo "of certain cluster floating ips to accurately determine which link should be preferred"
     echo "The command \"ip -o -f inet route list match xxx.xxx.xxx.xxx/32 scope link\" needs to"
-    echo "Be able to return a device for each floating IP configured"
+    echo "be able to return a device for each floating IP configured"
+    echo "Recommended Resolution: remove the noprefixroute flag or otherwise ensure the"
+    echo " ip route list command given above can resolve the link on which you wish the"
+    echo " floating IP to be configured"
 fi
 
 if [[ ${RETURN_CODE} -eq 0 ]]; then

install/scripts.d/ta/520_bucket_and_process_uptime.sh

@@ -36,12 +36,20 @@ CURRENT_TIME_EPOCH=$(                 date +%s)
 if [[ $((${CURRENT_TIME_EPOCH}-${MOST_RECENT_BUCKET_STARTTIME_EPOCH})) -lt 3600 ]]; then
     RETURN_CODE="254"
     echo "Weka buckets have been restarted within the last hour, or have never started. This may not be a problem on a new cluster"
-    echo "but could be indicative of problems (e.g. network flapping"
+    echo "but could be indicative of problems (e.g. network flapping)"
+    echo "Recommended Resolutions:"
+    echo " . If this is a new cluster, or hosts have been upgraded/reboot, this is likely expected"
+    echo " . Otherwise the most likely cause is network problems, such as link flapping or congestion."
+    echo " . Review hardware and network stability, then contact customer success"
 fi
 if [[ $((${CURRENT_TIME_EPOCH}-${MOST_RECENT_PROCESS_STARTTIME_EPOCH})) -lt 3600 ]]; then
     RETURN_CODE="254"
     echo "Weka processes have been restarted within the last hour, or have never started. This may not be a problem on a new cluster"
     echo "but could be indicative of problems (e.g. network flapping"
+    echo "Recommended Resolutions:"
+    echo " . If this is a new cluster, or hosts have been upgraded/reboot, this is likely expected"
+    echo " . Otherwise the most likely cause is network problems, such as link flapping or congestion."
+    echo " . Review hardware and network stability, then contact customer success"
 fi
 
 if [[ ${RETURN_CODE} -eq 0 ]]; then

install/scripts.d/ta/530_high_drive_read_ssd_ratio.sh

@@ -34,6 +34,9 @@ if [[ ${HIGHER_THAN_EXPECTED} == "YES" ]]; then
     echo "The ratio of NVMe read requests vs DRIVE node read operations is higher than expected over the last ${TIME_TO_EXAMINE}"
     echo "This could indicate a number of things, such as splitting of read requests or perhaps read amplification"
     echo "Review ${JIRA_REFERENCE} for details"
+    echo "Recommended Resolutions:"
+    echo " . This may be expected behavior for your workload"
+    echo " . The data may be read using much larger blocksizes than those in which it was written, and matching those may help"
 fi
 
 if [[ ${RETURN_CODE} -eq 0 ]]; then

install/scripts.d/ta/550_iptables_nats_local_traffic.sh

@@ -21,6 +21,7 @@ for IP_ADDRESS in $(hostname --all-ip-addresses) ; do
     if [[ $? -eq 0 ]] ; then
         echo "Warning: it is possible that traffic to or from local IP address ${IP_ADDRESS} will be subject to NAT"
         echo "This can cause intra-WEKA communication errors"
+        echo "Recommended Resolution: Do not NAT WEKA traffic"
         RETURN_CODE="254"
     fi
 done
@@ -29,6 +30,7 @@ for IP_ROUTE in $(ip -4 --json route list  | python3 -c 'import sys, json, colle
     if [[ $? -eq 0 ]] ; then
         echo "Warning: it is possible that traffic to or from subnet ${IP_ROUTE} will be subject to NAT"
         echo "This can cause intra-WEKA communication errors"
+        echo "Recommended Resolution: Do not NAT WEKA traffic"
         RETURN_CODE="254"
     fi
 done

install/scripts.d/ta/560_check_for_swap.sh

@@ -14,6 +14,7 @@ SWAPTOTAL=$(grep SwapTotal /proc/meminfo  | awk '{print $2}')
 if [[ ${SWAPTOTAL} -ne "0" ]] ; then
     echo "This host has swap configured - this is unlikely to be"
     echo "helpful in a large memory system"
+    echo "Recommended Resolution: if the host has enough RAM, disable swap with swapoff then disable swap at boot time (likely in /etc/fstab)"
     RETURN_CODE="254"
 fi
 

install/scripts.d/ta/570_does_weka_use_swap.sh

@@ -17,6 +17,10 @@ for WEKAPID in $(ps -eo pid,comm | grep weka_init | awk '{print $1}') ; do
     if [[ ${NUM_PROCS_USING_SWAP} -gt "0" ]] ; then
         echo "There are Weka processes using swap - this is likely to be"
         echo "detrimental to performance"
+        echo "Recommended Resolutions:"
+        echo " . Add more RAM if the host is truly constrained"
+        echo " . Review if the host has not correctly released RAM"
+        echo " . Reduce the amount of RAM allocated to WEKA (a last resort)"
         RETURN_CODE="254"
     fi
 done

install/scripts.d/ta/580_weka_version_available_everywhere.sh

@@ -27,6 +27,8 @@ CURRENT_AGENT_VERSION=$(weka local status | awk 'NR==1{print $5}' | tr -d ')')
 if [[ ${WEKA_CLUSTER_VERSION} != ${CURRENT_AGENT_VERSION} ]] ; then
     echo "The currently running cluster version ${WEKA_CLUSTER_VERSION} does not match the"
     echo "default installed local agent version ${CURRENT_AGENT_VERSION}"
+    echo "Recommended Resolution: update this host to the cluster version, either by"
+    echo " unmounting and re-mounting filesystems or using the weka local upgrade utility"
     RETURN_CODE="254"
 fi
 

install/scripts.d/ta/590_single_dns_entry.sh

@@ -43,6 +43,7 @@ fi
 if [[ ${NUMBER_OF_A_RECORDS} != "1" ]] ; then
     echo "There are ${NUMBER_OF_A_RECORDS} A records in DNS for ${HOSTNAME}"
     echo "This is very likely to cause problems with (at least) SMB-W clustering"
+    echo "Recommended Resolution: add a DNS record of type A for ${HOSTNAME} pointing to the IPv4 address"
     RETURN_CODE=254
 else
     echo "There is exactly one A record in DNS for ${HOSTNAME}"