nf_numerical

#!/usr/bin/env bash

# nf_numerical
#
#USAGE:
#  ./nf_numerical --help
#
#DESCRIPTION:
#  A script for checking the output from the nftsim repo's .conf files
#  against their expected test output. The expected/test output is stored in 
#  the directory
#    ./test/numerical/data/
#  The script also supports updating those expected output files when code or
#  .conf file changes have been made that intentionally modify the output.
#
# Author: Stuart A. Knock
# Originally Written: 2016-11-23
#

#Colourise some of our output.
declare -r DIFFERS_CLR='\e[1;33mDIFFERS!\e[0m' # Bold Yellow-ish
declare -r ERROR_CLR='\e[1;31mERROR:\e[0m'     # Bold Red
declare -r SAME_CLR='\e[1;32mSAME\e[0m'        # Bold Green

#Capture args the script was called with.
INPUTARG1="$1" ; INPUTARG2="$2"
#Make them read-only so we can't accidentally modify them.
declare -r  INPUTARG1 INPUTARG2

#Check we were called with the right number of args:
(($# > 2)) && { printf '%s\n' "${BASH_SOURCE[0]} takes at most 2 args" ; exit 1 ; }

SCRIPTDIR="$(dirname "${BASH_SOURCE[0]}")"

nftsim_exists(){
  if [[ -e "$SCRIPTDIR"/bin/nftsim ]]; then
    return 0
  else
    printf '%s\n' "The nftsim executable doesn't exist, run make."
    return 1
  fi
}

#Run appropriate functions based on how we were called.
parse_command_flags(){
  #check for command line switches
  case "$INPUTARG1" in
    --regenerate|-r)
      nftsim_exists || return 1
      regenerate_output_file "$INPUTARG2"
      return "$?"
    ;;
    --check|-c)
      nftsim_exists || return 1
      if [[ "$INPUTARG2" = 'all' ]]; then
        check_all
        return "$?"
      else
        if ! check_output_file "$INPUTARG2"; then return 1; fi
        veiw_diff "$INPUTARG2"
      fi
      return "$DIFF_STATUS"
    ;;
    --clean)
      #Remove files left around by calls to --check that found differences.
      rm -f /tmp/nftsim_*
      return "$?"
    ;;
    --help|-h|-?)
      msg_help
      return 0
    ;;
  esac
  printf '%s\n' "Unrecognised command line switch: $INPUTARG1"
  return 1
}

msg_help(){
  printf '\n %s\n' "Usage:"
  printf '   %s\n\n' "nf_numerical --<mode> '<config_file>'"
  printf '%s\n'   " NOTE: For exact reproducibility on older gcc (School of Physics RHEL), you must first"
  printf '%s\n'   "       rebuild nftsim using:"
  printf '%s\n'   "           make clean"
  printf '%s\n\n' "           make debug"
  printf ' %s\n' "Modes:"
  printf '\n%s\n' "  --check"
  printf '%s\n'   "    Check if the output from nftsim has changed for a specified config file."
  printf '%s\n'   "    For example, to check the output of the 'example.conf' file, run:"
  printf '%s\n'   "        ./nf_numerical --check 'example'"
  printf '%s\n'   "    or:"
  printf '%s\n'   "        ./nf_numerical --check './test/numerical/example.conf'"
  printf '%s\n'   "    or to check all configurations specified in the test/numerical directory, run:"
  printf '%s\n'   "        ./nf_numerical --check 'all'"
  printf '\n%s\n' "  --regenerate"
  printf '%s\n'   "    Regenerates the test output for the specified config file. This should ONLY be"
  printf '%s\n'   "    used when you've intentionally made a change to the code or config file"
  printf '%s\n'   "    that should change the test output. The updated test output file should then be"
  printf '%s\n'   "    committed as part of the pull request for the code changes that caused the"
  printf '%s\n'   "    output to change."
  printf '%s\n'   "    eg: ./nf_numerical --regenerate 'example'"
  printf '%s\n'   "    eg: ./nf_numerical --regenerate './test/numerical/example.conf'"
  printf '\n%s\n' "  --clean"
  printf '%s\n'   "    Removes files that are left in the /tmp/ directory when a call to --check"
  printf '%s\n'   "    finds differences."
  printf '%s\n'   "    eg: ./nf_numerical --clean"
  printf '\n%s\n' "  --help"
  printf '%s\n'   "    Displays this message"
  printf '%s\n\n'   "    eg: ./nf_numerical --help"
}

#Sets the global variable OUTPUT_FILE to the default location for the
#specified configuration file.
set_output_file(){
  local config_file="$1"

  #Set the default output file path for the current config file.
  OUTPUT_FILE="$SCRIPTDIR/test/numerical/data/${config_file#*/numerical/*}"
  OUTPUT_FILE="${OUTPUT_FILE%.*}.output"

}

# Run the config file provided as an argument, the first arg is expected
# to be a relative(to nftsim) path to the config file.
run_config(){
  local config_file="$1"
  local output_file
  local nftsim_status

  #If run_config() was called with a second arg use it as the output_file 
  if (( $# > 1 )); then
    output_file="$2"
  else #otherwise use the default location in the test directory
    output_file="$OUTPUT_FILE"
    local output_dir
    output_dir="$(dirname "$output_file")"
    #If it doesn't already exist: Create the output directory.
    [[ -d "$output_dir" ]] || mkdir --parents "$output_dir"
  fi
  
  #Run nftsim for the requested config file
  "$SCRIPTDIR/bin/nftsim" -i "$config_file" -o "$output_file"
  nftsim_status="$?"

  #If the run was successful then print an error message.
  if ((nftsim_status != 0)); then
    printf "$ERROR_CLR %s\n" "Failed running: $SCRIPTDIR/bin/nftsim -i $config_file -o $output_file"
  fi
  
  return "$nftsim_status"
}

get_all_config_files(){
  #NOTE: currently restricting the search for conf files to a max-depth of 1 as
  #      we are placing all .conf files directly in the ./test/numerical/ directory.
  #SAK#requires BASH >=4.2# declare -ga CONFIG_FILES
  CONFIG_FILES=($(find "$SCRIPTDIR/test/numerical" -maxdepth 1 -type f -name '*.conf'))
  return 0
}

#Creates an output file in ./test/numerical/data/ corresponding to the specified .conf file.
regenerate_output_file(){
  local config_file="$1"
  if [[ "$config_file" == "all" ]]; then
    get_all_config_files
    for (( i = 0; i < ${#CONFIG_FILES[@]}; i++ )) ; do
      set_output_file "${CONFIG_FILES[i]}"
      run_config "${CONFIG_FILES[i]}"
    done
    return 0
  elif [[ "$config_file" == "changed" ]]; then
    [[ -f "/tmp/nftsim_changed_output_list_$(whoami).txt" ]] || { printf '%s\n' 'No changed list.'; return 1; }
    local regen_file
    local -i regen_errors=0
    while read -r output_to_regenerate ; do
      regen_file="${output_to_regenerate##*/}"
      regen_file="${regen_file%*.output}"
      regen_file="$(find "$SCRIPTDIR/test/numerical" -maxdepth 1 -type f -name "${regen_file}.conf")"
      printf '%s\n' "Regenerating '$regen_file'..."
      set_output_file "$regen_file"
      if ! run_config "$regen_file"; then ((++regen_errors)); fi
    done < "/tmp/nftsim_changed_output_list_$(whoami).txt"
    if ((regen_errors!=0)); then
      printf '%s\n' "There were '$regen_errors' errors while regenerating changed output."
      return "$regen_errors"
    fi
    return 0
  fi
  if [[ ! -f "$config_file" ]]; then
    config_file="$(find "$SCRIPTDIR/test/numerical" -maxdepth 1 -type f -name "${config_file}.conf")"
  fi
  if [[ -f "$config_file" ]]; then
    set_output_file "$config_file"
    run_config "$config_file"
    return "$?"
  else
    printf '%s\n' "You need to specify which config file to regenerate output for."
    printf '%s\n' "eg. nf_numerical --regenerate 'example'"
    printf '%s\n' "or nf_numerical --regenerate './test/numerical/example.conf'"
  fi
  return 1
}

check_all(){
  local -r changed_ouput_list="/tmp/nftsim_changed_output_list_$(whoami).txt"
  [[ -f "$changed_ouput_list" ]] && rm -f "$changed_ouput_list"
  local error_count=0
  local conf_count=0
  get_all_config_files
  for (( i = 0; i < ${#CONFIG_FILES[@]}; i++ )) ; do
    ((++conf_count))
    printf '%-64s' "Checking '${CONFIG_FILES[i]}'... "
    if ! check_output_file "${CONFIG_FILES[i]}"; then ((++error_count)); fi
    if [[ "$DIFF_STATUS" = '0' ]]; then
      printf "${SAME_CLR}.\n"
      rm -f "${OUTPUT_FILE_TMP}"
    elif [[ "$DIFF_STATUS" = '1' ]]; then
      printf "${DIFFERS_CLR}\n"
      printf '%s\n' "$(readlink -f "${OUTPUT_FILE}")" >> "$changed_ouput_list"
    fi
  done
  if [[ "$error_count" != '0' ]]; then
    printf '%s\n' "There were '$error_count' of '$conf_count' config files that could not be checked."
  fi
  return "$error_count"
}

#Generates a temporary output file in /tmp/ for a specified config file and
#compares it to the corresponding output file in nftsim/test/numerical/data/
check_output_file(){
  local config_file="$1"
  local config_file_name
  DIFF_STATUS=42

  #If arg isn't a file, try mapping it to a file name.
  if [[ ! -f "$config_file" ]]; then
    config_file="$(find "$SCRIPTDIR/test/numerical" -maxdepth 1 -type f -name "${config_file}.conf")"
  fi

  #Compare the current output of nftsim against stored output.
  if [[ -f "$config_file" ]]; then
    set_output_file "$config_file"
    [[ -f "${OUTPUT_FILE}" ]] || { printf "$ERROR_CLR %s\n" "No existing file to check against."; return 1; }
    config_file_name="$(basename "$config_file")"
    #Path to file containing output of current nftsim.
    OUTPUT_FILE_TMP="$(mktemp --tmpdir="/tmp" --suffix=.output nftsim_"$config_file_name"-XXXX 2>/dev/null \
                     || mktemp /tmp/nftsim_"$config_file_name".XXXX)" #Alternative for OSX
    run_config "$config_file" "$OUTPUT_FILE_TMP" || return 1
    diff -q "${OUTPUT_FILE_TMP}" "${OUTPUT_FILE}" &> /dev/null
    DIFF_STATUS="$?" #NOTE: No-diff = 0; diff = 1; error = 2
    return 0
  else
    printf '%s\n' "You need to specify which config file to regenerate output for."
    printf '%s\n' "eg. nf_numerical --regenerate 'example'"
  fi
  return 1
}

veiw_diff(){
  local config_file="$1"
  case "$DIFF_STATUS" in
    0)
      printf '%s\n' "'$config_file' output hasn't changed."
      rm -f "${OUTPUT_FILE_TMP}"
  ;;
    1)
      diff --side-by-side --suppress-common-lines --width=180 "${OUTPUT_FILE}" "${OUTPUT_FILE_TMP}" | less
      printf '%s\n' "'$config_file' output DIFFERS!"
      printf '%s\n' "See: '${OUTPUT_FILE_TMP}'"
  ;;
    2)
      printf '%s\n' "Error while running diff."
  ;;
  esac
  return "$DIFF_STATUS"
}

#The function run when the script is called
main(){
  parse_command_flags
  exit "$?"
}

#Run the script's main function.
main