.bash-workutils

function achoo  { >&2 echo $@; }

function silent { "$@" > /dev/null 2> /dev/null; }


declare -A ls_shorthelp
     ls_shorthelp[ls_bigold]="<depth>"
     ls_shorthelp[ls_mouldy]="<days-old> <depth>"
   ls_shorthelp[ls_size_any]="[low] [high]        # 4k 10M 2G etc"
ls_shorthelp[ls_size_suffix]="<suffix> [low]      # 4k 10M 2G etc"
ls_shorthelp[ls_file_spread]="# no argument"
ls_shorthelp[ls_count_files]="<dir name>"


function ls_ls {
  local f
  local help=false
  list_long="ls_bigold ls_mouldy ls_size_any ls_size_suffix ls_file_spread"
  list_short="ls_count_files"
  if [[ $1 == '-l' ]]; then
    help=true
    for f in $list_long; do
      echo -e "\n--- $f"
      $f -h
    done
    echo
  fi
  for f in $list_long; do
    printf "%-20s %s\n" "$f" "${ls_shorthelp[$f]}"
  done
  for f in $list_short; do
    printf "%-20s %s\n" "$f" "${ls_shorthelp[$f]}"
  done
  echo "^--"
  echo "All these functions traverse directories. Use with care."
  echo "Use type -a funcname to see the function definition."
  echo "---"
  ls_misc
  if ! $help; then
    echo "Use ls_ls -l to see longer descriptions."
  fi
}


function ls_bigold {
if [[ -z $1 || $1 == '-h' ]]; then
cat <<DOC
  List directories up to a certain depth, ordered by disk usage,
  with the number of days since last modified.
  Argument: directory depth.
  Example:
  ${FUNCNAME[0]} 2
  NOTE: in a project/team root directory this may take some time and
  tax the file system. Perhaps best to save the output in a file.
  CAVEAT subdirectories of a directory may have changed. Use as guide!
  USEFUL order the output by the third column to group directories together,
    e.g. ls_bigold 2 > out.bigold; sort -k 3 out.bigold
DOC
return 1
fi
  local depth=$1
  (   # do not export in user shell.
  export now=$(date +%s)
  du -h -d $depth --time --time-style=+%s |\
  perl -ane '$F[1]=int(($ENV{now}-$F[1])/86400); local $"="\t"; print "@F\n";' |\
  sort -h | column -t
  )
}


function ls_mouldy {
if [[ -z $1 || $1 == '-h' ]]; then
cat <<DOC
  Find directories left untouched for longer than first argument (in days)
  up to a depth of second argument.
  Example:
  ${FUNCNAME[0]} 183 3
  CAVEAT subdirectories of a directory may have changed. Use as guide!
DOC
return 1
fi
  (  # do not export in user shell.
  local staleness=${1?Need staleness as first parameter}
  local depth=${2?Need depth as second parameter}
  export now=$(date +%s)

  find -maxdepth "$depth" -type d -mtime +"$staleness" -printf "%Cs\t%p\n" |\
  perl -ane '$F[0]=int(($ENV{now}-$F[0])/86400); local $"="\t"; print "@F\n";' |\
  sort -nk 1 | column -t
  )
}


function ls_size_suffix {
if [[ -z $1 || $1 == '-h' ]]; then
cat <<DOC
  Find files ending with suffix recursively, sort by human-readable size.
  First argument: suffix, e.g. .cram or .fastq.gz
  Second (optional) argument: a lower bound for size, e.g. 10M or 64k.
  Example:
  ${FUNCNAME[0]} .fastq.gz
  ${FUNCNAME[0]} .cram 500M
  ${FUNCNAME[0]} .cram 1G
DOC
return 1
fi
  local pat=${1?Please supply a file suffix to search for, e.g. .cram or .fastq.gz}
  local low=${2:-0}
  find -name "*$pat" -size +$low -print0 | du -hc --files0-from=- | sort -h
}



function ls_size_any {
if [[ $1 == '-h' ]]; then
cat <<DOC
  List all regular files recursively and sort by human-readable size.
  First optional argument:   lower bound e.g. 10M or 16k, or 0k
  Second optional argument:  upper bound e.g. 4k (useful for small files)
  Example:
  ${FUNCNAME[0]} 10M     # find files larger than 10M
  ${FUNCNAME[0]} 0k 4k   # find small files
DOC
return 1
fi
  local low=${1:-0}
  local high=${2:+-size -$2}      # if $2 exists (:+) use the expression -size -$2.
  find -type f -size +$low $high -print0 | du -hc --files0-from=- | sort -h
}



function ls_lastfile() {
if [[ $1 == '-h' ]]; then
cat <<DOC
  Set variable lf to the last regular file that was modified.
  Optional first argument: a suffix
  Example:
  ls_lastfile .gz
DOC
return 1
fi
   local thefile=$(find  -maxdepth 1 -type f -print0 | xargs -0 \ls -rt1 | grep "$1"'$' | tail -n 1)
   if [[ -z $thefile ]]; then
      echo "not found: $1"
   elif [[ ! -f $thefile ]]; then
      echo "not a regular file: $thefile"
   else
      lf=$PWD/${thefile:2}
      echo set lf to $lf
      return 0
   fi
   lf=last-lf-request-failed
   return 1
}


  # I wonder if there are better ways of doing this. Filesystem locality probably means
  # there is nothing substantially better than using find.
  # Don't use this in a root directory or high-level directory (as it will
  # take a long time and taxes the file system); I use it
  # to check the number of files a particular piece of software creates.
  #
function ls_count_files {
  dir=${1:-.}
  (
  if ! cd $dir; then
    return 1
  fi
  find .//. ! -name . -print | grep -c //
  )
}

function public {
  local dir=${1:-.}
  if [[ ! -d $dir ]]; then
    echo "Need a directory"
    return 1
  fi
  find $dir -type d -exec chmod a+rx {} ';'
  find $dir -type f -exec chmod a+r {} ';'
}


function ls_file_spread {
if [[ $1 == '-h' ]]; then
cat <<DOC
  For each directory count the number of files in it, recursively.
  The output is sorted by count, with a total tally added.
  Useful to check if applications are well-behaved and do not
  crush the file system with large numbers of files in a single directory.
  Modified from code by Glenn Jackman on stackoverflow.
DOC
return 1
fi
  local dir
  local count=0
  (
  shopt -s nullglob
  shopt -s dotglob
  while read -d '' -r dir; do
      files=("$dir"/*)
      delta=${#files[@]}
      count=$(($count+$delta))
      printf "%8d %s\n" "$delta" "$dir"
  done < <(find -type d -print0)
  printf "%8d Total sum of subcounts\n" "$count"
  ) | sort -n
}


  # TODO
  #   - Check (-c) and all (-a) do not mix currently. add -h.
  #   - Add option(s) to allow/deny/request first row delta
  #   - Be graceful on files with fewer than three lines.
  #
function colcount()
{ local check=false
  local all=false
  local ret=0
  if [[ ! -z "$1" ]] && [[ $1 == -c ]]; then
    check=true
    shift
  elif [[ ! -z "$1" ]] && [[ $1 == -a ]]; then
    all=true
    shift
  fi
  if [[ -z "$1" ]]; then
    achoo "Need file name argument"
    return 1
  fi

  if $all; then
    perl -ne '$n=tr/\t/\t/+1;print"$n\n"' $1 | sort | uniq -c | sort -n | perl -pe 's/^\s+(\d+)\s+(\d+)/$2\t$1/'
  else
    c1=$(head -n 1 $1 | tr '\t' '\n' | wc -l)
    c2=$(head -n 2 $1 | tail -n 1 | tr '\t' '\n' | wc -l)
    c3=$(head -n 3 $1 | tail -n 1 | tr '\t' '\n' | wc -l)
    if $check && ! (( $c2 == $c3 && ($c1+1 == $c2 || $c1 == $c2) )); then
      ret=1
    fi
    echo -e "$c1\t$c2\t$c3"
  fi
  return $ret
}


function colnames()
{ if [[ -z "$1" ]]; then
    achoo "Need file name argument"
    return 1
  fi
  head -n 1 $1 | tr '\t' '\n'
}


function bj {
  bjobs -o 'jobid queue stat exec_host run_time job_name memlimit delimiter="	"' | perl -pe 's/second(\(s\))?//' | column -t -s $'\t'
}
function bjl {
  bjobs -o 'jobid queue stat exec_host run_time job_name memlimit sub_cwd delimiter="	"' | perl -pe 's/second(\(s\))?//' | column -t -s $'\t'
}


function procli() {
if [[ -z $1 || $1 == '-h' ]]; then
cat <<DOC
  Given PID argument, I print /proc/PID/cmdline
DOC
return 1
fi
  (set -euo pipefail; cat /proc/$1/cmdline  | tr '\0' ' ') && echo
}


  # Give full file name for arguments.
  #
function ffn()
{  for f in $*; do
      echo "$PWD/$f" | tee -a $HOME/.ffn.mycache
   done
}


function ls_func {
   file=${1:?I need a file name argument}
   bash --norc --noprofile  -c "source $file && declare -F" | cut -f 3 -d ' '
}


function tailafter {
    pattern=${1:?I need a pattern (used in perl regex)}
    if [[ $# < 2 ]]; then
      file=/dev/stdin
    else
      file=$2
    fi
    perl -ne "print unless 1 .. /$pattern/" $file
}

function grab {
    p1=${1:?I need a first pattern (used in perl regex)}
    p2=${2:?I need a second pattern (used in perl regex)}
    if [[ $# < 3 ]]; then
      file=/dev/stdin
    else
      file=$3
    fi
    perl -ne "print if /$p1/ .. /$p2/" $file
}


function howoldami {
   age=$(ps -o etime= -p "$$")
   name=$(whoami)
   host=$HOSTNAME
   if [[ ! -z $LSB_JOBID ]]; then
     info=($(bjobs -o "queue slots memlimit" $LSB_JOBID | tail -n 1))
    echo "$name $age $host (queue ${info[0]}, cpus ${info[1]}, mem ${info[2]})"
   else
    echo "$name $age $host"
  fi
}

function funcfile {
  funcname=${1:?I need a function name}
  local extdebug=$(shopt -p extdebug)
  shopt -s extdebug
  local result=($(declare -F $funcname))
  echo ${result[2]}
  $extdebug
}

function ls_misc() {
cat <<DOC
F: a file name. F+ one or more file names.
fu: a function name
S: a string
i: integer
n: integer
  colcount F    output field counts for first three lines (tab separator)
  colcount -a F output field counts for all lines -c and -a do not mix yet
  colcount -c F check field count consistency for first three lines.
                NOTE. first line discrepancy is allowed.
  colnames F    output the first row of a tab-separated file, one field per line
  procli PID    print command line for a PID according to /proc/PID/cmdline
  bj            bjobs information -- outputs useful bjobs status fields
  ls_func F     list the names of bash functions defined in a file
  ls_misc       This (list miscellaneous functions)
  ffn F+        print full(y qualified) file name for arguments
  funcfile fu   print name of file where function fu is defined
  howoldami     how old is the current shell
  tailafter P [F]     print lines in file after first pattern P match
  grab P1 P2 [F]  print chunks inbetween patterns P1 and P2, inclusive
  nchar S       how many bytes in string S
  lines i n     print n lines from i onwards, first line is 1.
  groupify		  make directory and below group writable
  ungroupify		unmake directory and below group writable
  gimme_sum		  sum of arguments
  myman					process troff source, read in less
  perusual			permission chain upwards for argument
DOC
}


function groupify()
{  find $1 -user $USER -print0 | xargs -0 chmod g+w
}

function ungroupify()
{  find $1 -user $USER -print0 | xargs -0 chmod g-w
}

function nchar()
{  echo -n "$1" | wc -c
}

function lines()
{ [[ -z $1 ]] && echo "Need <lineno> or <lineno> <count>, I only read STDIN" && return 1
  count=1
  if [[ ! -z $2 ]]; then count=$2; fi
  head -n $(($1+$count-1)) | tail -n $count
}

function gimme_sum()
{ echo "$(( $(tr ' ' '+' <<< "$@" ) ))"
}

function myman()
{  local src=${1?Need roff file argument}
   groff -t -e -mandoc -Tascii $src | less
}

function perusal()
{ a=${1?Need file or directory name}
  b=${a%\/*}
  list=($a)
  while [[ $b != $a ]]; do
    a=$b
    b=${a%\/*}
    list+=($b)
  done
  ls -ld "${list[@]}"
}

function c80()
{  num=${1-80}
   cut -b 1-$num
}

function vw () 
{  IFS= read -r header;
   echo "$header";
   sort $@
}

function rc() {
  if [[ -n $1 ]]; then
    echo $1 | tr GCATUgcatu  CGTAAcgtaa  | rev
  else
     tr GCATUgcatu  CGTAAcgtAa  | rev
   fi
}

function trtn {
   tr '\t' '\n'
}