tests

#!/bin/bash

# This script handles management of testing.
# It supports the creation, verification, and updating
# of tests.
#
# Tests are run in the context of an edlib git tree
# with a given hash - currently the same hash for all tests.
# Each test provides a list of keystrokes and mouse actions
# together with occasions checksums of the current display.
# We run a test and record the output at the same time, then
# confirm that the recording matching the input test.

# Usage:
# ./tests new name
#     Record a new test and store it with given name.  Then re-run and confirm.
# ./tests run [name|failed]
#     If a name is given, just that test is run.  If no name is given, all tests
#     are run.  If any test fails, that failure is recorded.
#     If the special name 'failed' is given, then only failed tests are run.
# ./tests refresh
#     Run all tests and replace the test with the new log - expecting it to be different.

# Ensure configuration (e.g. of aspell) in $HOME doesn't affect result
export HOME=/tmp

# ensure timestamps don't change
export EDLIB_TESTING=1
# Don't check for autosaves
export EDLIB_AUTOSAVE=/no/autosave/dir

export TEST_DIR=/tmp/edlib-tests
export TEST_COMMIT=5875a3514f4e753fa183a2e19000e803384fae77

export ASPELL_CONF="personal $TEST_DIR/DOC/extra/aspell.en.pws;repl $TEST_DIR/DOC/extra/aspell.en.prepl"

runtest()
{
	case $1 in
	new ) EDLIB_RECORD=$2 \
		screen -m sh -c 'stty rows 30 cols 80; ./edlib $TEST_DIR'
	;;
	run ) EDLIB_RECORD=$2 EDLIB_REPLAY=$3 \
		screen -D -m sh -c 'stty rows 30 cols 80; ./edlib $TEST_DIR'
	;;
	view ) EDLIB_RECORD=/dev/null EDLIB_REPLAY=$2 \
		screen -m sh -c 'stty rows 30 cols 80; ./edlib $TEST_DIR'
	;;
	extend ) EDLIB_RECORD=$2 EDLIB_REPLAY=$3 \
		screen -m sh -c 'stty rows 30 cols 80; ./edlib $TEST_DIR'
	;;
	esac
}

cmd=$0

setup_test_dir() {
    if [ -d $TEST_DIR -a -d $TEST_DIR/.git ]; then
        rm -rf $TEST_DIR/DOC/extra
	(cd $TEST_DIR ; git reset > /dev/null --hard $TEST_COMMIT ;
	 git clean -fxd; git config core.abbrev 12 )
    else
	rm -rf $TEST_DIR
	git clone . $TEST_DIR
	(cd $TEST_DIR ; git reset --hard $TEST_COMMIT; git config core.abbrev 12 )
    fi
    rm -rf $TEST_DIR/DOC/extra
    mkdir -p $TEST_DIR/DOC/extra
    cp -a tests.d/[A-Za-z]* $TEST_DIR/DOC/extra
}

new_test() {
    if [ $# -ne 1 ]; then
	echo >&2 "$0: please provide name for new test"
	exit 1
    fi
    t=$1
    case $t in
	[0-9][0-9]-* ) : OK ;;
	* )  echo >&2 "Please name the test NN-something"
	     exit 1
    esac
    tt=tests.d/$t
    if [ -e "$tt" ]; then
	echo >&2 "$0: $tt already exists, please choose another name"
	exit 1
    fi
    if ! touch "$tt"; then
	echo >&2 "$0: Cannot create $tt - strange"
	exit 1
    fi
    setup_test_dir
    rm -f tests.d/.tmp
    runtest new tests.d/.tmp
    if ! grep -s '^Close' tests.d/.tmp > /dev/null ; then
	echo >&2 "$0: test file not Closed - probably crash"
	exit 1
    fi
    setup_test_dir
    runtest run $tt tests.d/.tmp
    diff -u tests.d/.tmp $tt
    #rm tests.d/.tmp
    exit 0
}

extend_test() {
    if [ $# -ne 1 ]; then
	echo >&2 "$0: please provide name for test to extend"
	exit 1
    fi
    t=$1
    tt=tests.d/$t
    if [ ! -e "$tt" ]; then
	echo >&2 "$0: test $t doesn't exist"
	exit 1
    fi
    rm -f tests.d/.tmp tests.d/.tmp2
    # find :C-X :C-C that closes the editor.
    n=$(grep -n 'Key ":C-X"' $tt | tail -n 1| sed 's/:.*//')
    if [ -z "$n" ]; then
	echo >&2 "$0: cannot find tailing :C-X"
	exit 1;
    fi
    head -n $[n-1] $tt > tests.d/.tmp2
    runtest extend tests.d/.tmp tests.d/.tmp2
    if ! grep -s '^Close' tests.d/.tmp > /dev/null ; then
	echo >&2 "$0: test file not Closed - probably crash"
	exit 1
    fi
    setup_test_dir
    runtest run $tt tests.d/.tmp
    #diff -u tests.d/.tmp $tt
    rm -f tests.d/.tmp tests.d/.tmp2
    exit 0
}

run_one_test() {
    if [ ! -f "tests.d/$1" ]; then
	echo "$cmd: test $1 doesn't exist"
	exit_code=2
	return
    fi
    setup_test_dir
    echo -n "run $1 ..."
    runtest run tests.d/.out tests.d/$1
    if diff -u "tests.d/$1" tests.d/.out;
    then echo "succeeded."
    else echo "FAILED."
	exit_code=1
	line=`diff tests.d/$1 tests.d/.out | sed -n '1s/^\([0-9]*\).*/\1/p'`
	echo "Use: ./tests view $1 $line"
    fi
    rm -f tests.d/.out
}

run_tests() {
    if [ $# -gt 1 ]; then
	echo >&2 "$0: please provide at most one test to run"
	exit 1
    fi
    exit_code=0
    if [ $# -eq 0 ]; then
	for f in tests.d/[0-9]*; do
	    b=${f#tests.d/}
	    run_one_test "$b"
	    if [ $exit_code -gt 0 ]; then
		if [ "x$b" = "x02-grep" ]; then
		    echo "02-grep failure NOT fatal"
		    exit_code=0
		else
		    break
		fi
	    fi
	done
    elif [ " $1" = " -a" ]; then
	for f in tests.d/[0-9]*; do
	    b=${f#tests.d/}
	    run_one_test "$b"
	done
    else
	run_one_test "$1"
    fi
    exit $exit_code
}

refresh_one() {
    b=$1
    f=tests.d/$b
    if [ ! -f "$f" ]; then
	echo >&2 "Cannot find test $f"
	return
    fi
    setup_test_dir
    # ensure we see delayed updates
    export EDLIB_REPLAY_DELAY=500
    #grep -v '^Display' $f > tests.d/.in
    cat $f > tests.d/.in
    runtest run tests.d/.out tests.d/.in
    mv tests.d/.out $f
    rm -f tests.d/.in
}

refresh_one_fast() {
    b=$1
    f=tests.d/$b
    if [ ! -f "$f" ]; then
	echo >&2 "Cannot find test $f"
	return
    fi
    setup_test_dir
    runtest run tests.d/.out "tests.d/$1"
    mv tests.d/.out tests.d/"$1"
}

refresh_tests() {
    case "$#:$1:$2" in
        1:-f:* ) tst=""; cmd=refresh_one_fast ;;
	1:* ) tst=$1; cmd=refresh_one ;;
        2:*:-f ) tst=$1; cmd=refresh_one_fast ;;
	0: ) tst=""; cmd=refresh_one ;;
        * )
	   echo >&2 "$0: please provide at most one test to refresh"
	   exit 1
    esac
    if [ -z "$tst" ]; then
	for f in tests.d/[0-9]*; do
	    $cmd ${f#tests.d/}
	done
    else
	$cmd "$tst"
    fi
}

view_test() {
    if [ $# -lt 1 ]; then
	echo >&2 "$0: please provide a test name to view"
	exit 1
    fi
    t=$1
    if [ ! -f tests.d/$1 ]; then
	echo >&2 "Test $1 doesn't exist."
	exit 1
    fi
    if [ $# -ge 2 -a " $2" != " -" ]; then
	head -n $2 tests.d/$t > tests.d/.in
    else
	cat tests.d/$t > tests.d/.in
    fi
    delay=20
    if [ $# -ge 3 ]; then
	delay=$3
    fi
    export EDLIB_REPLAY_DELAY=$delay
    runtest view tests.d/.in
    rm -f tests.d/.in
}


cmd=$1
shift
case $cmd in
	new) new_test ${1+"$@"};;
	run) setup_test_dir; run_tests ${1+"$@"};;
	refresh) setup_test_dir; refresh_tests ${1+"$@"};;
	view ) setup_test_dir; view_test ${1+"$@"};;
	extend ) setup_test_dir; extend_test ${1+"$@"};;
	* )
		echo >&2 "Usage: $0 new test-name"
		echo >&2 "       $0 run [test-name|-a]"
		echo >&2 "       $0 refresh [test-name]"
		echo >&2 "       $0 view test-name [lines [delay-ms]]"
		echo >&2 "       $0 extend test-name"
		exit 1;
esac
exit 0