watchdog

#!/usr/bin/env python
import sys
import subprocess
import time
import os
import csv
import datetime
from qiang import config

checkers = {}


def start_checker(destination):
    checkers[destination]['process'] = subprocess.Popen(
        './find-router %s' % destination, shell=True,
        stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
    checkers[destination]['started_at'] = time.time()


def log_error(error):
    sys.stderr.write(error)
    sys.stderr.write('\n')
    sys.stderr.flush()


def log_message(message):
    sys.stdout.write(message)
    sys.stdout.write('\n')
    sys.stdout.flush()
    sys.stderr.write(message)
    sys.stderr.write('\n')
    sys.stderr.flush()


def record_event(event, destination):
    log_message('%s: %s' % (event, destination))
    with open(os.path.join(config.output_dir, 'watchdog.csv'), 'a') as f:
        csv_writer = csv.writer(f)
        csv_writer.writerow([datetime.datetime.now().isoformat(), event, destination])


for destination in sys.argv[1:]:
    checkers[destination] = {
        'was_blocked': None # unknown yet
    }
    start_checker(destination)

while True:
    for destination, checker in checkers.items():
        result = checker['process'].poll()
        if result is None:
            # not finished yet
            if time.time() - checker['started_at'] > 60 * 5:
                # timeout
                log_message('TIMEOUT: %s' % destination)
                log_error(checker['process'].stdout.read())
                checker['process'].kill()
                start_checker(destination)
        elif 0 == result:
            if checker['was_blocked'] is None:
                record_event('INITIALLY NOT BLOCKED', destination)
            elif checker['was_blocked']:
                record_event('UNBLOCKED', destination)
            start_checker(destination)
            checker['was_blocked'] = False
        elif 42 == result:
            if checker['was_blocked'] is None:
                record_event('INITIALLY BLOCKED', destination)
            elif not checker['was_blocked']:
                record_event('BLOCKED', destination)
            start_checker(destination)
            checker['was_blocked'] = True
        else:
            log_message('ERROR [%s]: %s' % (result, destination))
            log_error(checker['process'].stdout.read())
            start_checker(destination)
    time.sleep(1)