From e7682b9603451ab126537e6fc42dc51068276669 Mon Sep 17 00:00:00 2001 From: Esa Jokinen Date: Wed, 14 Aug 2024 14:58:10 +0300 Subject: [PATCH] follow-cvelist.py hard reset on pull failure (-F) mode Add recovery capabilities from 'git pull' failures. Continue on non-permanent (network) issues. Exit on permanent failures, or try 'git reset --hard origin/main' with -F/--force. --- README.md | 2 +- bin/follow-cvelist.py | 95 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 91 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 57f4b12..b1204f6 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Miscellaneous scripts for different purposes. Mostly unrelated to each other. | Email | [`mail-prepender.sh`](bin/mail-prepender.sh)
Shell (bash) | Prepends (to stdin/stdout) email header strings given in as flags `i`, `I`, `a`, or `A`; after possible mbox `From` & `Return-Path` header lines. Intended as a limited `formail` replacement that ignores the nyanses of the flags and simply prepends the valid (RFC 5322, 2.2) non-empty headers keeping the other headers as is. Flags `x` & `X` are implemented. Any other flags are ignored. | | Git | [`git-find-commits-by-file-hash.sh`](bin/git-find-commits-by-file-hash.sh)
Shell (bash) | Search Git repository history for commits with SHA-256 checksum of a file. Answers the question "Has this version of this file ever been committed as the file on this path of this Git repository?" and shows a summary (`git show --stat`) of the matching commit(s). The `path` should be relative to the repository root.
`git-find-commits-by-file-hash.sh sha256sum path`| | Infosec | [`netcat-proxy.sh`](bin/netcat-proxy.sh)
Shell (sh) | Creates a simple persistent TCP proxy with netcat & named pipes.
`netcat-proxy.sh listenport targethost targetport` | -| Infosec | [`follow-cvelist.py`](bin/follow-cvelist.py)
Python 3 | Follow changes (commits) in CVEProject / [cvelistV5](https://github.com/CVEProject/cvelistV5). Requires git. Working directory must be the root of the cvelistV5 repository.
`follow-cvelist.py [-haoru4] [-vvvv] [-i s] [-c N] [-w N]`| +| Infosec | [`follow-cvelist.py`](bin/follow-cvelist.py)
Python 3 | Follow changes (commits) in CVEProject / [cvelistV5](https://github.com/CVEProject/cvelistV5). Requires git. Working directory must be the root of the cvelistV5 repository.
`follow-cvelist.py [-haForu4] [-vvvv] [-i s] [-c N] [-w N]`| | Infosec | [`partialpassword.sh`](bin/partialpassword.sh)
Shell (bash) | Creates a new wordlist from a wordlist by replacing all ambiguous characters with all their possible combinations.
`partialpassword.sh input.txt output.txt O0 [Il1 ...]` | | Infosec | [`duplicate-ssh-hostkeys.sh`](bin/duplicate-ssh-hostkeys.sh)
Shell (bash) | Find duplicate SSH host keys in a CIDR range. Examine your network for shared host keys that could potentially be dangerous.
`duplicate-ssh-hostkeys.sh CIDR [HostKeyAlgorithm ...]` | | Infosec
Automation | [`make-mac-prefixes.py`](bin/make-mac-prefixes.py)
Python 3 | Processes registered MAC address prefixes from [IEEE MA-L Assignments (CSV)](https://standards.ieee.org/products-programs/regauth/) (stdin) to Nmap's [`nmap-mac-prefixes`](https://github.com/nmap/nmap/blob/master/nmap-mac-prefixes) (stdout) with a few additional unregistered OUIs.
`curl https://standards-oui.ieee.org/oui/oui.csv \| make-mac-prefixes.py > nmap-mac-prefixes` | diff --git a/bin/follow-cvelist.py b/bin/follow-cvelist.py index 0c08011..c27da0e 100755 --- a/bin/follow-cvelist.py +++ b/bin/follow-cvelist.py @@ -3,10 +3,11 @@ # ------------------------------------------------------------------------------ # Follow changes (commits) in CVEProject / cvelistV5 # -# Usage: follow-cvelist.py [-haoru4] [-vvvv] [-i s] [-c N] [-w N] +# Usage: follow-cvelist.py [-haForu4] [-vvvv] [-i s] [-c N] [-w N] # # -h, --help show this help message and exit # -a, --ansi add ansi colors to the output (default: False) +# -F, --force origin/main hard reset if git pull fails (default: False) # -o, --once only the current tail; no active follow (default: False) # -r, --reload-only skip pulls & only follow local changes (default: False) # -u, --url prefix cve with url to nvd nist details (default: False) @@ -165,7 +166,7 @@ def monitor(self) -> None: cursor = new_cursor def pull(self) -> None: - """Runs git pull. Exits if it fails.""" + """Runs git pull. Exits on permanent, unrecoverable errors""" result = subprocess.run( ["git", "pull"], stdout=subprocess.PIPE, stderr=subprocess.PIPE ) @@ -176,12 +177,89 @@ def pull(self) -> None: file=sys.stderr, ) if result.returncode > 0: + if self.fetch_all(): + if self.args.force: + if self.args.verbose > 1: + print( + f"{result.stderr.decode('utf-8').strip()}", + file=sys.stderr, + ) + self.reset_repo() + else: + print( + f"{result.stderr.decode('utf-8').strip()}", + file=sys.stderr, + ) + begin = "" + end = "" + if self.args.ansi: + begin = f"{ANSI.code('red')}" + end = f"{ANSI.code('end')}" + print( + f"{begin}Manual intervention (or -F/--force) " + f"required; 'git pull' failed permanently!{end}", + file=sys.stderr, + ) + sys.exit(1) + + def fetch_all(self) -> bool: + """Try fetch; good for distinguishing connectivity issues from other failures""" + result = subprocess.run( + ["git", "fetch", "--all"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + if self.args.verbose > 1: + print(f"{result.stdout.decode('utf-8').strip()}", file=sys.stderr) + if result.returncode > 0: + begin = "" + end = "" + if self.args.ansi: + begin = f"{ANSI.code('yellow')}" + end = f"{ANSI.code('end')}" + print( + f"{begin}{time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime())} " + f"Fetch failed; connectivity issues?{end}", + file=sys.stderr, + ) + return False # fetch failed + return True # fetch successful + + def reset_repo(self) -> None: + """Tries to recover from git pull errors by hard resetting to origin/main""" + begin = "" + end = "" + if self.args.ansi: + begin = f"{ANSI.code('yellow')}" + end = f"{ANSI.code('end')}" + print( + f"{begin}{time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime())}" + f" Recovering from a failed git pull...{end}", + file=sys.stderr, + ) + result = subprocess.run( + ["git", "reset", "--hard", "origin/main"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + if self.args.verbose > 1: + print(f"{result.stdout.decode('utf-8').strip()}", file=sys.stderr) + if result.returncode > 0: + begin = "" + end = "" + if self.args.ansi: + begin = f"{ANSI.code('red')}" + end = f"{ANSI.code('end')}" print( - f"\n{result.stderr.decode('utf-8').strip()}\n{ANSI.code('red')}" - f"Manual intervention required; 'git pull' failed!{ANSI.code('end')}", + f"{begin}Hard reset to origin/main failed; " + f"manual intervention required!{end}", file=sys.stderr, ) sys.exit(1) + begin = "" + end = "" + if self.args.ansi: + begin = f"{ANSI.code('green')}" + end = f"{ANSI.code('end')}" + print(f"{begin}Successfully recovered.{end}", file=sys.stderr) def get_cursor(self, offset: int = 0) -> str: """Gets commit id at the offset from the current head""" @@ -552,7 +630,7 @@ def check_positive(value: str) -> int: if __name__ == "__main__": argParser = argparse.ArgumentParser( description="Follow changes (commits) in CVEProject / cvelistV5", - usage="%(prog)s [-haoru4] [-vvvv] [-i s] [-c N] [-w N]", + usage="%(prog)s [-haForu4] [-vvvv] [-i s] [-c N] [-w N]", epilog="Requires git. " "Working directory must be the root of the cvelistV5 repository.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, @@ -564,6 +642,13 @@ def check_positive(value: str) -> int: help="add ansi colors to the output", default=False, ) + argParser.add_argument( + "-F", + "--force", + action="store_true", + help="origin/main hard reset if git pull fails", + default=False, + ) argParser.add_argument( "-o", "--once",