diff --git a/bin/evaluator.py b/bin/evaluator.py index f062167..e24a732 100644 --- a/bin/evaluator.py +++ b/bin/evaluator.py @@ -18,6 +18,7 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))) from common.config import Config from common.db import SQLiteWrapper +from common.db_helpers import persist_content_snapshot, update_url_field from common.utils import is_valid, extract_commands, process_new_session @@ -92,44 +93,74 @@ def search_for_nested_urls(content, src_url): Extract new URLs from downloaded shell script and add them to the DB """ + logger.debug(f"search_for_nested_urls: scanning {len(content)} bytes from {src_url}") try: decoded_content = content.decode("utf-8") if session := extract_commands(decoded_content): + logger.debug(f"search_for_nested_urls: extracted session with {len(session)} command(s) from {src_url}") if new_urls := process_new_session(db, config, session, None, datetime.now(timezone.utc).isoformat(), "URL content", src_url): logger.info(f"{len(new_urls)} new URLs found in a shell script downloaded from {src_url}: {new_urls}") + else: + logger.debug(f"search_for_nested_urls: no new URLs in session from {src_url}") + else: + logger.debug(f"search_for_nested_urls: no commands extracted from {src_url}") except UnicodeDecodeError: + logger.debug(f"search_for_nested_urls: content from {src_url} is not UTF-8 decodable, skipping") return def analyze_content(url): """ - Download content from given URL and check its hash on VirusTotal / MalwareBazaar + Download content from the given URL, store it and check its hash on VirusTotal / MalwareBazaar. + + If the download fails for any reason, the URL is marked as ``status='inactive'``. + If it was the first download attempt and the URL has not been classified by any other + method, it is additionally classified as ``unreachable``. + + :returns: dict containing classification result and metadata. On failure, includes + ``status='inactive'`` and may include ``classification='unreachable'``. """ + logger.debug(f"analyze_content: START {url}") try: with requests.get(url, stream=True, proxies=proxies, timeout=10) as response: + logger.debug(f"analyze_content: {url} -> HTTP {response.status_code}, headers: {dict(response.headers)}") if not response.ok: - return dict(classification="unreachable", classification_reason=f"Status code {response.status_code}") + # Download failed -> URL inactive. + logger.debug(f"analyze_content: {url} unreachable (HTTP {response.status_code})") + return _unreachable_result(url, f"Status code {response.status_code}") if (content_size := response.headers.get('Content-Length')) is None: + logger.debug(f"analyze_content: {url} has no Content-Length header") return dict(classification="unclassified", classification_reason="No content") if (content_size_mb := int(content_size) / (1024 ** 2)) > config.max_file_size: + logger.debug(f"analyze_content: {url} too large ({content_size_mb:.2f} MB > {config.max_file_size} MB)") return dict(classification="unclassified", classification_reason=f"File too large: {content_size_mb:.2f} MB") + logger.debug(f"analyze_content: {url} downloaded {len(response.content)} bytes (declared Content-Length: {content_size})") # Determine file type file_type = "" if "content-type" in response.headers: file_type = response.headers['content-type'].split(";")[0] + logger.debug(f"analyze_content: {url} content-type from header: {file_type}") else: try: file_type = magic.from_buffer(response.content, mime=True) + logger.debug(f"analyze_content: {url} content-type from magic: {file_type}") except Exception as e: logger.debug(f"Couldn't determine file type: {e}") # Search the downloaded content for new URLs if file_type in ["application/x-sh", "application/x-shellscript", "text/plain", "text/x-shellscript", "text/x-sh"]: + logger.debug(f"analyze_content: {url} is a text/shell type, searching for nested URLs") search_for_nested_urls(response.content, url) - sha1 = hashlib.sha1(response.content).hexdigest() + # Persist the downloaded content (deduplicated file storage + snapshot/link/history in DB) + # and capture connection metadata (IPs, HTTP status, response headers). + logger.debug(f"analyze_content: {url} persisting content snapshot to {config.content_storage_path}") + persisted = persist_content_snapshot(db, config.content_storage_path, url, response, response.content, file_type or None) + logger.debug(f"analyze_content: {url} persisted -> sha1={persisted['hash']}, sha256={persisted['latest_content_hash']}, storage_path={persisted['storage_path']}, is_new={persisted['is_new']}") + + sha1 = persisted["hash"] result = dict(hash=sha1, content_size=content_size) if file_type: result.update(file_mime_type=file_type) @@ -137,29 +168,70 @@ def analyze_content(url): # check content hash on MalwareBazaar mb_resp = None try: + logger.debug(f"analyze_content: {url} querying MalwareBazaar for sha1={sha1}") mb_resp = requests.post(config.mb_url, data={'query': 'get_info', 'hash': sha1}, headers={'Auth-Key': config.mb_key}) if mb_resp.json().get('query_status') == 'ok': + logger.debug(f"analyze_content: {url} flagged malicious by MalwareBazaar") result.update(classification="malicious", classification_reason="MB file check") return result + logger.debug(f"analyze_content: {url} not known to MalwareBazaar (status: {mb_resp.json().get('query_status')})") except Exception as e: logger.warning(f"Unexpected response from MalwareBazaar: {mb_resp if mb_resp is not None else e}") # if not found, check content hash on VirusTotal + logger.debug(f"analyze_content: {url} querying VirusTotal for file sha1={sha1}") result.update(**vt_request("file", sha1)) + logger.debug(f"analyze_content: DONE {url} -> {result.get('classification')} ({result.get('classification_reason')})") return result except (requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout): - return dict(classification="unreachable", classification_reason="Connection timeout") + logger.debug(f"analyze_content: {url} connection/read timeout") + return _unreachable_result(url, "Connection timeout") except requests.exceptions.TooManyRedirects: - return dict(classification="unreachable", classification_reason="Too many redirects") + logger.debug(f"analyze_content: {url} too many redirects") + return _unreachable_result(url, "Too many redirects") except requests.exceptions.ConnectionError: - return dict(classification="unreachable", classification_reason="Connection refused") + logger.debug(f"analyze_content: {url} connection refused") + return _unreachable_result(url, "Connection refused") except Exception as e: # this is usually caused by requests.get() trying to parse invalid URLs logger.warning(f"Failed to analyze URL content: {type(e)}: {e}") return dict(classification="unclassified", classification_reason="Internal error") +def _has_stored_sample(url): + """ + Return True when we already stored a content snapshot for the given URL. + """ + + row = db.execute( + "SELECT latest_content_hash FROM urls WHERE url = ?", (url,) + ).fetchone() + if row and row[0]: + return True + row = db.execute( + "SELECT 1 FROM url_content WHERE url = ? LIMIT 1", (url,) + ).fetchone() + return row is not None + + +def _unreachable_result(url, reason): + """ + Build a result for a failed download. + + Marks the URL as ``status='inactive'``. If no sample was previously stored, + it also flags it as a first attempt. + """ + + result = dict(status="inactive", unreachable=True) + if not _has_stored_sample(url): + result["first_attempt"] = True + + # We don't set classification='unreachable' here anymore; + # it's handled in evaluate_url based on prior classifications. + return result + + def is_blacklisted(url): """ Check if the URL is blacklisted @@ -200,13 +272,25 @@ def check_domain_threshold(url): def evaluate_url(url): """ - 1. Check that the URL is valid - 2. Check if the URL is listed on URLhaus blacklist - 3. Check for entries on VirusTotal - 4. Download and analyze the URL content - - check hash on MalwareBazaar - - check hash on VirusTotal - - search for new URLs in downloaded shell scripts + Evaluate a single URL. + + Flow: + 1. Check that the URL is valid + 2. Apply the per-domain threshold (may delete URLs) + 3. Check the URLhaus blacklist + 4. Check for entries on VirusTotal (URL check) + 5. Download and analyze the URL content (Sample download) + - check hash on MalwareBazaar + - check hash on VirusTotal + - search for new URLs in downloaded shell scripts + + Sample download (step 5) is performed ALWAYS, unless the URL was already + classified as legitimate (``harmless``) AND a sample is already stored. + + If the download fails: + - The URL is marked ``status='inactive'``. + - If it was the first download attempt AND no prior classification was + found in steps 3-4, it is classified as ``unreachable``. """ result = dict(evaluated="yes", eval_later="no") @@ -225,24 +309,48 @@ def evaluate_url(url): logger.debug("Checking evaluation blacklist") if is_blacklisted(url): result.update(classification="malicious", classification_reason="Blacklist check") - return result + #return result logger.debug("Not found") logger.debug("Checking VirusTotal") url_id = urlsafe_b64encode(url.encode()).decode().strip("=") result.update(**vt_request("URL", url_id)) - if result.get("classification") != "unclassified": + + # Always attempt to download the URL content, regardless of the classification + # produced by the earlier (non-content) methods. The only exception is a URL + # already classified as legitimate ("harmless") for which a sample is already + # stored — in that case no new sample is re-downloaded. + if result.get("classification") == "harmless" and _has_stored_sample(url): + logger.debug(f"Skipping content download for {url}: classified as legitimate (harmless) and a sample is already stored") return result logger.debug("Checking content hash") cls = analyze_content(url) + if cls.get("classification_reason") == "VT limit exceeded": logger.debug(f"URL {url} will be re-evaluated after VirusTotal rate limit is reset") result.update(evaluated="no", eval_later="yes") else: - if cls.get("classification_reason") == "No entry": - cls.update(**result) - result.update(**cls) + failed = bool(cls.get("unreachable")) + first_attempt = cls.pop("first_attempt", False) + + if failed: + result["status"] = "inactive" + # If it's the first attempt and we have no classification yet, mark as unreachable + if first_attempt and result.get("classification") in ("unclassified", None): + result.update(classification="unreachable", classification_reason=cls.get("classification_reason", "Download failed")) + + # Remove internal flag + cls.pop("unreachable", None) + + # If analyze_content provided a classification, it takes precedence over URL-level checks + # unless it's just "No entry" + if cls.get("classification_reason") != "No entry": + result.update(**cls) + elif not failed: + # If it didn't fail but found nothing, we still keep the URL-level results + result.update(**cls) + return result @@ -305,12 +413,20 @@ def sigint_handler(signum, frame): logger.info("Started") running_flag = True while running_flag: - url = db.execute("SELECT url FROM urls WHERE evaluated = 'no'" + (" AND eval_later = 'no'" if vt_daily_quota_exceeded else "") + " LIMIT 1;").fetchone() - if not url: + # Pick the newest URL that hasn't been evaluated yet. + query = "SELECT url FROM urls WHERE evaluated = 'no'" + if vt_daily_quota_exceeded: + query += " AND eval_later = 'no'" + + query += " ORDER BY COALESCE(first_seen, '1970-01-01') DESC LIMIT 1" + + row = db.execute(query).fetchone() + if not row: logger.debug("No URLs to check, sleeping for 10 seconds") time.sleep(10) continue - url = url[0] + + url = row[0] try: logger.debug(f"Evaluating {url}") @@ -318,18 +434,22 @@ def sigint_handler(signum, frame): continue logger.info(f"URL {url} was classified as {result['classification']}, reason: {result['classification_reason']}") - # Update DB record - items = list(result.items()) - set_clause = ", ".join([f"{k} = ?" for k, _ in items]) - params = tuple(v for _, v in items) + (url,) - db.execute(f"UPDATE urls SET {set_clause} WHERE url = ?", params) + # Update DB record — use update_url_field per field so that + # classification/classification_reason/note changes are recorded + # in classification_history (changed_by="system"). + for field, value in result.items(): + update_url_field(db, url, field, value, changed_by="system") # If the URL was classified as malicious, mark all source URLs that led to it as malicious if result["classification"] == "malicious": rows = db.execute("SELECT urls.url FROM discovered_urls AS s JOIN urls ON urls.url = s.src_url WHERE s.url = ? AND urls.classification != 'malicious'", (url,)).fetchall() - if src_urls := ", ".join(f"'{row[0]}'" for row in rows): - db.execute(f"UPDATE urls SET classification = 'malicious', classification_reason = 'Downloading from malicious URL' WHERE url IN ({src_urls})") - logger.info(f"URLs {src_urls} were classified as malicious because they downloaded content from a malicious URL ({url})") + if rows: + src_url_list = [] + for (src_url,) in rows: + update_url_field(db, src_url, "classification", "malicious", changed_by="system") + update_url_field(db, src_url, "classification_reason", "Downloading from malicious URL", changed_by="system") + src_url_list.append(src_url) + logger.info(f"URLs {', '.join(src_url_list)} were classified as malicious because they downloaded content from a malicious URL ({url})") except Exception as e: logger.exception(f"Error while evaluating URL {url}: {type(e)}: {e}") diff --git a/bin/honeynetasia2evaluator.py b/bin/honeynetasia2evaluator.py index aa829e8..b77d5eb 100644 --- a/bin/honeynetasia2evaluator.py +++ b/bin/honeynetasia2evaluator.py @@ -13,7 +13,7 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))) from common.config import Config from common.db import SQLiteWrapper -from common.utils import is_valid, get_domain +from common.utils import is_valid, get_domain, record_url_source def honeynetasia2evaluator(): @@ -39,7 +39,8 @@ def honeynetasia2evaluator(): last_seen = excluded.last_seen, occurrences = urls.occurrences + 1; """, (url, current_date, current_date, get_domain(url))).rowcount - db.execute("INSERT OR IGNORE INTO url_source (url, source) VALUES (?, ?)", (url, "HoneyNet.Asia")) + # Record/update per-source observation statistics (first/last seen, occurrences) + record_url_source(db, url, "HoneyNet.Asia", date=current_date) logger.info(f"{num_inserted} URLs inserted or updated") logger.info("Job finished") diff --git a/common/content_storage.py b/common/content_storage.py new file mode 100644 index 0000000..80d5636 --- /dev/null +++ b/common/content_storage.py @@ -0,0 +1,111 @@ +""" +File-system content storage for URL Evaluator. + +Binary payloads (downloaded URL content / malware samples) are stored on +disk, deduplicated by their SHA-256 digest. The database only keeps metadata +plus the relative ``storage_path`` produced here, so identical content served +by multiple URLs results in a single on-disk file. + +Layout: + ///.blob + +where ``aa``/``bb`` are the first two pairs of hex characters of the digest. +Files are immutable — the same content is never written twice. +""" + +import os +import hashlib +import logging +from pathlib import Path + +logger = logging.getLogger(__name__) + +BLOB_SUFFIX = ".blob" + + +def compute_hashes(content: bytes): + """Return (sha1, sha256) hex digests of *content*.""" + sha1 = hashlib.sha1(content).hexdigest() + sha256 = hashlib.sha256(content).hexdigest() + return sha1, sha256 + + +def storage_path(base_dir, content_hash: str) -> Path: + """Return the absolute :class:`Path` where *content_hash* would be stored. + + The path is derived purely from the hash, so it is stable and independent + of when/by whom the content was downloaded. + """ + rel = relative_storage_path(content_hash) + return Path(base_dir) / rel + + +def relative_storage_path(content_hash: str) -> str: + """Return the relative storage path (``aa/bb/.blob``) for *content_hash*.""" + if not content_hash or len(content_hash) < 4: + raise ValueError(f"content_hash too short to build storage path: {content_hash!r}") + return os.path.join(content_hash[0:2], content_hash[2:4], content_hash + BLOB_SUFFIX) + + +def content_exists(base_dir, content_hash: str) -> bool: + """Return True if a blob for *content_hash* already exists on disk.""" + return storage_path(base_dir, content_hash).is_file() + + +def save_content(base_dir, content: bytes): + """Persist *content* under *base_dir*, deduplicated by SHA-256. + + :param base_dir: base storage directory (from config ``content_storage_path``) + :param content: raw downloaded bytes + :returns: ``(sha256, sha1, relative_path, is_new)`` where ``is_new`` is True + when a file was actually written (False on dedup-hit). + """ + logger.debug(f"save_content: hashing {len(content)} bytes") + sha1, sha256 = compute_hashes(content) + dest = storage_path(base_dir, sha256) + rel = relative_storage_path(sha256) + logger.debug(f"save_content: sha256={sha256}, target={dest}") + + if dest.is_file(): + logger.debug(f"Content deduplicated (already stored): {sha256}") + return sha256, sha1, rel, False + + dest.parent.mkdir(parents=True, exist_ok=True) + # Write atomically: tmp file then rename, so concurrent readers never see a + # partially-written blob. + tmp = dest.with_suffix(dest.suffix + ".tmp") + logger.debug(f"save_content: writing tmp file {tmp}") + with open(tmp, "wb") as fh: + fh.write(content) + os.replace(tmp, dest) + logger.info(f"Stored new content blob: {rel} ({len(content)} bytes)") + logger.debug(f"save_content: atomically moved to {dest}") + return sha256, sha1, rel, True + + +def load_content(base_dir, content_hash: str) -> bytes: + """Read and return the stored bytes for *content_hash*. + + :raises FileNotFoundError: if no blob exists for the given hash. + """ + path = storage_path(base_dir, content_hash) + with open(path, "rb") as fh: + return fh.read() + + +def delete_content(base_dir, content_hash: str) -> bool: + """Remove the blob for *content_hash* if present. Returns True when removed. + + Prune any now-empty parent directories as well. + """ + path = storage_path(base_dir, content_hash) + if not path.is_file(): + return False + path.unlink() + # Try to clean up the (now possibly empty) aa/bb directories. + for parent in (path.parent, path.parent.parent): + try: + parent.rmdir() + except OSError: + pass + return True diff --git a/common/db_helpers.py b/common/db_helpers.py new file mode 100644 index 0000000..e6dd667 --- /dev/null +++ b/common/db_helpers.py @@ -0,0 +1,232 @@ +""" +Shared database helper functions for URL Evaluator. + +These helpers own all writes to the history-tracking and content tables so +individual backend modules don't duplicate SQL: + +- :func:`record_url_history` – append-on-change audit entries (``url_history``) +- :func:`update_url_field` – update a ``urls`` column and record history +- :func:`set_url_latest_content` – flip the ``url_content.is_latest`` marker +- :func:`persist_content_snapshot` – save a downloaded payload to disk (dedup) + and persist ``content_snapshot`` + ``url_content`` + ``urls`` metadata. + +Used by the evaluator, the web edit handlers and ingestion modules. +""" + +import json +import logging +from datetime import datetime, timezone + +from common.content_storage import save_content + +logger = logging.getLogger(__name__) + +FETCH_IP_TIMEOUT = 5 + +# Columns of the ``urls`` table that update_url_field() is allowed to change. +# Business columns only – primary key and auto-tracked timestamps are excluded. +URL_UPDATABLE_FIELDS = { + "hash", + "classification", + "classification_reason", + "note", + "reported", + "occurrences", + "vt_stats", + "evaluated", + "file_mime_type", + "content_size", + "threat_label", + "status", + "last_active", + "status_changed", + "last_edit", + "eval_later", + "domain", + "latest_content_hash", +} + + +def _now(): + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + + +def record_url_history(db, url, field, old_value, new_value, changed_by="system"): + """Append a change record to ``url_history`` when a value actually changed. + + No row is written when ``old_value == new_value`` so the history stays free + of no-op noise. + """ + if old_value == new_value: + return False + db.execute( + """ + INSERT INTO url_history (url, changed_at, field, old_value, new_value, changed_by) + VALUES (?, ?, ?, ?, ?, ?) + """, + (url, _now(), field, old_value, new_value, changed_by), + ) + + # Also record in classification_history if the field was classification, classification_reason, or note + if field in ("classification", "classification_reason", "note"): + # For classification history, capture a snapshot of all three fields so + # the UI can clearly separate what is the reason and what is the note. + row = db.execute("SELECT classification, classification_reason, note FROM urls WHERE url = ?", (url,)).fetchone() + if row: + curr_class, curr_reason, curr_note = row + db.execute( + "INSERT INTO classification_history (url, changed_at, classification, reason, note, changed_by) VALUES (?, ?, ?, ?, ?, ?)", + (url, _now(), curr_class, (curr_reason or "").strip(), (curr_note or "").strip(), changed_by), + ) + return True + + +def update_url_field(db, url, field, new_value, changed_by="system"): + """Update a single ``urls`` column and record the change in history. + + :returns: True when the value changed (and history was written), False when + the new value equals the stored one (no update performed). + :raises ValueError: when *field* is not an updatable ``urls`` column — this + also protects against SQL injection via the column name. + """ + if field not in URL_UPDATABLE_FIELDS: + raise ValueError(f"Invalid URL field: {field}") + + row = db.execute(f"SELECT {field} FROM urls WHERE url = ?", (url,)).fetchone() + old_value = row[0] if row else None + + if old_value == new_value: + return False + + db.execute(f"UPDATE urls SET {field} = ? WHERE url = ?", (new_value, url)) + record_url_history(db, url, field, old_value, new_value, changed_by=changed_by) + return True + + +def set_url_latest_content(db, url, content_hash): + """Mark *content_hash* as the current content of *url* in ``url_content``. + + An existing (url, content_hash) row only gets ``is_latest='yes'`` and a + bumped ``last_seen``; a new one is created with ``first_seen == last_seen``. + All other rows for the URL are flipped to ``is_latest='no'``. + """ + now = _now() + logger.debug(f"set_url_latest_content: marking {content_hash[:16]}... as latest for {url}") + + existing = db.execute( + "SELECT id FROM url_content WHERE url = ? AND content_hash = ?", + (url, content_hash), + ).fetchone() + + if existing: + logger.debug(f"set_url_latest_content: updating existing url_content row id={existing[0]}") + db.execute( + "UPDATE url_content SET last_seen = ?, is_latest = 'yes' WHERE id = ?", + (now, existing[0]), + ) + else: + logger.debug(f"set_url_latest_content: creating new url_content row for {url}") + db.execute( + "INSERT INTO url_content (url, content_hash, first_seen, last_seen, is_latest) VALUES (?, ?, ?, ?, 'yes')", + (url, content_hash, now, now), + ) + + db.execute( + "UPDATE url_content SET is_latest = 'no' WHERE url = ? AND content_hash != ?", + (url, content_hash), + ) + logger.debug(f"set_url_latest_content: other content rows for {url} marked is_latest='no'") + + +def _extract_connection_ips(response): + """Best-effort extraction of (source_ip, server_ip) from a requests response. + + Reads the underlying urllib3 connection socket. Returns ``(None, None)`` + when the information isn't available (e.g. mocked responses in tests). + """ + source_ip = server_ip = None + try: + sock = response.raw._connection.sock + if sock is not None: + local = sock.getsockname() + peer = sock.getpeername() + if local: + source_ip = local[0] + if peer: + server_ip = peer[0] + logger.debug(f"_extract_connection_ips: source_ip={source_ip}, server_ip={server_ip}") + else: + logger.debug("_extract_connection_ips: underlying socket is None") + except Exception: + pass + return source_ip, server_ip + + +def persist_content_snapshot(db, base_dir, url, response, content, mime_type): + """Store a downloaded payload and persist its metadata + history link. + + Steps: + 1. Write *content* to the file storage (SHA-256 dedup). + 2. Insert-or-ignore a ``content_snapshot`` row (one per unique hash). + 3. Refresh the ``url_content`` link and mark it latest. + 4. Update ``urls.hash``/``latest_content_hash``/``file_mime_type``/``content_size`` + and record a ``latest_content_hash`` change in ``url_history``. + + :param base_dir: content storage base directory (config ``content_storage_path``) + :param url: URL the content was downloaded from + :param response: the ``requests`` response object (for status/headers/IPs) + :param content: raw downloaded bytes + :param mime_type: detected MIME type of the content + :returns: dict with the new ``hash`` (sha1), ``latest_content_hash`` (sha256), + ``file_mime_type``, ``content_size`` and ``storage_path``. + """ + logger.debug(f"persist_content_snapshot: persisting {len(content)} bytes for {url} (base_dir={base_dir}, mime={mime_type})") + sha256, sha1, rel_path, _is_new = save_content(base_dir, content) + logger.debug(f"persist_content_snapshot: saved -> sha256={sha256}, sha1={sha1}, path={rel_path}, is_new={_is_new}") + + http_status = getattr(response, "status_code", None) + headers = getattr(response, "headers", {}) or {} + try: + http_headers = json.dumps(dict(headers)) + except (TypeError, ValueError): + http_headers = json.dumps({str(k): str(v) for k, v in headers.items()}) + + source_ip, server_ip = _extract_connection_ips(response) + downloaded_at = _now() + + logger.debug(f"persist_content_snapshot: inserting content_snapshot row (hash={sha256[:16]}..., status={http_status}, src={source_ip}, dst={server_ip})") + db.execute( + """ + INSERT INTO content_snapshot + (content_hash, url, downloaded_at, source_ip, server_ip, http_status, + http_headers, mime_type, content_size, storage_path, sha1, sha256) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(content_hash) DO NOTHING + """, + (sha256, url, downloaded_at, source_ip, server_ip, http_status, + http_headers, mime_type, len(content), rel_path, sha1, sha256), + ) + + previous_hash_row = db.execute( + "SELECT latest_content_hash FROM urls WHERE url = ?", (url,) + ).fetchone() + previous_hash = previous_hash_row[0] if previous_hash_row else None + logger.debug(f"persist_content_snapshot: previous latest_content_hash for {url}: {previous_hash}") + + set_url_latest_content(db, url, sha256) + + db.execute( + "UPDATE urls SET hash = ?, latest_content_hash = ?, file_mime_type = ?, content_size = ? WHERE url = ?", + (sha1, sha256, mime_type, len(content), url), + ) + record_url_history(db, url, "latest_content_hash", previous_hash, sha256, changed_by="system") + logger.debug(f"persist_content_snapshot: urls row updated + history recorded for {url}") + + return { + "hash": sha1, + "latest_content_hash": sha256, + "file_mime_type": mime_type, + "content_size": len(content), + "storage_path": rel_path, + "is_new": _is_new, + } diff --git a/common/utils.py b/common/utils.py index 652f00d..17d060a 100644 --- a/common/utils.py +++ b/common/utils.py @@ -72,6 +72,129 @@ def get_domain(url: str): return None +def _origin_source_of(db, url): + """ + Resolve the (source, source_detail) of a URL's first observation, used to + derive the original honeynet of a URL extracted from that URL's content. + Dict/tuple-row tolerant. Returns (None, None) when not observed yet. + """ + origin = db.execute( + "SELECT source, source_detail FROM url_source WHERE url = ? ORDER BY observed_at, rowid LIMIT 1", + (url,)).fetchone() + if not origin: + return None, None + if isinstance(origin, dict): + return origin.get("source"), origin.get("source_detail") + return origin[0], origin[1] + + +def record_url_source(db, url, source, date=None, count=1, source_detail=None, origin_url=None, + observed_at=None, idea_id=None, session_hash=None): + """ + Record that a URL was observed in a source (e.g. a honeynet feed). + + Tracks per-source observation statistics in the url_source table: + - first_seen: date of the first observation of the URL in this source + - last_seen: date of the most recent observation (updated to the latest) + - occurrences: how many times the URL was observed in this source + - observed_at: when the URL was first observed (kept at the earliest value) + - source_detail / origin_url / idea_id / session_hash: provenance metadata + + For URLs extracted from a script hosted on another URL (``origin_url``), the + original source (``origin_source`` / ``origin_source_detail``) is resolved + from the origin URL's own first observation, so the original honeynet can be + derived. + + :param db: database wrapper with an execute() method + :param url: observed URL + :param source: name of the source (honeynet) the URL was observed in + :param date: date of the observation (defaults to today, UTC) + :param count: how many observations (occurrences) to add (default 1) + :param source_detail: detail of the source (sensor/node name) + :param origin_url: URL from which this URL was extracted (if any) + :param observed_at: when the URL was observed (defaults to now, UTC) + :param idea_id: IDEA event ID + :param session_hash: hash of the session the URL was observed in + """ + from datetime import datetime, timezone + if date is None: + date = datetime.now(timezone.utc).strftime('%Y-%m-%d') + if observed_at is None: + observed_at = datetime.now(timezone.utc).isoformat() + + # Resolve the original source of a URL extracted from another URL's content, + # so the original honeynet can be derived. Uses the origin URL's first observation. + origin_source = origin_source_detail = None + if origin_url: + origin_source, origin_source_detail = _origin_source_of(db, origin_url) + + # Schema-adaptive column list: the production url_source carries cumulative + # stats columns (first_seen/last_seen/occurrences), but a minimal schema may not. + def _col_name(row): + # PRAGMA table_info row: (cid, name, ...) as tuple, or {'name': ...} with a dict row factory + if isinstance(row, dict): + return row.get("name") + return row[1] + + existing = {_col_name(row) for row in db.execute("PRAGMA table_info(url_source)").fetchall()} + + columns = ["url", "source"] + values = [url, source] + if "first_seen" in existing: + columns += ["first_seen", "last_seen", "occurrences"] + values += [date, date, count] + columns += ["source_detail", "origin_url", "origin_source", "origin_source_detail", + "observed_at", "idea_id", "session_hash"] + values += [source_detail, origin_url, origin_source, origin_source_detail, + observed_at, idea_id, session_hash] + + updates = [] + if "occurrences" in existing: + updates.append("occurrences = url_source.occurrences + excluded.occurrences") + if "last_seen" in existing: + updates.append("last_seen = MAX(url_source.last_seen, excluded.last_seen)") + updates.append("observed_at = MIN(url_source.observed_at, excluded.observed_at)") + + placeholders = ", ".join("?" * len(columns)) + db.execute( + f""" + INSERT INTO url_source ({", ".join(columns)}) + VALUES ({placeholders}) + ON CONFLICT(url, source) DO UPDATE SET + {", ".join(updates)}; + """, + values) + + +def record_discovered_url(db, url, src_url, discovered_at=None): + """ + Record that a URL was found in the content of another URL. + + Stores the link in discovered_urls, caching the original source of the + ``src_url`` (the URL hosting the script this URL was extracted from), so the + original honeynet can be derived. + + :param db: database wrapper with an execute() method + :param url: the discovered (extracted) URL + :param src_url: the URL in whose content ``url`` was found + :param discovered_at: when the URL was discovered (defaults to now, UTC) + """ + from datetime import datetime, timezone + if discovered_at is None: + discovered_at = datetime.now(timezone.utc).isoformat() + + # Cache the original source of the src_url (its first observation). + origin_source, origin_source_detail = _origin_source_of(db, src_url) + + db.execute( + """ + INSERT INTO discovered_urls (url, src_url, discovered_at, origin_source, origin_source_detail) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(url, src_url) DO NOTHING; + """, + (url, src_url, discovered_at, origin_source, origin_source_detail)) + + def process_new_session(db, config, session, idea_id, detect_time, source, source_url): """ Process a new session: @@ -103,9 +226,17 @@ def process_new_session(db, config, session, idea_id, detect_time, source, sourc ) for url, occurrences in Counter(extracted_urls).items(): db.execute("INSERT OR IGNORE INTO url_session (url, session) VALUES (?, ?)", (url, session_hash)) - db.execute("INSERT OR IGNORE INTO url_source (url, source) VALUES (?, ?)", (url, source)) + # Record/update per-source observation statistics (first/last seen, occurrences, provenance) + record_url_source( + db, url, source, + date=date, + source_detail=source_url, + origin_url=source_url, + observed_at=detect_time, + idea_id=idea_id, + session_hash=session_hash) if source_url: - db.execute("INSERT OR IGNORE INTO discovered_urls (url, src_url) VALUES (?, ?)", (url, source_url)) + record_discovered_url(db, url, source_url, discovered_at=detect_time) db.execute( """ INSERT INTO urls (url, first_seen, last_seen, domain) VALUES (?, ?, ?, ?) diff --git a/etc/config.yaml b/etc/config.yaml index b9490a8..e53e028 100644 --- a/etc/config.yaml +++ b/etc/config.yaml @@ -37,6 +37,15 @@ max_age_invalid: 7 # days # Max size of downloaded content max_file_size: 100 # MB +# Base directory for downloaded content snapshots (deduplicated file storage) +content_storage_path: "/data/url_evaluator/content" + +# Optional soft limit / warning threshold for total content storage (GB); 0 disables +max_storage_size_gb: 0 + +# Placeholder list of sandbox providers for future integration +sandbox_providers: [] + # How often should evaluation blacklist be updated bl_update_time: 15 # minutes diff --git a/install/create_db.sql b/install/create_db.sql index 3df10a2..2db8fef 100644 --- a/install/create_db.sql +++ b/install/create_db.sql @@ -16,18 +16,31 @@ CREATE TABLE url_session CREATE TABLE url_source ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - url TEXT REFERENCES urls(url), - source TEXT, + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT REFERENCES urls(url), + source TEXT, + first_seen DATE, + last_seen DATE, + occurrences INTEGER DEFAULT 1, + source_detail TEXT, + origin_url TEXT, + origin_source TEXT, + origin_source_detail TEXT, + observed_at DATETIME, + idea_id TEXT, + session_hash TEXT REFERENCES sessions(session_hash), CONSTRAINT url_source_unique UNIQUE (url, source) ); CREATE TABLE discovered_urls ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - url TEXT REFERENCES urls(url), - src_url TEXT REFERENCES urls(url), + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT REFERENCES urls(url), + src_url TEXT REFERENCES urls(url), + discovered_at DATETIME, + origin_source TEXT, + origin_source_detail TEXT, CONSTRAINT discovered_urls_unique UNIQUE (url, src_url) ); @@ -53,5 +66,90 @@ CREATE TABLE urls status_changed TEXT DEFAULT 'no' CHECK (status_changed IN ('yes', 'no')), last_edit TEXT, eval_later TEXT DEFAULT 'no' CHECK (eval_later IN ('yes', 'no')), - domain TEXT + domain TEXT, + latest_content_hash TEXT +); + +-- One row per unique downloaded content (deduplicated by SHA-256). +-- Binary payload lives on disk under ///.blob +-- this table only keeps metadata + the storage pointer. +CREATE TABLE content_snapshot +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + content_hash TEXT UNIQUE, + url TEXT, + downloaded_at DATETIME, + source_ip TEXT, + server_ip TEXT, + http_status INTEGER, + http_headers TEXT, + mime_type TEXT, + content_size INTEGER, + storage_path TEXT, + sha1 TEXT, + sha256 TEXT +); + +-- Many-to-many between URLs and content snapshots, carrying per-URL +-- first/last seen timestamps and the "current content" marker. +-- When a URL's content changes, a new row is inserted with is_latest='yes' +-- and previous rows flip to is_latest='no'. Unchanged content just bumps +-- last_seen, giving the UI the "merged" view. +CREATE TABLE url_content +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT REFERENCES urls(url), + content_hash TEXT REFERENCES content_snapshot(content_hash), + first_seen DATETIME, + last_seen DATETIME, + is_latest TEXT CHECK (is_latest IN ('yes', 'no')), + + CONSTRAINT url_content_unique UNIQUE (url, content_hash) +); + +-- Audit trail of changes to URL fields (classification, status, note, latest content, ...). +CREATE TABLE url_history +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT REFERENCES urls(url), + changed_at DATETIME, + field TEXT, + old_value TEXT, + new_value TEXT, + changed_by TEXT ); + +-- Prepared for future sandbox integration; UI exposes a "request analysis" stub. +-- mime_type + content_size snapshot the metadata of the submitted content at +-- the time of submission so the UI can display exactly what was sent, even if +-- the underlying content_snapshot row is later superseded for the URL. +CREATE TABLE sandbox_job +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + content_hash TEXT REFERENCES content_snapshot(content_hash), + url TEXT REFERENCES urls(url), + provider TEXT, + external_id TEXT, + status TEXT CHECK (status IN ('pending', 'running', 'completed', 'failed')), + submitted_at DATETIME, + completed_at DATETIME, + report_url TEXT, + report_json TEXT, + requested_by TEXT, + mime_type TEXT, + content_size INTEGER +); + +-- ---------------------------------------------------------------------------- +-- Indexes for common lookup patterns +-- ---------------------------------------------------------------------------- +CREATE INDEX idx_url_source_lookup ON url_source(url, source, observed_at); +CREATE INDEX idx_url_source_origin ON url_source(origin_url); +CREATE INDEX idx_url_source_session ON url_source(session_hash); +CREATE INDEX idx_url_content_latest ON url_content(url, is_latest); +CREATE INDEX idx_url_content_hash ON url_content(content_hash); +CREATE INDEX idx_url_history_url ON url_history(url, changed_at); +CREATE INDEX idx_content_snapshot_sha1 ON content_snapshot(sha1); +CREATE INDEX idx_sandbox_job_status ON sandbox_job(content_hash, status); +CREATE INDEX idx_sandbox_job_url ON sandbox_job(url); +CREATE INDEX idx_discovered_urls_src ON discovered_urls(src_url); diff --git a/web/main.py b/web/main.py index b6b1278..d2b3486 100644 --- a/web/main.py +++ b/web/main.py @@ -7,9 +7,11 @@ import argparse import logging import base64 +import json +import io from datetime import datetime, timezone -from flask import Flask, jsonify, render_template, make_response, redirect, url_for +from flask import Flask, jsonify, render_template, make_response, redirect, url_for, send_file, abort from werkzeug.exceptions import BadRequestKeyError from pymisp import PyMISP, PyMISPError @@ -18,6 +20,7 @@ from common.config import Config from common.db import SQLiteWrapper from common.utils import is_valid, get_domain +from common.content_storage import load_content # Global variables page = 1 @@ -245,10 +248,51 @@ def __init__(self, url_detail): self.last_active = url_detail[15] self.last_edit = url_detail[16] self.eval_later = url_detail[17] + self.latest_content_hash = url_detail[18] self.ip = get_ip(self.url) self.src = [] + # per-source observation rows for the Sources tab: + # (source, first_seen, last_seen, occurrences, derived_from) + self.source_rows = [] self.src_urls = [] self.contained_urls = [] + # content history rows for the Content tab (dicts; see detail()) + self.content_rows = [] + # sandbox jobs for the Sandbox tab (dicts keyed on the sandbox_job row) + self.sandbox_jobs = [] + # set of content hashes that already have a sandbox job (used by the + # Content tab to mark rows that have any associated sandbox analysis) + self.content_hashes_with_sandbox = set() + + +def get_detail_menu(url, show=None, counts=None): + """Build the tab menu displayed under the URL on the detail page. + + :param url: the URL whose detail page it is + :param show: optional "show" filter to keep in tab links + :param counts: optional dict mapping tab id -> badge count; tabs without + an entry in the dict display no badge + """ + counts = counts or {} + tabs = [ + ("overview", "Overview"), + ("content", "Content"), + ("sources", "Sources"), + ("sandbox", "Sandbox"), + ("class_history", "Class. History"), + ] + menu = [] + for tab_id, label in tabs: + params = {"url": url, "tab": tab_id} + if show: + params["show"] = show + menu.append({ + "id": tab_id, + "label": label, + "count": counts.get(tab_id), + "href": url_for("detail", **params), + }) + return menu @app.route('/detail', methods=['GET', 'POST']) @@ -256,6 +300,7 @@ def detail(): user = get_user(flask.request.environ) show = flask.request.args.get('show') url = flask.request.args.get('url') + active_tab = flask.request.args.get('tab', 'overview') with SQLiteWrapper(config.db_path) as db: if flask.request.method == 'POST': @@ -263,8 +308,89 @@ def detail(): return redirect(url_for('detail', url=url)) # get url details - url_detail = URLDetail(db.execute("SELECT url, first_seen, last_seen, hash, classification, classification_reason, note, reported, occurrences, vt_stats, evaluated, file_mime_type, content_size, threat_label, status, last_active, last_edit, eval_later FROM urls WHERE url = ? LIMIT 1", (url,)).fetchone()) + url_detail = URLDetail(db.execute("SELECT url, first_seen, last_seen, hash, classification, classification_reason, note, reported, occurrences, vt_stats, evaluated, file_mime_type, content_size, threat_label, status, last_active, last_edit, eval_later, latest_content_hash FROM urls WHERE url = ? LIMIT 1", (url,)).fetchone()) + + # classification history for the Class. History tab; + # `reason` and `note` are stored separately so the UI can show each distinctly + class_history = db.execute( + "SELECT changed_at, classification, reason, note, changed_by FROM classification_history WHERE url = ? ORDER BY changed_at DESC", + (url,), + ).fetchall() + + # Content history for the Content tab (newest first), with per-URL first/last seen + snapshots = db.execute(""" + SELECT uc.content_hash, uc.first_seen, uc.last_seen, uc.is_latest, + cs.downloaded_at, cs.http_status, cs.mime_type, cs.content_size, + cs.storage_path, cs.sha1, cs.http_headers + FROM url_content uc + JOIN content_snapshot cs ON cs.content_hash = uc.content_hash + WHERE uc.url = ? + ORDER BY uc.first_seen DESC + """, (url,)).fetchall() + + prev_hash = None + for content_hash, first_seen, last_seen, is_latest, downloaded_at, http_status, mime, csize, storage_path, sha1, http_headers in snapshots: + row = { + "hash": content_hash, + "first_seen": first_seen, + "last_seen": last_seen, + "downloaded_at": downloaded_at, + "http_status": http_status, + "mime": mime, + "size": csize, + "path": storage_path, # serves as the download link target + "sha1": sha1, + "is_latest": is_latest == "yes", + "headers": json.loads(http_headers) if http_headers else None, + } + # Non-latest rows are "changed" when superseded by different (newer) content + row["changed"] = not row["is_latest"] and prev_hash is not None and prev_hash != content_hash + prev_hash = content_hash + url_detail.content_rows.append(row) + # Sandbox jobs for the Sandbox tab (newest first). The LEFT JOIN pulls + # the snapshot metadata as a fallback when the job hasn't snapped it + # into its own mime_type/content_size columns yet (legacy rows). + sandbox_rows = db.execute(""" + SELECT sj.id, sj.content_hash, sj.provider, sj.external_id, sj.status, + sj.submitted_at, sj.completed_at, sj.report_url, sj.report_json, + sj.requested_by, + COALESCE(sj.mime_type, cs.mime_type) AS mime_type, + COALESCE(sj.content_size, cs.content_size) AS content_size + FROM sandbox_job sj + LEFT JOIN content_snapshot cs ON cs.content_hash = sj.content_hash + WHERE sj.url = ? + ORDER BY sj.submitted_at DESC, sj.id DESC + """, (url,)).fetchall() + for row in sandbox_rows: + url_detail.sandbox_jobs.append({ + "id": row[0], + "content_hash": row[1], + "provider": row[2] or "PSNC Sandbox", + "external_id": row[3], + "status": row[4] or "pending", + "submitted_at": row[5], + "completed_at": row[6], + "report_url": row[7], + "report_json": json.loads(row[8]) if row[8] else None, + "requested_by": row[9], + "mime_type": row[10], + "content_size": row[11], + }) + url_detail.content_hashes_with_sandbox = { + j["content_hash"] for j in url_detail.sandbox_jobs if j["content_hash"] + } + url_detail.src = [row[0] for row in db.execute("SELECT source FROM url_source WHERE url = ?", (url,)).fetchall()] + # per-source observation stats for the Sources tab (source, first_seen, last_seen, occurrences, derived_from) + # "derived_from" is the source URL this URL was extracted from (discovered_urls), + # which allows deriving the original source for URLs extracted from a script hosted on another URL + url_detail.source_rows = db.execute(""" + SELECT us.source, us.first_seen, us.last_seen, us.occurrences, + (SELECT du.src_url FROM discovered_urls du WHERE du.url = us.url LIMIT 1) AS derived_from + FROM url_source us + WHERE us.url = ? + ORDER BY us.source + """, (url,)).fetchall() url_detail.src_urls = db.execute("SELECT src_url FROM discovered_urls WHERE url = ?", (url_detail.url,)).fetchall() url_detail.contained_urls = db.execute("SELECT url FROM discovered_urls WHERE src_url = ?", (url,)).fetchall() sessions = db.execute("SELECT sessions.session, sessions.idea_id FROM sessions JOIN url_session ON url_session.session=sessions.session_hash WHERE url_session.url = ?", (url,)).fetchall() @@ -294,7 +420,15 @@ def detail(): "joe-sandbox": f"https://www.joesandbox.com/analysis/search?q={url_detail.hash}" } - return render_template('detail.html', user=user, url=url_detail, sessions=sessions, show=show, links=links, inactive_for=inactive_for) + # tab menu under the URL name; badge counts reflect real DB data where available + menu = get_detail_menu(url, show, counts={ + "content": len(url_detail.content_rows), + "sources": len(url_detail.source_rows), + "sandbox": len(url_detail.sandbox_jobs), + "class_history": len(class_history), + }) + + return render_template('detail.html', user=user, url=url_detail, sessions=sessions, show=show, links=links, inactive_for=inactive_for, menu=menu, active_tab=active_tab, class_history=class_history) @app.route('/edit_detail', methods=['GET', 'POST']) @@ -309,7 +443,15 @@ def edit_detail(): classification = flask.request.form['class'] reason = flask.request.form['reason'] evaluated = "yes" if classification != "unclassified" else "no" - db.execute("UPDATE urls SET note = ?, classification = ?, classification_reason = ?, last_edit = ?, evaluated = ? WHERE url = ?", (note, classification, reason, user, evaluated, url)) + # Use update_url_field to ensure record_url_history is called for each changed field + # this will trigger the insertion into classification_history table + from common.db_helpers import update_url_field + update_url_field(db, url, "note", note, changed_by=user) + update_url_field(db, url, "classification", classification, changed_by=user) + update_url_field(db, url, "classification_reason", reason, changed_by=user) + + # Update last_edit and evaluated separately as they might not be in URL_UPDATABLE_FIELDS or need different handling + db.execute("UPDATE urls SET last_edit = ?, evaluated = ? WHERE url = ?", (user, evaluated, url)) if classification == "malicious": back_propagation(db, url) return redirect(url_for("list_all", show=show)) @@ -345,18 +487,42 @@ def bulk_edit_action(): evaluated = "yes" if classification != "unclassified" else "no" urls_string = "('" + "', '".join(selected_urls) + "')" with SQLiteWrapper(config.db_path) as db: - if note: - db.execute(f"UPDATE urls SET note = ?, last_edit = ?, evaluated = ? WHERE url IN {urls_string}", (note, user, evaluated)) - if classification: - db.execute(f"UPDATE urls SET classification = ?, last_edit = ?, evaluated = ? WHERE url IN {urls_string}", (classification, user, evaluated)) - if classification_reason: - db.execute(f"UPDATE urls SET classification_reason = ?, last_edit = ?, evaluated = ? WHERE url IN {urls_string}", (classification_reason, user, evaluated)) + from common.db_helpers import update_url_field + for url in selected_urls: + if note: + update_url_field(db, url, "note", note, changed_by=user) + if classification: + update_url_field(db, url, "classification", classification, changed_by=user) + if classification_reason: + update_url_field(db, url, "classification_reason", classification_reason, changed_by=user) + + db.execute("UPDATE urls SET last_edit = ?, evaluated = ? WHERE url = ?", (user, evaluated, url)) if classification == "malicious": for url in selected_urls: back_propagation(db, url) return redirect(url_for("list_all")) +@app.route('/api/search_url', methods=['GET']) +def api_search_url(): + """Lightweight live search endpoint — returns up to 15 URLs matching the + query substring. Used by the quick-search input in the top panel.""" + q = (flask.request.args.get('q') or '').strip() + if not q: + return make_response(jsonify({'results': []}), 200) + # escape LIKE wildcards in user input so searching for e.g. "%" doesn't + # blow up into a full table scan + q_escaped = q.replace('\\', '\\\\').replace('%', '\\%').replace('_', '\\_') + with SQLiteWrapper(config.db_path) as db: + rows = db.execute( + "SELECT url, classification, status FROM urls WHERE url LIKE ? ESCAPE '\\' LIMIT 15", + (f"%{q_escaped}%",) + ).fetchall() + return make_response(jsonify({'results': [ + {'url': r[0], 'classification': r[1], 'status': r[2]} for r in rows + ]}), 200) + + @app.route('/api/url_stats', methods=['GET']) def api_url_stats(): try: @@ -387,3 +553,52 @@ def api_url_stats(): "src": ", ".join([s[0] for s in url_sources]), } return make_response(jsonify(return_dict), 200) + + +@app.route('/content/download', methods=['GET']) +def download_content(): + """Serve a stored content blob by its SHA-256 hash.""" + content_hash = flask.request.args.get('hash') + if not content_hash: + abort(404) + try: + data = load_content(config.content_storage_path, content_hash) + except FileNotFoundError: + abort(404) + return send_file(io.BytesIO(data), download_name=content_hash[:16], as_attachment=True) + + +@app.route('/sandbox/request', methods=['POST']) +def request_sandbox(): + """Record a sandbox analysis request for a content snapshot. + + Copies the snapshot's mime_type/content_size onto the job so the Sandbox + tab can show exactly what was submitted even if the underlying snapshot + row is later superseded. + """ + user = get_user(flask.request.environ) + content_hash = flask.request.form.get('hash') or flask.request.args.get('hash') + url = flask.request.form.get('url') or flask.request.args.get('url') + provider = flask.request.form.get('provider') or flask.request.args.get('provider') or 'PSNC Sandbox' + if not content_hash: + abort(404) + with SQLiteWrapper(config.db_path) as db: + snap = db.execute( + "SELECT mime_type, content_size FROM content_snapshot WHERE content_hash = ?", + (content_hash,), + ).fetchone() + mime_type = snap[0] if snap else None + content_size = snap[1] if snap else None + db.execute( + """INSERT INTO sandbox_job + (content_hash, url, provider, status, submitted_at, requested_by, mime_type, content_size) + VALUES (?, ?, ?, 'pending', ?, ?, ?, ?)""", + (content_hash, url, provider, + datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'), + user, mime_type, content_size)) + return redirect(url_for('detail', url=url, tab='sandbox')) + + +if __name__ == '__main__': + app.run(host='127.0.0.1', port=5000, debug=True) + diff --git a/web/static/detail.css b/web/static/detail.css index 9e4f393..f140b14 100755 --- a/web/static/detail.css +++ b/web/static/detail.css @@ -16,6 +16,56 @@ overflow-wrap: anywhere; } +/* --- tab menu under the URL name --- */ +.detail .detail-menu { + margin: 0 20px; + border-bottom: 2px solid #e0e0e0; +} + +.detail .detail-menu ul { + display: flex; + flex-direction: row; + flex-wrap: wrap; + gap: 4px; + list-style: none; + margin: 0; + padding: 0; +} + +.detail .detail-menu-item { + display: inline-flex; + align-items: center; + gap: 8px; + padding: 8px 14px; + margin-bottom: -2px; + color: #818181; + text-decoration: none; + border-bottom: 2px solid transparent; +} + +.detail .detail-menu-item:hover { + color: #00838f; +} + +.detail .detail-menu-item.active { + color: #00acc1; + font-weight: bold; + border-bottom: 2px solid #00acc1; +} + +.detail .detail-menu-badge { + min-width: 20px; + height: 20px; + padding: 0 6px; + line-height: 20px; + text-align: center; + font-size: 12px; + font-weight: normal; + color: white; + background-color: #9e9e9e; + border-radius: 4px; +} + .detail .status { font-weight: bold; } @@ -104,6 +154,127 @@ border-radius: 10px; } +/* --- Class. History tab --- */ +.detail .class-history { + display: flex; + flex-direction: column; + gap: 20px; + padding: 10px 0; +} + +.detail .class-history-item { + position: relative; + padding-left: 30px; +} + +.detail .class-history-item.class-history-grouped { + padding-top: 5px; + padding-bottom: 5px; +} + +.detail .class-history-arrow { + font-weight: bold; + color: #818181; + font-size: small; + padding: 2px 8px; +} + + + +.detail .class-history-item::before { + content: ""; + position: absolute; + left: 0; + top: 0; + bottom: 0; + width: 2px; + background-color: #e0e0e0; +} + +.detail .class-history-item::after { + content: ""; + position: absolute; + left: -4px; + top: 5px; + width: 10px; + height: 10px; + border-radius: 50%; + background-color: #ccc; + border: 2px solid #fff; +} + +/* Color dots based on classification */ +.detail .class-history-item:has(.class-history-class.malicious)::after { + background-color: red; +} +.detail .class-history-item:has(.class-history-class.unclassified)::after { + background-color: blue; +} +.detail .class-history-item:has(.class-history-class.harmless)::after { + background-color: green; +} + +.detail .class-history-meta { + display: flex; + flex-direction: row; + align-items: center; + gap: 10px; + margin-bottom: 5px; + font-family: monospace; +} + +.detail .class-history-date { + color: #818181; + font-size: small; +} + +.detail .class-history-class { + font-weight: bold; + padding: 2px 8px; + border-radius: 4px; + font-size: small; +} + +.detail .class-history-class.malicious { + color: red; +} +.detail .class-history-class.unclassified { + color: blue; +} +.detail .class-history-class.harmless { + color: green; +} + +.detail .class-history-user { + color: #818181; + font-size: small; +} + +.detail .class-history-details { + display: flex; + font-size: small; + padding-left: 20px; + margin-top: 2px; +} + +.detail .class-history-reason { + font-style: italic; + color: #555; +} + +.detail .class-history-note { + color: #777; + font-style: normal; +} + +.detail .reason-label, .detail .note-label { + font-weight: bold; + margin-right: 5px; + color: #333; + min-width: 160px; + display: inline-block; +} + .detail .classification p::before { display: inline-block; content: " "; @@ -209,6 +380,51 @@ padding-left: 20px !important; } +/* --- Sources tab table --- */ +.detail .sources-table { + width: 100%; + border-collapse: collapse; +} + +.detail .sources-table th { + text-align: left; + color: #818181; + font-weight: normal; + padding: 8px 12px 8px 0; + border-bottom: 1px solid #e0e0e0; +} + +.detail .sources-table td { + padding: 10px 12px 10px 0; + border-bottom: 1px solid #eee; + vertical-align: middle; +} + +.detail .sources-table th.num, +.detail .sources-table td.num { + text-align: right; +} + +.detail .sources-table td.empty { + color: #818181; + text-align: center; + padding: 20px 0; +} + +.detail .source-badge { + display: inline-block; + padding: 2px 10px; + background-color: #e0e0e0; + border-radius: 4px; + font-size: small; +} + +.detail .sources-table a.src-url { + text-decoration: underline; + color: grey; + overflow-wrap: anywhere; +} + .detail .list-urls { background-color: #f8f8f8; border: 1px solid #ddd; @@ -217,4 +433,291 @@ max-height: 100px; overflow: scroll; /* margin: 10px; */ +} + +/* --- Content tab table --- */ +.detail .content-table { + width: 100%; + border-collapse: collapse; +} + +.detail .content-table th { + text-align: left; + color: #818181; + font-weight: normal; + padding: 8px 12px 8px 0; + border-bottom: 1px solid #e0e0e0; +} + +.detail .content-table td { + padding: 10px 12px 10px 0; + border-bottom: 1px solid #eee; + vertical-align: middle; +} + +.detail .content-table th.num, +.detail .content-table td.num { + text-align: right; +} + +.detail .content-table td.hash { + font-family: "Lucida Console", "Courier New", monospace; + font-size: smaller; + white-space: nowrap; +} + +.detail .content-table td.empty { + color: #818181; + text-align: center; + padding: 20px 0; +} + +/* File path download link */ +.detail .content-table a.content-download { + text-decoration: underline; + color: #00acc1; + overflow-wrap: anywhere; +} + +/* "content changed" badge + highlighted row */ +.detail .content-table tr.content-changed td { + background-color: #fff4e5; +} + +.detail .badge-changed { + display: inline-block; + padding: 2px 10px; + background-color: #f0a83c; + color: #fff; + border-radius: 12px; + font-size: small; + white-space: nowrap; +} + +/* Collapsible HTTP headers */ +.detail .content-headers-row td { + padding-top: 0; + background-color: #fafafa; +} + +.detail .content-headers summary { + cursor: pointer; + color: #818181; + font-size: small; +} + +.detail .content-headers dl { + margin: 8px 0 0 0; + font-size: small; +} + +.detail .content-headers dl div { + display: flex; + padding: 2px 0; + border-bottom: 1px solid #f0f0f0; +} + +.detail .content-headers dt { + font-weight: bold; + min-width: 220px; + color: #555; + margin-left: 0; +} + +.detail .content-headers dd { + margin: 0; + overflow-wrap: anywhere; +} + +/* --- Sandbox tab --- */ +.detail .sandbox { + display: flex; + flex-direction: column; + gap: 20px; +} + +/* Submit panel */ +.detail .sandbox-submit { + border: 1px solid #e0e0e0; + border-radius: 8px; + padding: 20px; + background-color: #fff; +} + +.detail .sandbox-submit-title { + color: #818181; + margin-bottom: 12px; +} + +.detail .sandbox-submit-buttons { + display: flex; + flex-wrap: wrap; + gap: 25px; + margin-bottom: 12px; +} + +.detail .sandbox-submit-buttons form { + margin: 0; +} + +.detail .sandbox-btn { + padding: 10px 20px; + background-color: transparent; + border: 1px solid #ccc; + border-radius: 6px; + cursor: pointer; + font-size: 14px; + color: #333; +} + +.detail .sandbox-btn:hover { + border-color: #00acc1; + color: #00acc1; +} + +.detail .sandbox-submit-note { + color: #bdbdbd; + font-size: small; +} + +.detail .sandbox-note { + color: #818181; + font-style: italic; +} + +/* Submission cards */ +.detail .sandbox-card { + position: relative; + border: 1px solid #e0e0e0; + border-radius: 8px; + padding: 20px; + background-color: #fff; +} + +.detail .sandbox-card-header { + display: flex; + justify-content: flex-end; + margin-bottom: 10px; +} + +.detail .sandbox-card-title { + position: absolute; + top: 20px; + left: 20px; + font-weight: bold; + color: #333; +} + +.detail .sandbox-status { + display: inline-block; + padding: 4px 12px; + border-radius: 4px; + font-size: small; + font-weight: normal; +} + +.detail .sandbox-status-done { + background-color: #e6f4ea; + color: #1e7e34; +} + +.detail .sandbox-status-pending, +.detail .sandbox-status-running { + background-color: #fff4e5; + color: #92610a; +} + +.detail .sandbox-status-failed { + background-color: #fdecea; + color: #a01f1f; +} + +/* Submission fields */ +.detail .sandbox-card-body { + margin: 30px 0 0 0; /* leave room for .sandbox-card-title */ + display: flex; + flex-direction: column; + gap: 10px; +} + +.detail .sandbox-row { + display: flex; + flex-direction: row; + align-items: baseline; + gap: 30px; +} + +.detail .sandbox-row dt { + min-width: 120px; + color: #818181; + margin: 0; +} + +.detail .sandbox-row dd { + margin: 0; + color: #333; +} + +.detail .sandbox-hash { + text-decoration: none; + color: #333; +} + +.detail .sandbox-hash code { + font-family: "Lucida Console", "Courier New", monospace; + font-size: smaller; + background-color: #f4f4f4; + padding: 2px 6px; + border-radius: 4px; +} + +.detail .sandbox-hash:hover code { + color: #00acc1; +} + +.detail .sandbox-hash-meta { + margin-left: 10px; + color: #818181; + font-size: small; +} + +.detail .sandbox-verdict { + font-weight: bold; +} + +.detail .sandbox-verdict-malicious { + color: #c62828; +} + +.detail .sandbox-verdict-harmless { + color: #1e7e34; +} + +.detail .sandbox-report-link { + color: #00acc1; + text-decoration: underline; +} + +/* Inline "has sandbox data" indicator on the Content tab */ +.detail .sandbox-badge { + display: inline-block; + padding: 2px 10px; + border-radius: 12px; + font-size: small; + white-space: nowrap; + text-decoration: none; +} + +.detail .sandbox-badge.sandbox-yes { + background-color: #e6f4ea; + color: #1e7e34; +} + +.detail .sandbox-badge.sandbox-yes:hover { + background-color: #d0e9d7; +} + +.detail .sandbox-badge.sandbox-none { + background-color: #f0f0f0; + color: #9e9e9e; } \ No newline at end of file diff --git a/web/static/list_all.css b/web/static/list_all.css index 3ebd155..b8fd36e 100755 --- a/web/static/list_all.css +++ b/web/static/list_all.css @@ -351,4 +351,105 @@ select { .add-response.success { background-color: rgba(104, 228, 104, 0.849); -} \ No newline at end of file +} + +/* ---------------------------------------------------------------- */ +/* Quick URL search in the top button panel */ +/* ---------------------------------------------------------------- */ + +#quick-search-wrap { + position: relative; + display: inline-block; + margin: 10px; + margin-bottom: 0; +} + +#quick-url-search { + width: 260px; + padding: 6px 10px; + border: 1px solid #bbb; + border-radius: 4px; + font-size: 14px; + background-color: #fff; +} + +#quick-url-search:focus { + outline: none; + border-color: #555; + box-shadow: 0 0 0 2px rgba(0, 0, 0, 0.12); +} + +.quick-search-results { + position: absolute; + top: calc(100% + 4px); + left: 0; + right: 0; + z-index: 1000; + background: #fff; + border: 1px solid #bbb; + border-radius: 4px; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); + max-height: 320px; + overflow-y: auto; +} + +.qs-item { + display: flex; + justify-content: space-between; + align-items: center; + gap: 10px; + padding: 7px 10px; + cursor: pointer; + font-size: 13px; + border-bottom: 1px solid #eee; +} + +.qs-item:last-child { + border-bottom: none; +} + +.qs-item:hover, +.qs-item.qs-active { + background-color: #f0f0f0; +} + +.qs-item.qs-empty { + color: #888; + cursor: default; + font-style: italic; +} + +.qs-item.qs-empty:hover { + background-color: transparent; +} + +.qs-url { + flex: 1; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-family: monospace; +} + +.qs-url mark { + background-color: #ffe58a; + padding: 0; +} + +.qs-badge { + flex-shrink: 0; + padding: 1px 7px; + border-radius: 10px; + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.3px; + background-color: #ddd; + color: #333; +} + +.qs-badge.qs-malicious { background-color: #e74c3c; color: #fff; } +.qs-badge.qs-harmless { background-color: #2ecc71; color: #fff; } +.qs-badge.qs-unclassified { background-color: #f39c12; color: #fff; } +.qs-badge.qs-unreachable { background-color: #95a5a6; color: #fff; } +.qs-badge.qs-invalid { background-color: #7f8c8d; color: #fff; } +.qs-badge.qs-miner { background-color: #9b59b6; color: #fff; } \ No newline at end of file diff --git a/web/static/script.js b/web/static/script.js index 7aaa977..e3e9a6e 100755 --- a/web/static/script.js +++ b/web/static/script.js @@ -108,7 +108,7 @@ for (let i = 0; i < list.length; i++) { urls.push(list[i].value); } - + var listString = JSON.stringify(urls); localStorage.setItem('selectedURLs', listString); @@ -151,7 +151,7 @@ localStorage.removeItem('selectedURLs'); localStorage.removeItem('selectActive'); console.log('List loaded'); - + } function activeSelect() { @@ -200,3 +200,142 @@ function toggleHelp(element) { } } +// ---------------------------------------------------------------- +// Quick URL search (top panel) - live results while typing +// ---------------------------------------------------------------- +(function () { + var DEBOUNCE_MS = 200; + var AMP = String.fromCharCode(38); + + function escapeHtml(s) { + return String(s) + .split('&').join(AMP + 'amp;') + .split('<').join(AMP + 'lt;') + .split('>').join(AMP + 'gt;') + .split('"').join(AMP + 'quot;') + .split("'").join(AMP + '#39;'); + } + + function initQuickSearch() { + var input = document.getElementById('quick-url-search'); + var resultsBox = document.getElementById('quick-search-results'); + if (!input || !resultsBox) return; + + var debounceTimer = null; + var activeIndex = -1; + var currentResults = []; + + function hideResults() { + resultsBox.style.display = 'none'; + resultsBox.innerHTML = ''; + activeIndex = -1; + currentResults = []; + } + + function highlightMatch(url, query) { + var idx = url.toLowerCase().indexOf(query.toLowerCase()); + if (idx === -1) return escapeHtml(url); + return escapeHtml(url.slice(0, idx)) + + '' + escapeHtml(url.slice(idx, idx + query.length)) + '' + + escapeHtml(url.slice(idx + query.length)); + } + + function renderResults(results, query) { + currentResults = results; + activeIndex = -1; + if (!results.length) { + resultsBox.innerHTML = '
No matching URLs
'; + resultsBox.style.display = 'block'; + return; + } + resultsBox.innerHTML = results.map(function (r, i) { + var cls = (r.classification || 'unclassified'); + return '
' + + '' + highlightMatch(r.url, query) + '' + + '' + escapeHtml(cls) + '' + + '
'; + }).join(''); + resultsBox.style.display = 'block'; + } + + function goToDetail(url) { + var params = new URLSearchParams({ url: url }); + var show = new URLSearchParams(window.location.search).get('show'); + if (show) params.set('show', show); + window.location.href = '/detail?' + params.toString(); + } + + function setActive(index) { + var items = resultsBox.querySelectorAll('.qs-item:not(.qs-empty)'); + items.forEach(function (el) { el.classList.remove('qs-active'); }); + if (index >= 0 && index < items.length) { + activeIndex = index; + items[index].classList.add('qs-active'); + items[index].scrollIntoView({ block: 'nearest' }); + } else { + activeIndex = -1; + } + } + + input.addEventListener('input', function () { + var q = input.value.trim(); + clearTimeout(debounceTimer); + if (!q) { + hideResults(); + return; + } + debounceTimer = setTimeout(function () { + fetch('/api/search_url?q=' + encodeURIComponent(q)) + .then(function (resp) { return resp.ok ? resp.json() : { results: [] }; }) + .then(function (data) { renderResults(data.results || [], q); }) + .catch(function () { hideResults(); }); + }, DEBOUNCE_MS); + }); + + input.addEventListener('keydown', function (e) { + var items = resultsBox.querySelectorAll('.qs-item:not(.qs-empty)'); + if (e.key === 'ArrowDown') { + e.preventDefault(); + setActive(Math.min(activeIndex + 1, items.length - 1)); + } else if (e.key === 'ArrowUp') { + e.preventDefault(); + setActive(Math.max(activeIndex - 1, -1)); + } else if (e.key === 'Enter') { + if (activeIndex >= 0 && currentResults[activeIndex]) { + goToDetail(currentResults[activeIndex].url); + } else if (currentResults.length > 0) { + goToDetail(currentResults[0].url); + } + } else if (e.key === 'Escape') { + hideResults(); + input.blur(); + } + }); + + resultsBox.addEventListener('mousedown', function (e) { + // mousedown (not click) so it fires before blur hides the dropdown + var item = e.target.closest('.qs-item:not(.qs-empty)'); + if (item) { + e.preventDefault(); + goToDetail(item.dataset.url); + } + }); + + input.addEventListener('blur', function () { + // small delay so mousedown on a result can fire first + setTimeout(hideResults, 150); + }); + + input.addEventListener('focus', function () { + if (input.value.trim() && currentResults.length) { + resultsBox.style.display = 'block'; + } + }); + } + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', initQuickSearch); + } else { + initQuickSearch(); + } +})(); diff --git a/web/templates/detail.html b/web/templates/detail.html index bb4803d..81fa8d6 100755 --- a/web/templates/detail.html +++ b/web/templates/detail.html @@ -23,7 +23,259 @@ onclick="location.href='{{ url_for('edit_detail', url=url.url) }}';"> + {% if menu %} + + {% endif %} +
+ {% if active_tab == 'class_history' %} + +
+ {% set grouped_history = [] %} + {% for row in class_history %} + {% if grouped_history and grouped_history[-1][0] == row[0] %} + {% set last_item = grouped_history[-1] %} + {% set updated_item = last_item ~ [row] %} + {% set _ = grouped_history.pop() %} + {% set _ = grouped_history.append(updated_item) %} + {% else %} + {% set _ = grouped_history.append([row]) %} + {% endif %} + {% endfor %} + + {% for group in grouped_history %} + {% set row = group[0] %} + {% set current_class = row[1] %} + {% set next_group = grouped_history[loop.index] if loop.index < grouped_history|length else none %} + {% set next_row = next_group[0] if next_group else none %} + {% set is_same_as_next = next_row and next_row[1] == current_class %} +
+
+ + {{ row[0] }} UTC + {% if not is_same_as_next %} + {{ row[1] }} + {% else %} + ➜ + {% endif %} + {{ row[2] }} + {{ row[4] }} +
+ {% for item in group %} + {% if item[3] %} +
+ {{ item[3] }} +
+ {% endif %} + {% endfor %} +
+ {% else %} +
No classification history recorded.
+ {% endfor %} +
+ {% elif active_tab == 'content' %} + + + + + + + + + + + + + + + + + {% for row in url.content_rows %} + + + + + + + + + + + + {% if row.headers %} + + + + {% endif %} + {% else %} + + {% endfor %} + +
First seenLast seenFile pathSHA-256Mime typeSizeHTTPSandbox
{{ row.first_seen or '—' }}{{ row.last_seen or '—' }} + {% if row.path %} + {{ row.path }} + {% else %} + — + {% endif %} + {{ row.hash[:16] }}…{{ row.mime or '—' }}{{ row.size if row.size is not none else '—' }}{{ row.http_status or '—' }} + {% if row.hash in url.content_hashes_with_sandbox %} + analysis + {% else %} + none + {% endif %} + + {% if row.changed %}content changed{% endif %} +
+
+ HTTP response headers +
+ {% for k, v in row.headers.items() %} +
{{ k }}
{{ v }}
+ {% endfor %} +
+
+
No content downloaded for this URL yet.
+ {% elif active_tab == 'sandbox' %} + +
+
+
Submit to sandbox for analysis
+
+ {% set latest = url.content_rows[0] if url.content_rows else none %} + {% if latest %} +
+ + + + +
+
+ + + + +
+
+ + + + +
+ {% else %} +

No content snapshot available yet – cannot submit.

+ {% endif %} +
+
The latest available content snapshot will be submitted.
+
+ + {% for job in url.sandbox_jobs %} +
+
+ {{ job.provider or 'Sandbox' }} + {% set status_label = 'done' if job.status in ('completed', 'done') else job.status %} + {{ status_label }} +
+
+
+
Submitted
+
{{ job.submitted_at or '—' }}{% if job.submitted_at %} UTC{% endif %}
+
+
+
By
+
{{ job.requested_by or '—' }}
+
+ {% if job.content_hash %} +
+
Content
+
+ {{ job.content_hash }} + {% if job.mime_type or job.content_size is not none %} + + {% if job.mime_type %}{{ job.mime_type }}{% endif %} + {% if job.content_size is not none %}· {{ job.content_size }} bytes{% endif %} + + {% endif %} +
+
+ {% endif %} + {% set verdict = job.report_json.get('verdict') if job.report_json and job.report_json.get('verdict') else None %} + {% if verdict %} +
+
Verdict
+
{{ verdict }}
+
+ {% endif %} + {% if job.report_url %} +
+
Report
+
View full report →
+
+ {% endif %} +
+
+ {% else %} +
No sandbox submissions yet.
+ {% endfor %} +
+ {% elif active_tab == 'sources' %} + + + + + + + + + + + + + {% for row in url.source_rows %} + + + + + + + + {% else %} + + {% endfor %} + +
HoneynetFirst observedLast observedOccurrencesDerived from
{{ row[0] }}{{ row[1] if row[1] else '—' }}{{ row[2] if row[2] else '—' }}{{ row[3] if row[3] is not none else '—' }} + {% if row[4] %} + {{ row[4] }} + {% else %} + — + {% endif %} +
No sources recorded for this URL
+ {% else %} + @@ -192,9 +444,10 @@ {% endif %} + {% endif %} - +
{% if url.evaluated == 'yes' %}
diff --git a/web/templates/list_all.html b/web/templates/list_all.html index 37d3809..355a455 100755 --- a/web/templates/list_all.html +++ b/web/templates/list_all.html @@ -99,6 +99,11 @@
+
+ + +