fix: remove forbidden characters & add encoding reversal scripts for …

…htcondor and slurm

CHANGELOG.md

@@ -7,9 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Breaking changes
+- AUDITOR: Remove forbidden characters ([@raghuvar-vijay](https://github.com/raghuvar-vijay))
 - pyauditor + Apel plugin + HTCondor collector: drop support for Python 3.8 ([@dirksammel](https://github.com/dirksammel))
 
 ### Security
+- [RUSTSEC-2024-0421]: Update idna from 0.5.0 to 1.0.3 ([@raghuvar-vijay](https://github.com/raghuvar-vijay))
 - [RUSTSEC-2024-0363]: Update sqlx from 0.7.4 to 0.8.2 (missed some occurrences) ([@dirksammel](https://github.com/dirksammel))
 - [RUSTSEC-2024-0399]: Update rustls from 0.23.16 to 0.23.19 ([@dirksammel](https://github.com/dirksammel))
 - [RUSTSEC-2024-0402]: Update hashbrown from 0.15.0 to 0.15.2 ([@dirksammel](https://github.com/dirksammel))

Cargo.toml

@@ -8,6 +8,10 @@ members = [
     "plugins/priority",
 ]
 
+exclude = [
+  "auditor/scripts/slurm_revert_encoding",
+  ]
+
 [workspace.dependencies]
 actix-web = "4.8.0"
 actix-web-opentelemetry = { version = "0.17", features = ["metrics", "metrics-prometheus"] }

auditor/scripts/htcondor_revert_encoding/.env

@@ -0,0 +1,6 @@
+DB_NAME=auditor
+DB_USER=postgres
+DB_PASSWORD=securepassword
+DB_HOST=localhost
+DB_PORT=5432
+

auditor/scripts/htcondor_revert_encoding/htcondor_revert_encodings.py

@@ -0,0 +1,80 @@
+import json
+import os
+from urllib.parse import unquote
+
+import psycopg2
+from dotenv import load_dotenv
+
+load_dotenv(dotenv_path="auditor/scripts/htcondor_revert_encodings")
+
+
+DB_CONFIG = {
+    "dbname": os.getenv("DB_NAME", "auditor"),
+    "user": os.getenv("DB_USER", "postgres"),
+    "password": os.getenv("DB_PASSWORD", "password"),
+    "host": os.getenv("DB_HOST", "localhost"),
+    "port": os.getenv("DB_PORT", "5432"),
+}
+
+
+def decode_record(record_id, meta):
+    """
+    Decode the record_id and meta values.
+    """
+    decoded_record_id = unquote(record_id)
+
+    try:
+        decoded_meta = {
+            key: [unquote(value) for value in values] for key, values in meta.items()
+        }
+    except json.JSONDecodeError:
+        decoded_meta = {}  # If JSON decoding fails, return an empty dict
+    return decoded_record_id, json.dumps(decoded_meta)
+
+
+def main():
+    BATCH_SIZE = 1000
+    offset = 0
+
+    try:
+        conn = psycopg2.connect(**DB_CONFIG)
+        while True:
+            cursor = conn.cursor()
+
+            fetch_query = f"SELECT id, record_id, meta FROM auditor_accounting ORDER BY id LIMIT {BATCH_SIZE} OFFSET {offset};"
+            cursor.execute(fetch_query)
+            rows = cursor.fetchall()
+
+            if not rows:
+                break
+
+            for row in rows:
+                record_id, meta = row[1], row[2]
+                decoded_record_id, decoded_meta = decode_record(record_id, meta)
+
+                update_query = """
+                    UPDATE auditor_accounting
+                    SET record_id = %s, meta = %s
+                    WHERE id = %s;
+                """
+                cursor.execute(update_query, (decoded_record_id, decoded_meta, row[0]))
+
+            conn.commit()
+            offset += BATCH_SIZE
+
+        print("Database updated successfully!")
+
+    except Exception as e:
+        print(f"Error: {e}")
+        if conn:
+            conn.rollback()  # Rollback on error
+
+    finally:
+        if cursor:
+            cursor.close()
+        if conn:
+            conn.close()
+
+
+if __name__ == "__main__":
+    main()

auditor/scripts/htcondor_revert_encoding/requirements.txt

@@ -0,0 +1,2 @@
+psycopg2==2.9.10
+python-dotenv==1.0.1

auditor/scripts/slurm_revert_encoding/.env

@@ -0,0 +1 @@
+DATABASE_URL=postgres://postgres:password@localhost:5432/auditor