clean-up (linting) customising scripts

.vscode/launch.template.json

@@ -158,7 +158,7 @@
             "name": "py-import-mgm",
             "type": "debugpy",
             "request": "launch",
-            "program": "${workspaceFolder}/roles/importer/files/importer/import-mgm.py",
+            "program": "${workspaceFolder}/roles/importer/files/importer/import_mgm.py",
             "console": "integratedTerminal",
             "env": {
                 "PYTHONPATH": "${PYTHONPATH}:${workspaceRoot}"
@@ -167,13 +167,7 @@
                 "-m1",
                 "-d1",
                 "-f",
-                "-s",
-                // "-c",
-                // "-l66",
-                // "-l500",
-                // "-ihttps://fwodemodata.cactus.de/demo04_cpr8x.json",
-                // "-ihttps://fwodemodata.cactus.de/demo05_fortiMgr2.json"
-                // "-ihttps://fwodemodata.cactus.de/demo11-r82_v9.json",
+                "-s"
             ]
         },
         {
@@ -396,36 +390,16 @@
                 "-i/tmp/fworch-config.graphql"
             ]
         },
-        // {
-        //     "name": "py-cpr8x-autodiscovery",
-        //     "type": "debugpy",
-        //     "request": "launch",
-        //     "program": "${workspaceFolder}/roles/importer/files/importer/checkpointR8x/auto-discover.py",
-        //     "console": "integratedTerminal",
-        //     "cwd": "${workspaceFolder}/roles/importer/files/importer/checkpointR8x",
-        //     "env": {
-        //         "PYTHONPATH": "${PYTHONPATH}:${workspaceRoot}"
-        //     },
-        //     "args": [
-        //         "-a192.168.100.111",
-        //         "-w/home/tim/secrets/cp_apiuser_pwd",
-        //         "-uitsecorg",
-        //         "-d4",
-        //         "-fjson"
-        //     ]
-        // },
         {
-            "name": "py-import-main-loop",
+            "name": "py-import_main_loop",
             "type": "debugpy",
             "request": "launch",
-            "program": "${workspaceFolder}/roles/importer/files/importer/import-main-loop.py",
+            "program": "${workspaceFolder}/roles/importer/files/importer/import_main_loop.py",
             "console": "integratedTerminal",
             "env": {
                 "PYTHONPATH": "${PYTHONPATH}:${workspaceRoot}"
             },
             "args": [
-                //"-c",
-                //"-f",
                 "-d0"
             ]
         },
@@ -444,7 +418,6 @@
             "name": "c#-FWO Test",
             "type": "coreclr",
             "request": "launch",
-            // "preLaunchTask": "build_test",
             "program": "/usr/bin/dotnet",
             "args": [
                 "test"

documentation/developer-docs/api/api_handling_documentation.md

@@ -42,7 +42,7 @@ tim@fworch-comp:~$ sudo systemctl status fworch-*
       Tasks: 1 (limit: 4637)
      Memory: 16.8M
      CGroup: /system.slice/fworch-importer-api.service
-             └─341171 /usr/bin/python3 /usr/local/fworch/importer/import-main-loop.py
+             └─341171 /usr/bin/python3 /usr/local/fworch/importer/import_main_loop.py
 
 Dez 09 20:07:18 fworch-comp systemd[1]: Starting fworch importer pure python...
 Dez 09 20:07:28 fworch-comp systemd[1]: Started fworch importer pure python.
@@ -83,4 +83,4 @@ Dez 10 10:45:59 fworch-comp fworch.middleware-server[396568]: Info - Found ldap
 Dez 10 10:45:59 fworch-comp fworch.middleware-server[396568]: info: Microsoft.Hosting.Lifetime[14]
 Dez 10 10:45:59 fworch-comp fworch.middleware-server[396568]:       Now listening on: http://127.0.0.1:8880
 tim@fworch-comp:~$ 
-```
+```

documentation/importer/readme.md

@@ -24,8 +24,8 @@ need to change import id in all csv-files to next id, e.g.:
 ## python importer
 
 ```console
-user@test:~$ ./import-mgm.py --help
-usage: import-mgm.py [-h] -m management_id [-c] [-f] [-d debug_level] [-v] [-s] [-l api_limit] [-i config_file_input] [-n config_file_normalized_input]
+user@test:~$ ./import_mgm.py --help
+usage: import_mgm.py [-h] -m management_id [-c] [-f] [-d debug_level] [-v] [-s] [-l api_limit] [-i config_file_input] [-n config_file_normalized_input]
 
 Read configuration from FW management via API calls
 
@@ -83,4 +83,4 @@ Ruff format ( only needed for VS, but also run pre-commit)
 
 ```
 ruff format
-```
+```

documentation/revision-history-develop.md

@@ -324,7 +324,7 @@ Breaking changes
     sudo -u fworch -i
     cd importer
     source venv/bin/activate
-    ./import-mgm.py -m xy -f -s -d 8 
+    ./import_mgm.py -m xy -f -s -d 8 
     As we now need support for pip, in installations behind url filter, make sure that all sub-domains of
     pythonhosted.org
     are also allowed.

pyproject.toml

@@ -1,19 +1,20 @@
 [tool.pyright]
 include = [
     "roles/importer/files/importer",
+    "scripts"
 ]
 exclude = [
     "**/node_modules",
     "**/__pycache__",
     ".git",
     ".venv",
     "venv",
-    "scripts",
     "roles/importer/files/importer/test",
-]
-
-ignore = [
-    "./scripts",
+    "roles/api/files/scripts",
+    "scripts/acquire_lock.py",
+    "scripts/gh-search.py",
+    "scripts/json_pretty_print.py",
+    "scripts/release_lock.py"    
 ]
 
 venvPath = "."
@@ -23,8 +24,6 @@ typeCheckingMode = "strict"
 reportMissingImports = false
 reportMissingTypeStubs = true
 
-
-
 [tool.ruff]
 line-length = 120
 indent-width = 4
@@ -57,9 +56,13 @@ exclude = [
     "site-packages",
     "venv",
     "__init__.py",
-    "scripts",
     "roles/tests-integration",
-    "roles/importer/files/importer/test"
+    "roles/importer/files/importer/test",
+    "roles/api/files/scripts",
+    "scripts/acquire_lock.py",
+    "scripts/gh-search.py",
+    "scripts/json_pretty_print.py",
+    "scripts/release_lock.py"
 ]
 
 [tool.ruff.lint]
@@ -81,5 +84,3 @@ quote-style = "double"
 indent-style = "space"
 skip-magic-trailing-comma = false
 line-ending = "auto"
-
-

roles/importer/files/importer/common.py

@@ -265,7 +265,7 @@ def get_config_from_api(
         raise
 
     # check for changes from product-specific FW API, if we are importing from file we assume config changes
-    # TODO: implement real change detection
+    # open issue: implement real change detection
     config_changed_since_last_import = fw_module.has_config_changed(
         config_in, import_state, import_state.state.force_import
     )

roles/importer/files/importer/import-main-loop.py → roles/importer/files/importer/import_main_loop.py

@@ -1,4 +1,5 @@
-# main importer loop in python (also able to run distributed)  # noqa: N999
+#!/usr/bin/env python3
+# main importer loop in python (also able to run distributed)
 # run import loop every x seconds (adjust sleep time per management depending on the change frequency )
 
 import argparse
@@ -9,7 +10,7 @@
 
 import fwo_globals
 import urllib3
-from common import import_management
+from common import import_management  # type: ignore[import-not-found]
 from fwo_api import FwoApi
 from fwo_api_call import FwoApiCall
 from fwo_base import init_service_provider, register_global_state
@@ -34,16 +35,16 @@ def get_fwo_jwt(import_user: str, import_pwd: str, user_management_api: str) ->
         FWOLogger.error(e.message)
     except Exception:
         FWOLogger.error(
-            "import-main-loop - unspecified error during FWO API login - skipping: " + str(traceback.format_exc())
+            "import_main_loop - unspecified error during FWO API login - skipping: " + str(traceback.format_exc())
         )
 
 
 def wait_with_shutdown_check(sleep_time: int):
     counter = 0
     while counter < sleep_time:
         if fwo_globals.shutdown_requested:
-            FWOLogger.info("import-main-loop - shutdown requested. Exiting...")
-            raise SystemExit("import-main-loop - shutdown requested")
+            FWOLogger.info("import_main_loop - shutdown requested. Exiting...")
+            raise SystemExit("import_main_loop - shutdown requested")
         time.sleep(1)
         counter += 1
 
@@ -76,7 +77,7 @@ def import_single_management(
         mgm_details = mgm_controller.get_mgm_details(fwo_api, mgm_id)
     except Exception:
         FWOLogger.error(
-            "import-main-loop - error while getting FW management details for mgm_id="
+            "import_main_loop - error while getting FW management details for mgm_id="
             + str(mgm_id)
             + " - skipping: "
             + str(traceback.format_exc())
@@ -88,18 +89,18 @@ def import_single_management(
     if mgm_details["deviceType"]["id"] not in (9, 12, 17, 22, 23, 24, 28, 29):
         return
 
-    FWOLogger.debug(f"import-main-loop: starting import of mgm_id={mgm_id}")
+    FWOLogger.debug(f"import_main_loop: starting import of mgm_id={mgm_id}")
 
     try:
         import_management(
             mgm_id, fwo_api_call, verify_certificates, api_fetch_limit, clear, suppress_certificate_warnings
         )
     except (FwoApiFailedLockImportError, FwLoginFailedError):
-        FWOLogger.info(f"import-main-loop - minor error while importing mgm_id={mgm_id}, {traceback.format_exc()!s}")
+        FWOLogger.info(f"import_main_loop - minor error while importing mgm_id={mgm_id}, {traceback.format_exc()!s}")
         return  # minor errors for a single mgm, go to next one
     except Exception:  # all other exceptions are logged here
         FWOLogger.error(
-            f"import-main-loop - unspecific error while importing mgm_id={mgm_id}, {traceback.format_exc()!s}"
+            f"import_main_loop - unspecific error while importing mgm_id={mgm_id}, {traceback.format_exc()!s}"
         )
 
 
@@ -121,7 +122,7 @@ def main_loop(
         with open(importer_pwd_file) as f:
             importer_pwd = f.read().replace("\n", "")
     except Exception:
-        FWOLogger.error("import-main-loop - error while reading importer pwd file")
+        FWOLogger.error("import_main_loop - error while reading importer pwd file")
         raise
 
     jwt = get_fwo_jwt(importer_user_name, importer_pwd, user_management_api_base_url)
@@ -142,7 +143,7 @@ def main_loop(
     try:
         mgm_ids = fwo_api_call.get_mgm_ids()
     except Exception:
-        FWOLogger.error(f"import-main-loop - error while getting FW management ids: {traceback.format_exc()!s}")
+        FWOLogger.error(f"import_main_loop - error while getting FW management ids: {traceback.format_exc()!s}")
         wait_with_shutdown_check(sleep_timer)
         return
 
@@ -166,7 +167,7 @@ def main_loop(
             fwo_api,
         )
 
-    FWOLogger.info(f"import-main-loop: sleeping for {sleep_timer} seconds until next import cycle")
+    FWOLogger.info(f"import_main_loop: sleeping for {sleep_timer} seconds until next import cycle")
     wait_with_shutdown_check(sleep_timer)
 
 
@@ -188,10 +189,10 @@ def main(
     if suppress_certificate_warnings:
         urllib3.disable_warnings()
 
-    FWOLogger.info("importer-main-loop starting ...")
+    FWOLogger.info("importer_main_loop starting ...")
     if IMPORTER_BASE_DIR not in sys.path:
         sys.path.append(IMPORTER_BASE_DIR)
-    importer_user_name = "importer"  # TODO: move to config file?
+    importer_user_name = "importer"  # move to config file?
     importer_pwd_file = BASE_DIR + "/etc/secrets/importer_pwd"
 
     # setting defaults (only as fallback if config defaults cannot be fetched via API):

roles/importer/files/importer/import-mgm.py → roles/importer/files/importer/import_mgm.py

@@ -1,14 +1,15 @@
-import argparse  # noqa: N999
+#!/usr/local/fworch/importer_venv/bin/python3
+import argparse
 import sys
 import traceback
 import warnings
 
 import urllib3
-from common import IMPORTER_BASE_DIR, import_management
+from common import import_management  # type: ignore[import-not-found]
 from fwo_api import FwoApi
 from fwo_api_call import FwoApiCall
 from fwo_base import init_service_provider, register_global_state
-from fwo_const import BASE_DIR
+from fwo_const import BASE_DIR, IMPORTER_BASE_DIR
 from fwo_exceptions import FwoApiLoginFailedError
 from fwo_log import FWOLogger
 from model_controllers.import_state_controller import ImportStateController
@@ -24,7 +25,7 @@ def get_fwo_jwt(import_user: str, import_pwd: str, user_management_api: str) ->
         FWOLogger.error(e.message)
     except Exception:
         FWOLogger.error(
-            "import-main-loop - unspecified error during FWO API login - skipping: " + str(traceback.format_exc())
+            "import_main_loop - unspecified error during FWO API login - skipping: " + str(traceback.format_exc())
         )
 
 
@@ -54,7 +55,7 @@ def main(
     if IMPORTER_BASE_DIR not in sys.path:
         sys.path.append(IMPORTER_BASE_DIR)
 
-    importer_user_name = "importer"  # TODO: move to config file?
+    importer_user_name = "importer"  # move to config file?
     importer_pwd_file = BASE_DIR + "/etc/secrets/importer_pwd"
 
     try:

roles/importer/templates/fworch-importer-api.service.j2

@@ -15,7 +15,7 @@ After=network.target remote-fs.target nss-lookup.target
 Type=simple
 WorkingDirectory={{ importer_home }}
 ExecStartPre=/bin/sleep 10
-ExecStart={{ importer_venv_dir }}/bin/python {{ importer_home}}/import-main-loop.py
+ExecStart={{ importer_venv_dir }}/bin/python {{ importer_home}}/import_main_loop.py
 Environment="PATH={{ importer_venv_dir }}/bin:/usr/bin:/bin"
 Environment="VIRTUAL_ENV={{ importer_venv_dir }}"
 TimeoutStopSec=300min

roles/tests-integration/tasks/test-importer.yml

@@ -1,5 +1,4 @@
 ---
-
 - name: find management id for checkpoint test
   postgresql_query:
     db: fworchdb
@@ -20,7 +19,7 @@
   ansible.builtin.command:
     cmd: >
       {{ importer_venv_dir }}/bin/python3
-      ./import-mgm.py -f -s -m{{ test_checkpoint_mgm_id.query_result.0.mgm_id }}
+      ./import_mgm.py -f -s -m{{ test_checkpoint_mgm_id.query_result.0.mgm_id }}
     chdir: "{{ fworch_home }}/importer"
   become: true
   become_user: "{{ fworch_user }}"
@@ -30,9 +29,8 @@
 - name: show checkpoint import result
   debug:
     var: test_checkpoint_import_result
-
 # - name: make test fortigate import
-#   command: "python3 ./import-mgm.py -f -s -m{{ test_fortigate_mgm_id.query_result.0.mgm_id }}"
+#   command: "python3 ./import_mgm.py -f -s -m{{ test_fortigate_mgm_id.query_result.0.mgm_id }}"
 #   args:
 #     chdir: "{{ fworch_home }}/importer"
 #   become: true

scripts/customizing/app_data_import/__init__.py

@@ -0,0 +1 @@
+