Update python dependencies and other improvements.

- Update python dependencies - Use docker compose .env handling in favor of dotenv python library - Silence bokeh warnings - Prevent MongoDB password from being logged - Default sequence limit is now 100
DistriNet · Feb 27, 2024 · 3e4c13f · 3e4c13f
1 parent 7b7033f
commit 3e4c13f
Show file tree

Hide file tree

Showing 13 changed files with 234 additions and 223 deletions.
diff --git a/analysis/plot_factory.py b/analysis/plot_factory.py
@@ -1,16 +1,19 @@
+from bokeh.core.validation import silence
+from bokeh.core.validation.warnings import PALETTE_LENGTH_FACTORS_MISMATCH
 from bokeh.embed import file_html
 from bokeh.models import BasicTickFormatter, ColumnDataSource, HoverTool
 from bokeh.models.glyphs import Circle
 from bokeh.palettes import Iridescent23
 from bokeh.plotting import figure, output_file, show
 from bokeh.resources import CDN
 from bokeh.transform import factor_cmap
-from dotenv import load_dotenv
 
 from bci.database.mongo.mongodb import MongoDB
 from bci.evaluations.logic import PlotParameters
 
 
+silence(PALETTE_LENGTH_FACTORS_MISMATCH, True)
+
 class PlotFactory:
 
     @staticmethod
@@ -120,19 +123,3 @@ def __add_outcome_info(params: PlotParameters, docs: dict):
                 docs_with_outcome.append(new_doc)
         docs_with_outcome = PlotFactory.__transform_to_bokeh_compatible(docs_with_outcome)
         return docs_with_outcome
-
-
-if __name__ == '__main__':
-    load_dotenv()
-    MongoDB.connect()
-    db = MongoDB.get_instance()
-    params = PlotParameters(
-        'c487155',
-        'c487155',
-        'chromium',
-        'csp_chromium',
-        major_version_range=(40, 105),
-    )
-    PlotFactory.show_plot(params, db)
-    html = PlotFactory.create_html_plot_string(params, db)
-    print(html)
diff --git a/bci/database/mongo/container.py b/bci/database/mongo/container.py
@@ -52,7 +52,7 @@ def __create_new_container(user: str, pw: str, db_name, db_host):
             remove=True,
             labels=['bh_db'],
             volumes=[
-                os.path.join(os.getenv('host_pwd'), 'database/data') + ':/data/db'
+                os.path.join(os.getenv('HOST_PWD'), 'database/data') + ':/data/db'
             ],
             ports={'27017/tcp': 27017},
             environment={

diff --git a/bci/distribution/worker_manager.py b/bci/distribution/worker_manager.py
@@ -42,7 +42,6 @@ def __run_container(self, params: WorkerParameters, cb: Callable, blocking_wait=
             time.sleep(5)
         container_id = self.container_id_pool.get()
         container_name = f'bh_worker_{container_id}'
-        command = [params.serialize()]
 
         def start_container_thread():
             try:
@@ -64,7 +63,7 @@ def start_container_thread():
                         logger.info(f'Removing old container \'{container.attrs["Name"]}\' to start new one')
                         container.remove(force=True)
                 self.client.containers.run(
-                    'registry.gitlab.kuleuven.be/distrinet/research/bughog/core/worker:latest',
+                    'bughog/worker:latest',
                     name=container_name,
                     hostname=container_name,
                     shm_size='2gb',
@@ -73,18 +72,18 @@ def start_container_thread():
                     detach=False,
                     remove=True,
                     labels=['bh_worker'],
-                    command=command,
+                    command=[params.serialize()],
                     volumes=[
-                        os.path.join(os.getenv('host_pwd'), 'config') + ':/app/config',
-                        os.path.join(os.getenv('host_pwd'), 'browser/binaries/chromium/artisanal') + ':/app/browser/binaries/chromium/artisanal',
-                        os.path.join(os.getenv('host_pwd'), 'browser/binaries/firefox/artisanal') + ':/app/browser/binaries/firefox/artisanal',
-                        os.path.join(os.getenv('host_pwd'), 'experiments') + ':/app/experiments',
-                        os.path.join(os.getenv('host_pwd'), 'browser/extensions') + ':/app/browser/extensions',
-                        os.path.join(os.getenv('host_pwd'), 'logs') + ':/app/logs',
+                        os.path.join(os.getenv('HOST_PWD'), 'config') + ':/app/config',
+                        os.path.join(os.getenv('HOST_PWD'), 'browser/binaries/chromium/artisanal') + ':/app/browser/binaries/chromium/artisanal',
+                        os.path.join(os.getenv('HOST_PWD'), 'browser/binaries/firefox/artisanal') + ':/app/browser/binaries/firefox/artisanal',
+                        os.path.join(os.getenv('HOST_PWD'), 'experiments') + ':/app/experiments',
+                        os.path.join(os.getenv('HOST_PWD'), 'browser/extensions') + ':/app/browser/extensions',
+                        os.path.join(os.getenv('HOST_PWD'), 'logs') + ':/app/logs',
                         '/dev/shm:/dev/shm',
                     ],
                 )
-                logger.debug(f'Container \'{container_name}\' finished experiments with command \'{command}\'')
+                logger.debug(f'Container \'{container_name}\' finished experiments with parameters \'{repr(params)}\'')
                 cb()
             except docker.errors.APIError:
                 logger.error(f'Could not run container \'{container_name}\' or container was unexpectedly removed', exc_info=True)

diff --git a/bci/evaluations/logic.py b/bci/evaluations/logic.py
@@ -174,15 +174,24 @@ def create_all_test_params(self) -> list[TestParameters]:
                 self.database_collection)
             for mech_group in self.mech_groups]
 
-    def serialize(self) -> str:
-        return json.dumps({
+    def _to_dict(self):
+        return {
             'browser_configuration': self.browser_configuration.to_dict(),
             'evaluation_configuration': self.evaluation_configuration.to_dict(),
             'state': state_factory.to_dict(self.state),
             'mech_groups': self.mech_groups,
             'database_collection': self.database_collection,
             'database_connection_params': self.database_connection_params.to_dict()
-        })
+        }
+
+    def serialize(self) -> str:
+        return json.dumps(self._to_dict())
+
+    def __repr__(self) -> str:
+        param_dict = self._to_dict()
+        # Mask password
+        param_dict['database_connection_params']['password'] = '*'
+        return json.dumps(param_dict)
 
     @staticmethod
     def deserialize(string: str) -> WorkerParameters:

diff --git a/bci/master.py b/bci/master.py
@@ -1,7 +1,5 @@
 import logging
 
-from dotenv import load_dotenv
-
 from analysis.plot_factory import PlotFactory
 from bci.configuration import Global
 from bci.database.mongo.mongodb import MongoDB, ServerException
@@ -41,8 +39,6 @@ def __init__(self):
         self.firefox_build = None
         self.chromium_build = None
 
-        load_dotenv('/app/config/.env')
-
         Global.initialize_folders()
         self.db_connection_params = Global.get_database_connection_params()
         self.connect_to_database(self.db_connection_params)

diff --git a/bci/ui/app.py b/bci/ui/app.py
@@ -34,4 +34,4 @@ def serve_static_files(file_path):
     logging.getLogger('werkzeug').addHandler(filer_handler)
 
     # Debug is set to false because it would otherwise auto-reload (run the program twice)
-    socketio.run(app, debug=False, host="0.0.0.0")
+    socketio.run(app, debug=False, host="0.0.0.0", allow_unsafe_werkzeug=True)
diff --git a/bci/ui/frontend/src/App.vue b/bci/ui/frontend/src/App.vue
@@ -48,7 +48,7 @@ export default {
         only_release_revisions: false,
         // Sequence config
         nb_of_containers: 8,
-        sequence_limit: 1000,
+        sequence_limit: 100,
         target_mech_id: null,
         target_cookie_name: "generic",
         search_strategy: "bin_seq",

diff --git a/bci/worker.py b/bci/worker.py
@@ -2,8 +2,6 @@
 import os
 import sys
 
-from dotenv import load_dotenv
-
 from bci.configuration import Loggers
 from bci.database.mongo.mongodb import MongoDB
 from bci.evaluations.custom.custom_evaluation import CustomEvaluationFramework
@@ -16,7 +14,6 @@ def run(params: WorkerParameters):
 
     # Only perform configuration steps for separate workers
     if __name__ == '__main__':
-        load_dotenv()
         Loggers.configure_loggers()
         MongoDB.connect(params.database_connection_params)
 

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -9,14 +9,14 @@ services:
     build:
       context: .
       target: core
+    env_file:
+      - config/.env
     pull_policy: if_not_present
     shm_size: '2gb'
     networks:
       bh_net:
         aliases:
           - core
-    environment:
-      - host_pwd=${PWD}
     # 192.168.0.1 instead of 172.17.0.1 because /etc/docker/daemon.json is modified
     ports:
       - "5000:5000"

diff --git a/requirements.in b/requirements.in
@@ -1,13 +1,7 @@
-attrs
-autopep8
-beautifulsoup4
 bokeh
 docker
 Flask
 Flask-SocketIO
-iniconfig
 mitmproxy
 pymongo
-python-dotenv
-python-dateutil
 requests