diff --git a/analysis/plot_factory.py b/analysis/plot_factory.py index a488dda..dfb4e85 100644 --- a/analysis/plot_factory.py +++ b/analysis/plot_factory.py @@ -1,3 +1,5 @@ +from bokeh.core.validation import silence +from bokeh.core.validation.warnings import PALETTE_LENGTH_FACTORS_MISMATCH from bokeh.embed import file_html from bokeh.models import BasicTickFormatter, ColumnDataSource, HoverTool from bokeh.models.glyphs import Circle @@ -5,12 +7,13 @@ from bokeh.plotting import figure, output_file, show from bokeh.resources import CDN from bokeh.transform import factor_cmap -from dotenv import load_dotenv from bci.database.mongo.mongodb import MongoDB from bci.evaluations.logic import PlotParameters +silence(PALETTE_LENGTH_FACTORS_MISMATCH, True) + class PlotFactory: @staticmethod @@ -120,19 +123,3 @@ def __add_outcome_info(params: PlotParameters, docs: dict): docs_with_outcome.append(new_doc) docs_with_outcome = PlotFactory.__transform_to_bokeh_compatible(docs_with_outcome) return docs_with_outcome - - -if __name__ == '__main__': - load_dotenv() - MongoDB.connect() - db = MongoDB.get_instance() - params = PlotParameters( - 'c487155', - 'c487155', - 'chromium', - 'csp_chromium', - major_version_range=(40, 105), - ) - PlotFactory.show_plot(params, db) - html = PlotFactory.create_html_plot_string(params, db) - print(html) diff --git a/bci/database/mongo/container.py b/bci/database/mongo/container.py index 7772750..0c0200a 100644 --- a/bci/database/mongo/container.py +++ b/bci/database/mongo/container.py @@ -52,7 +52,7 @@ def __create_new_container(user: str, pw: str, db_name, db_host): remove=True, labels=['bh_db'], volumes=[ - os.path.join(os.getenv('host_pwd'), 'database/data') + ':/data/db' + os.path.join(os.getenv('HOST_PWD'), 'database/data') + ':/data/db' ], ports={'27017/tcp': 27017}, environment={ diff --git a/bci/distribution/worker_manager.py b/bci/distribution/worker_manager.py index 8fe7e23..cc42b17 100644 --- a/bci/distribution/worker_manager.py +++ b/bci/distribution/worker_manager.py @@ -42,7 +42,6 @@ def __run_container(self, params: WorkerParameters, cb: Callable, blocking_wait= time.sleep(5) container_id = self.container_id_pool.get() container_name = f'bh_worker_{container_id}' - command = [params.serialize()] def start_container_thread(): try: @@ -64,7 +63,7 @@ def start_container_thread(): logger.info(f'Removing old container \'{container.attrs["Name"]}\' to start new one') container.remove(force=True) self.client.containers.run( - 'registry.gitlab.kuleuven.be/distrinet/research/bughog/core/worker:latest', + 'bughog/worker:latest', name=container_name, hostname=container_name, shm_size='2gb', @@ -73,18 +72,18 @@ def start_container_thread(): detach=False, remove=True, labels=['bh_worker'], - command=command, + command=[params.serialize()], volumes=[ - os.path.join(os.getenv('host_pwd'), 'config') + ':/app/config', - os.path.join(os.getenv('host_pwd'), 'browser/binaries/chromium/artisanal') + ':/app/browser/binaries/chromium/artisanal', - os.path.join(os.getenv('host_pwd'), 'browser/binaries/firefox/artisanal') + ':/app/browser/binaries/firefox/artisanal', - os.path.join(os.getenv('host_pwd'), 'experiments') + ':/app/experiments', - os.path.join(os.getenv('host_pwd'), 'browser/extensions') + ':/app/browser/extensions', - os.path.join(os.getenv('host_pwd'), 'logs') + ':/app/logs', + os.path.join(os.getenv('HOST_PWD'), 'config') + ':/app/config', + os.path.join(os.getenv('HOST_PWD'), 'browser/binaries/chromium/artisanal') + ':/app/browser/binaries/chromium/artisanal', + os.path.join(os.getenv('HOST_PWD'), 'browser/binaries/firefox/artisanal') + ':/app/browser/binaries/firefox/artisanal', + os.path.join(os.getenv('HOST_PWD'), 'experiments') + ':/app/experiments', + os.path.join(os.getenv('HOST_PWD'), 'browser/extensions') + ':/app/browser/extensions', + os.path.join(os.getenv('HOST_PWD'), 'logs') + ':/app/logs', '/dev/shm:/dev/shm', ], ) - logger.debug(f'Container \'{container_name}\' finished experiments with command \'{command}\'') + logger.debug(f'Container \'{container_name}\' finished experiments with parameters \'{repr(params)}\'') cb() except docker.errors.APIError: logger.error(f'Could not run container \'{container_name}\' or container was unexpectedly removed', exc_info=True) diff --git a/bci/evaluations/logic.py b/bci/evaluations/logic.py index d879104..dfb1358 100644 --- a/bci/evaluations/logic.py +++ b/bci/evaluations/logic.py @@ -174,15 +174,24 @@ def create_all_test_params(self) -> list[TestParameters]: self.database_collection) for mech_group in self.mech_groups] - def serialize(self) -> str: - return json.dumps({ + def _to_dict(self): + return { 'browser_configuration': self.browser_configuration.to_dict(), 'evaluation_configuration': self.evaluation_configuration.to_dict(), 'state': state_factory.to_dict(self.state), 'mech_groups': self.mech_groups, 'database_collection': self.database_collection, 'database_connection_params': self.database_connection_params.to_dict() - }) + } + + def serialize(self) -> str: + return json.dumps(self._to_dict()) + + def __repr__(self) -> str: + param_dict = self._to_dict() + # Mask password + param_dict['database_connection_params']['password'] = '*' + return json.dumps(param_dict) @staticmethod def deserialize(string: str) -> WorkerParameters: diff --git a/bci/master.py b/bci/master.py index 5dddc86..a1fba22 100644 --- a/bci/master.py +++ b/bci/master.py @@ -1,7 +1,5 @@ import logging -from dotenv import load_dotenv - from analysis.plot_factory import PlotFactory from bci.configuration import Global from bci.database.mongo.mongodb import MongoDB, ServerException @@ -41,8 +39,6 @@ def __init__(self): self.firefox_build = None self.chromium_build = None - load_dotenv('/app/config/.env') - Global.initialize_folders() self.db_connection_params = Global.get_database_connection_params() self.connect_to_database(self.db_connection_params) diff --git a/bci/ui/app.py b/bci/ui/app.py index 16f352d..d6f553b 100644 --- a/bci/ui/app.py +++ b/bci/ui/app.py @@ -34,4 +34,4 @@ def serve_static_files(file_path): logging.getLogger('werkzeug').addHandler(filer_handler) # Debug is set to false because it would otherwise auto-reload (run the program twice) - socketio.run(app, debug=False, host="0.0.0.0") + socketio.run(app, debug=False, host="0.0.0.0", allow_unsafe_werkzeug=True) diff --git a/bci/ui/frontend/src/App.vue b/bci/ui/frontend/src/App.vue index d68337f..298286e 100644 --- a/bci/ui/frontend/src/App.vue +++ b/bci/ui/frontend/src/App.vue @@ -48,7 +48,7 @@ export default { only_release_revisions: false, // Sequence config nb_of_containers: 8, - sequence_limit: 1000, + sequence_limit: 100, target_mech_id: null, target_cookie_name: "generic", search_strategy: "bin_seq", diff --git a/bci/worker.py b/bci/worker.py index 9294943..dfb5c27 100644 --- a/bci/worker.py +++ b/bci/worker.py @@ -2,8 +2,6 @@ import os import sys -from dotenv import load_dotenv - from bci.configuration import Loggers from bci.database.mongo.mongodb import MongoDB from bci.evaluations.custom.custom_evaluation import CustomEvaluationFramework @@ -16,7 +14,6 @@ def run(params: WorkerParameters): # Only perform configuration steps for separate workers if __name__ == '__main__': - load_dotenv() Loggers.configure_loggers() MongoDB.connect(params.database_connection_params) diff --git a/docker-compose.yml b/docker-compose.yml index 97d8de7..f7b7840 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,14 +9,14 @@ services: build: context: . target: core + env_file: + - config/.env pull_policy: if_not_present shm_size: '2gb' networks: bh_net: aliases: - core - environment: - - host_pwd=${PWD} # 192.168.0.1 instead of 172.17.0.1 because /etc/docker/daemon.json is modified ports: - "5000:5000" diff --git a/requirements.in b/requirements.in index ce07edc..35e1d5a 100644 --- a/requirements.in +++ b/requirements.in @@ -1,13 +1,7 @@ -attrs -autopep8 -beautifulsoup4 bokeh docker Flask Flask-SocketIO -iniconfig mitmproxy pymongo -python-dotenv -python-dateutil requests diff --git a/requirements.txt b/requirements.txt index 382acee..2325cbf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,44 +4,49 @@ # # pip-compile requirements.in # -asgiref==3.5.2 +aioquic==0.9.25 # via mitmproxy -attrs==19.3.0 - # via -r requirements.in -autopep8==1.5.7 - # via -r requirements.in -beautifulsoup4==4.11.1 - # via -r requirements.in -bokeh==3.1.1 +asgiref==3.7.2 + # via mitmproxy +attrs==23.2.0 + # via service-identity +bidict==0.23.1 + # via python-socketio +blinker==1.7.0 + # via flask +bokeh==3.3.4 # via -r requirements.in -brotli==1.0.9 +brotli==1.1.0 # via mitmproxy -certifi==2020.6.20 +certifi==2024.2.2 # via + # aioquic # mitmproxy # requests -cffi==1.15.1 +cffi==1.16.0 # via cryptography -chardet==3.0.4 +charset-normalizer==3.3.2 # via requests -click==7.1.2 +click==8.1.7 # via flask -contourpy==1.1.0 +contourpy==1.2.0 # via bokeh -cryptography==38.0.4 +cryptography==41.0.7 # via + # aioquic # mitmproxy # pyopenssl -dnspython==2.4.2 + # service-identity +dnspython==2.6.1 # via pymongo -docker==5.0.3 +docker==7.0.0 # via -r requirements.in -flask==1.1.2 +flask==3.0.2 # via # -r requirements.in # flask-socketio # mitmproxy -flask-socketio==4.3.1 +flask-socketio==5.3.6 # via -r requirements.in h11==0.14.0 # via @@ -55,13 +60,11 @@ hyperframe==6.0.1 # via # h2 # mitmproxy -idna==2.10 +idna==3.6 # via requests -iniconfig==2.0.0 - # via -r requirements.in -itsdangerous==1.1.0 +itsdangerous==2.1.2 # via flask -jinja2==2.11.2 +jinja2==3.1.3 # via # bokeh # flask @@ -69,93 +72,103 @@ kaitaistruct==0.10 # via mitmproxy ldap3==2.9.1 # via mitmproxy -markupsafe==1.1.1 - # via jinja2 -mitmproxy==9.0.1 +markupsafe==2.1.5 + # via + # jinja2 + # werkzeug +mitmproxy==10.2.2 # via -r requirements.in -mitmproxy-wireguard==0.1.23 +mitmproxy-macos==0.5.1 + # via mitmproxy-rs +mitmproxy-rs==0.5.1 # via mitmproxy -msgpack==1.0.5 +msgpack==1.0.7 # via mitmproxy -numpy==1.25.2 +numpy==1.26.4 # via # bokeh # contourpy # pandas -packaging==23.1 - # via bokeh -pandas==1.5.3 +packaging==23.2 + # via + # bokeh + # docker +pandas==2.2.1 # via bokeh passlib==1.7.4 # via mitmproxy -pillow==10.0.0 +pillow==10.2.0 # via bokeh -protobuf==4.24.0 +protobuf==4.25.3 # via mitmproxy publicsuffix2==2.20191221 # via mitmproxy -pyasn1==0.5.0 - # via ldap3 -pycodestyle==2.7.0 - # via autopep8 +pyasn1==0.5.1 + # via + # ldap3 + # pyasn1-modules + # service-identity +pyasn1-modules==0.3.0 + # via service-identity pycparser==2.21 # via cffi -pymongo==4.6.1 +pylsqpack==0.3.18 + # via aioquic +pymongo==4.6.2 # via -r requirements.in -pyopenssl==22.1.0 - # via mitmproxy -pyparsing==2.4.7 +pyopenssl==23.3.0 + # via + # aioquic + # mitmproxy +pyparsing==3.1.1 # via mitmproxy pyperclip==1.8.2 # via mitmproxy -python-dateutil==2.8.1 - # via - # -r requirements.in - # pandas -python-dotenv==1.0.0 - # via -r requirements.in -python-engineio==3.13.1 +python-dateutil==2.8.2 + # via pandas +python-engineio==4.9.0 # via python-socketio -python-socketio==4.6.0 +python-socketio==5.11.1 # via flask-socketio -pytz==2023.3 +pytz==2024.1 # via pandas pyyaml==6.0.1 # via bokeh -requests==2.24.0 +requests==2.31.0 # via # -r requirements.in # docker -ruamel-yaml==0.17.32 +ruamel-yaml==0.18.6 # via mitmproxy -ruamel-yaml-clib==0.2.7 +ruamel-yaml-clib==0.2.8 # via ruamel-yaml -six==1.15.0 - # via - # python-dateutil - # python-engineio - # python-socketio +service-identity==24.1.0 + # via aioquic +simple-websocket==1.0.0 + # via python-engineio +six==1.16.0 + # via python-dateutil sortedcontainers==2.4.0 # via mitmproxy -soupsieve==2.3.2.post1 - # via beautifulsoup4 -toml==0.10.2 - # via autopep8 -tornado==6.3.3 +tornado==6.4 # via # bokeh # mitmproxy -urllib3==1.25.9 - # via requests -urwid==2.1.2 +tzdata==2024.1 + # via pandas +urllib3==2.2.1 + # via + # docker + # requests +urwid-mitmproxy==2.1.2.1 # via mitmproxy -websocket-client==1.2.3 - # via docker -werkzeug==1.0.1 +werkzeug==3.0.1 # via flask wsproto==1.2.0 - # via mitmproxy -xyzservices==2023.7.0 + # via + # mitmproxy + # simple-websocket +xyzservices==2023.10.1 # via bokeh -zstandard==0.19.0 +zstandard==0.22.0 # via mitmproxy diff --git a/requirements_dev.in b/requirements_dev.in index bc0c414..891ddc3 100644 --- a/requirements_dev.in +++ b/requirements_dev.in @@ -1,5 +1,6 @@ -r requirements.txt anybadge +autopep8 boto3 botocore coverage diff --git a/requirements_dev.txt b/requirements_dev.txt index dcaf8ff..254370c 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,88 +1,103 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile requirements_dev.in # +aioquic==0.9.25 + # via + # -r requirements.txt + # mitmproxy anybadge==1.14.0 # via -r requirements_dev.in -asgiref==3.5.2 +asgiref==3.7.2 # via # -r requirements.txt # mitmproxy -astroid==2.15.6 +astroid==3.1.0 # via pylint -attrs==19.3.0 - # via -r requirements.txt -autopep8==1.5.7 - # via -r requirements.txt -beautifulsoup4==4.11.1 - # via -r requirements.txt -bokeh==3.1.1 +attrs==23.2.0 + # via + # -r requirements.txt + # service-identity +autopep8==2.0.4 + # via -r requirements_dev.in +bidict==0.23.1 + # via + # -r requirements.txt + # python-socketio +blinker==1.7.0 + # via + # -r requirements.txt + # flask +bokeh==3.3.4 # via -r requirements.txt -boto3==1.28.34 +boto3==1.7.84 # via -r requirements_dev.in -botocore==1.31.34 +botocore==1.10.84 # via # -r requirements_dev.in # boto3 # s3transfer -brotli==1.0.9 +brotli==1.1.0 # via # -r requirements.txt # mitmproxy -certifi==2020.6.20 +certifi==2024.2.2 # via # -r requirements.txt + # aioquic # mitmproxy # requests -cffi==1.15.1 +cffi==1.16.0 # via # -r requirements.txt # cryptography -chardet==3.0.4 +charset-normalizer==3.3.2 # via # -r requirements.txt # requests -click==7.1.2 +click==8.1.7 # via # -r requirements.txt # flask -contourpy==1.1.0 +contourpy==1.2.0 # via # -r requirements.txt # bokeh -coverage[toml]==7.3.0 +coverage[toml]==7.4.3 # via # -r requirements_dev.in # pytest-cov -cryptography==38.0.4 +cryptography==41.0.7 # via # -r requirements.txt + # aioquic # mitmproxy # pyopenssl -debugpy==1.6.7.post1 + # service-identity +debugpy==1.8.1 # via -r requirements_dev.in -dill==0.3.7 +dill==0.3.8 # via pylint -dnspython==2.4.2 +dnspython==2.6.1 # via # -r requirements.txt # pymongo -docker==5.0.3 +docker==7.0.0 # via -r requirements.txt -exceptiongroup==1.1.3 - # via pytest -flake8==3.9.2 +docutils==0.20.1 + # via botocore +flake8==7.0.0 # via # -r requirements_dev.in # pytest-flake8 -flask==1.1.2 +flask==3.0.2 # via # -r requirements.txt # flask-socketio # mitmproxy -flask-socketio==4.3.1 +flask-socketio==5.3.6 # via -r requirements.txt h11==0.14.0 # via @@ -102,28 +117,26 @@ hyperframe==6.0.1 # -r requirements.txt # h2 # mitmproxy -idna==2.10 +idna==3.6 # via # -r requirements.txt # requests iniconfig==2.0.0 - # via - # -r requirements.txt - # pytest -isort==5.12.0 + # via pytest +isort==5.13.2 # via # -r requirements_dev.in # pylint -itsdangerous==1.1.0 +itsdangerous==2.1.2 # via # -r requirements.txt # flask -jinja2==2.11.2 +jinja2==3.1.3 # via # -r requirements.txt # bokeh # flask -jmespath==1.0.1 +jmespath==0.10.0 # via # boto3 # botocore @@ -131,43 +144,47 @@ kaitaistruct==0.10 # via # -r requirements.txt # mitmproxy -lazy-object-proxy==1.9.0 - # via astroid ldap3==2.9.1 # via # -r requirements.txt # mitmproxy -markupsafe==1.1.1 +markupsafe==2.1.5 # via # -r requirements.txt # jinja2 -mccabe==0.6.1 + # werkzeug +mccabe==0.7.0 # via # flake8 # pylint -mitmproxy==9.0.1 +mitmproxy==10.2.2 # via -r requirements.txt -mitmproxy-wireguard==0.1.23 +mitmproxy-macos==0.5.1 + # via + # -r requirements.txt + # mitmproxy-rs +mitmproxy-rs==0.5.1 # via # -r requirements.txt # mitmproxy -msgpack==1.0.5 +msgpack==1.0.7 # via # -r requirements.txt # mitmproxy -numpy==1.25.2 +numpy==1.26.4 # via # -r requirements.txt # bokeh # contourpy # pandas -packaging==23.1 +packaging==23.2 # via # -r requirements.txt # anybadge # bokeh + # docker # pytest -pandas==1.5.3 +pandas==2.2.1 # via # -r requirements.txt # bokeh @@ -175,15 +192,15 @@ passlib==1.7.4 # via # -r requirements.txt # mitmproxy -pillow==10.0.0 +pillow==10.2.0 # via # -r requirements.txt # bokeh -platformdirs==3.10.0 +platformdirs==4.2.0 # via pylint -pluggy==1.2.0 +pluggy==1.4.0 # via pytest -protobuf==4.24.0 +protobuf==4.25.3 # via # -r requirements.txt # mitmproxy @@ -191,30 +208,40 @@ publicsuffix2==2.20191221 # via # -r requirements.txt # mitmproxy -pyasn1==0.5.0 +pyasn1==0.5.1 # via # -r requirements.txt # ldap3 -pycodestyle==2.7.0 + # pyasn1-modules + # service-identity +pyasn1-modules==0.3.0 # via # -r requirements.txt + # service-identity +pycodestyle==2.11.1 + # via # autopep8 # flake8 pycparser==2.21 # via # -r requirements.txt # cffi -pyflakes==2.3.1 +pyflakes==3.2.0 # via flake8 -pylint==2.17.5 +pylint==3.1.0 # via -r requirements_dev.in -pymongo==4.3.3 +pylsqpack==0.3.18 + # via + # -r requirements.txt + # aioquic +pymongo==4.6.2 # via -r requirements.txt -pyopenssl==22.1.0 +pyopenssl==23.3.0 # via # -r requirements.txt + # aioquic # mitmproxy -pyparsing==2.4.7 +pyparsing==3.1.1 # via # -r requirements.txt # mitmproxy @@ -222,31 +249,29 @@ pyperclip==1.8.2 # via # -r requirements.txt # mitmproxy -pytest==7.4.0 +pytest==8.0.2 # via # -r requirements_dev.in # pytest-cov # pytest-flake8 pytest-cov==4.1.0 # via -r requirements_dev.in -pytest-flake8==1.1.0 +pytest-flake8==1.1.1 # via -r requirements_dev.in -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # -r requirements.txt # botocore # pandas -python-dotenv==1.0.0 - # via -r requirements.txt -python-engineio==3.13.1 +python-engineio==4.9.0 # via # -r requirements.txt # python-socketio -python-socketio==4.6.0 +python-socketio==5.11.1 # via # -r requirements.txt # flask-socketio -pytz==2023.3 +pytz==2024.1 # via # -r requirements.txt # pandas @@ -254,80 +279,70 @@ pyyaml==6.0.1 # via # -r requirements.txt # bokeh -requests==2.24.0 +requests==2.31.0 # via # -r requirements.txt # docker -ruamel-yaml==0.17.32 +ruamel-yaml==0.18.6 # via # -r requirements.txt # mitmproxy -ruamel-yaml-clib==0.2.7 +ruamel-yaml-clib==0.2.8 # via # -r requirements.txt # ruamel-yaml -s3transfer==0.6.2 +s3transfer==0.1.13 # via boto3 -six==1.15.0 +service-identity==24.1.0 # via # -r requirements.txt - # python-dateutil - # python-engineio - # python-socketio -sortedcontainers==2.4.0 + # aioquic +simple-websocket==1.0.0 # via # -r requirements.txt - # mitmproxy -soupsieve==2.3.2.post1 + # python-engineio +six==1.16.0 # via # -r requirements.txt - # beautifulsoup4 -toml==0.10.2 + # python-dateutil +sortedcontainers==2.4.0 # via # -r requirements.txt - # autopep8 -tomli==2.0.1 - # via - # coverage - # pylint - # pytest -tomlkit==0.12.1 + # mitmproxy +tomlkit==0.12.4 # via pylint -tornado==6.3.3 +tornado==6.4 # via # -r requirements.txt # bokeh # mitmproxy -typing-extensions==4.7.1 - # via astroid -urllib3==1.25.9 +tzdata==2024.1 # via # -r requirements.txt - # botocore - # requests -urwid==2.1.2 + # pandas +urllib3==2.2.1 # via # -r requirements.txt - # mitmproxy -websocket-client==1.2.3 + # docker + # requests +urwid-mitmproxy==2.1.2.1 # via # -r requirements.txt - # docker -werkzeug==1.0.1 + # mitmproxy +werkzeug==3.0.1 # via # -r requirements.txt # flask -wrapt==1.15.0 - # via astroid wsproto==1.2.0 # via # -r requirements.txt # mitmproxy -xyzservices==2023.7.0 + # simple-websocket +xyzservices==2023.10.1 # via # -r requirements.txt # bokeh -zstandard==0.19.0 +zstandard==0.22.0 # via # -r requirements.txt # mitmproxy