WIP: add fixes and improvements

bci/database/mongo/mongodb.py

@@ -145,42 +145,35 @@ def has_all_results(self, params: WorkerParameters) -> bool:
                 return False
         return True
 
-    def get_evaluated_states(self, params: EvaluationParameters, outcome_checker: OutcomeChecker) -> list[State]:
+    def get_evaluated_states(
+        self, params: EvaluationParameters, boundary_states: tuple[State, State], outcome_checker: OutcomeChecker
+    ) -> list[State]:
         collection = self.get_collection(params.database_collection)
         query = {
             'browser_config': params.browser_configuration.browser_setting,
             'mech_group': params.evaluation_range.mech_groups[0],  # TODO: fix this
             'state.browser_name': params.browser_configuration.browser_name,
             'results': {'$exists': True},
+            'state.type': 'version' if params.evaluation_range.only_release_revisions else 'revision',
+            'state.revision_number': {
+                '$gte': boundary_states[0].revision_nb,
+                '$lte': boundary_states[1].revision_nb,
+            },
         }
-        if params.evaluation_range.major_version_range:
-            query.update(
-                {
-                    'state.type': 'version',
-                    'state.major_version': {
-                        '$gte': params.evaluation_range.major_version_range[0],
-                        '$lte': params.evaluation_range.major_version_range[1],
-                    },
-                }
-            )
-        elif params.evaluation_range.revision_number_range:
-            query.update(
-                {
-                    'state.type': 'revision',
-                    'state.revision_number': {
-                        '$gte': params.evaluation_range.revision_number_range[0],
-                        '$lte': params.evaluation_range.revision_number_range[1],
-                    },
-                }
-            )
-        query['extensions'] = {
-            '$size': len(params.browser_configuration.extensions),
-            '$all': params.browser_configuration.extensions,
-        }
-        query['cli_options'] = {
-            '$size': len(params.browser_configuration.cli_options),
-            '$all': params.browser_configuration.cli_options,
-        }
+        if params.browser_configuration.extensions:
+            query['extensions'] = {
+                '$size': len(params.browser_configuration.extensions),
+                '$all': params.browser_configuration.extensions,
+            }
+        else:
+            query['extensions'] = []
+        if params.browser_configuration.cli_options:
+            query['cli_options'] = {
+                '$size': len(params.browser_configuration.cli_options),
+                '$all': params.browser_configuration.cli_options,
+            }
+        else:
+            query['cli_options'] = []
         cursor = collection.find(query)
         states = []
         for doc in cursor:
@@ -231,7 +224,7 @@ def get_binary_availability_collection(browser_name: str):
     @staticmethod
     def has_binary_available_online(browser: str, state: State):
         collection = MongoDB.get_binary_availability_collection(browser)
-        document = collection.find_one({'state': state.to_dict(make_complete=False)})
+        document = collection.find_one({'state': state.to_dict()})
         if document is None:
             return None
         return document['binary_online']
@@ -251,10 +244,10 @@ def get_stored_binary_availability(browser):
         return result
 
     @staticmethod
-    def get_complete_state_dict_from_binary_availability_cache(state: State):
+    def get_complete_state_dict_from_binary_availability_cache(state: State) -> dict:
         collection = MongoDB.get_binary_availability_collection(state.browser_name)
         # We have to flatten the state dictionary to ignore missing attributes.
-        state_dict = {'state': state.to_dict(make_complete=False)}
+        state_dict = {'state': state.to_dict()}
         query = flatten(state_dict, reducer='dot')
         document = collection.find_one(query)
         if document is None:

bci/distribution/worker_manager.py

@@ -10,6 +10,7 @@
 from bci import worker
 from bci.configuration import Global
 from bci.evaluations.logic import WorkerParameters
+from bci.web.clients import Clients
 
 logger = logging.getLogger(__name__)
 
@@ -83,6 +84,7 @@ def start_container_thread():
                     ],
                 )
                 logger.debug(f"Container '{container_name}' finished experiments with parameters '{repr(params)}'")
+                Clients.push_results_to_all()
             except docker.errors.APIError:
                 logger.error(
                     f"Could not run container '{container_name}' or container was unexpectedly removed", exc_info=True
@@ -94,7 +96,7 @@ def start_container_thread():
         thread.start()
         logger.info(f"Container '{container_name}' started experiments for '{params.state}'")
         # To avoid race-condition where more than max containers are started
-        time.sleep(5)
+        time.sleep(3)
 
     def get_nb_of_running_worker_containers(self):
         return len(self.get_runnning_containers())

bci/evaluations/evaluation_framework.py

@@ -7,6 +7,7 @@
 from bci.configuration import Global
 from bci.database.mongo.mongodb import MongoDB
 from bci.evaluations.logic import TestParameters, TestResult, WorkerParameters
+from bci.version_control.states.state import StateCondition
 
 logger = logging.getLogger(__name__)
 
@@ -39,12 +40,10 @@ def evaluate(self, worker_params: WorkerParameters):
             try:
                 browser.pre_test_setup()
                 result = self.perform_specific_evaluation(browser, test_params)
-
-                state.set_evaluation_outcome(result)
                 self.db_class.get_instance().store_result(result)
                 logger.info(f'Test finalized: {test_params}')
             except Exception as e:
-                state.set_evaluation_error(str(e))
+                state.condition = StateCondition.FAILED
                 logger.error("An error occurred during evaluation", exc_info=True)
                 traceback.print_exc()
             finally:

bci/evaluations/outcome_checker.py

@@ -1,23 +1,22 @@
 import re
 
-from bci.evaluations.logic import SequenceConfiguration, StateResult
-from bci.version_control.states.state import State
+from bci.evaluations.logic import SequenceConfiguration
+from bci.version_control.states.state import StateResult
 
 
 class OutcomeChecker:
 
     def __init__(self, sequence_config: SequenceConfiguration):
         self.sequence_config = sequence_config
 
-    def get_outcome(self, state: State) -> bool | None:
+    def get_outcome(self, result: StateResult) -> bool | None:
         '''
         Returns the outcome of the test result.
 
         - None in case of an error.
         - True if the test was reproduced.
         - False if the test was not reproduced.
         '''
-        result = state.result
         if result.is_dirty:
             return None
         if result.reproduced:

bci/master.py

@@ -9,31 +9,23 @@
 from bci.evaluations.logic import (
     DatabaseConnectionParameters,
     EvaluationParameters,
-    SequenceConfiguration,
-    WorkerParameters,
 )
 from bci.evaluations.outcome_checker import OutcomeChecker
 from bci.evaluations.samesite.samesite_evaluation import SameSiteEvaluationFramework
 from bci.evaluations.xsleaks.evaluation import XSLeaksEvaluation
+from bci.search_strategy.bgb_search import BiggestGapBisectionSearch
+from bci.search_strategy.bgb_sequence import BiggestGapBisectionSequence
 from bci.search_strategy.composite_search import CompositeSearch
-from bci.search_strategy.n_ary_search import NArySearch
-from bci.search_strategy.n_ary_sequence import NArySequence, SequenceFinished
-from bci.search_strategy.sequence_strategy import SequenceStrategy
-from bci.version_control import factory
-from bci.version_control.states.state import State
+from bci.search_strategy.sequence_strategy import SequenceFinished, SequenceStrategy
+from bci.version_control.factory import StateFactory
 from bci.web.clients import Clients
 
 logger = logging.getLogger(__name__)
 
 
 class Master:
-
     def __init__(self):
-        self.state = {
-            'is_running': False,
-            'reason': 'init',
-            'status': 'idle'
-        }
+        self.state = {'is_running': False, 'reason': 'init', 'status': 'idle'}
 
         self.stop_gracefully = False
         self.stop_forcefully = False
@@ -50,20 +42,16 @@ def __init__(self):
         self.db_connection_params = Global.get_database_connection_params()
         self.connect_to_database(self.db_connection_params)
         self.inititialize_available_evaluation_frameworks()
-        logger.info("BugHog is ready!")
+        logger.info('BugHog is ready!')
 
     def connect_to_database(self, db_connection_params: DatabaseConnectionParameters):
         try:
             MongoDB.connect(db_connection_params)
         except ServerException:
-            logger.error("Could not connect to database.", exc_info=True)
+            logger.error('Could not connect to database.', exc_info=True)
 
     def run(self, eval_params: EvaluationParameters):
-        self.state = {
-            'is_running': True,
-            'reason': 'user',
-            'status': 'running'
-        }
+        self.state = {'is_running': True, 'reason': 'user', 'status': 'running'}
         self.stop_gracefully = False
         self.stop_forcefully = False
 
@@ -74,94 +62,74 @@ def run(self, eval_params: EvaluationParameters):
         evaluation_range = eval_params.evaluation_range
         sequence_config = eval_params.sequence_configuration
 
-        logger.info(f'Running experiments for {browser_config.browser_name} ({", ".join(evaluation_range.mech_groups)})')
-        self.evaluation_framework = self.get_specific_evaluation_framework(
-            evaluation_config.project
+        logger.info(
+            f'Running experiments for {browser_config.browser_name} ({", ".join(evaluation_range.mech_groups)})'
         )
+        self.evaluation_framework = self.get_specific_evaluation_framework(evaluation_config.project)
         self.worker_manager = WorkerManager(sequence_config.nb_of_containers)
 
         try:
-            state_list = factory.create_state_collection(browser_config, evaluation_range)
-
-            search_strategy = self.parse_search_strategy(sequence_config, state_list)
-
-            outcome_checker = OutcomeChecker(sequence_config)
-
-            # The state_lineage is put into self.evaluation as a means to check on the process through front-end
-            # self.evaluations.append(state_list)
+            search_strategy = self.create_sequence_strategy(eval_params)
 
             try:
-                current_state = search_strategy.next()
                 while (self.stop_gracefully or self.stop_forcefully) is False:
-                    worker_params = eval_params.create_worker_params_for(current_state, self.db_connection_params)
-
-                    # Callback function for sequence strategy
-                    update_outcome = self.get_update_outcome_cb(search_strategy, worker_params, sequence_config, outcome_checker)
+                    # Update search strategy with new potentially new results
+                    current_state = search_strategy.next()
 
-                    # Check whether state is already evaluated
-                    if self.evaluation_framework.has_all_results(worker_params):
-                        logger.info(f"'{current_state}' already evaluated.")
-                        update_outcome()
-                        current_state = search_strategy.next()
-                        continue
+                    # Prepare worker parameters
+                    worker_params = eval_params.create_worker_params_for(current_state, self.db_connection_params)
 
                     # Start worker to perform evaluation
-                    self.worker_manager.start_test(worker_params, update_outcome)
+                    self.worker_manager.start_test(worker_params)
 
-                    current_state = search_strategy.next()
             except SequenceFinished:
-                logger.debug("Last experiment has started")
+                logger.debug('Last experiment has started')
                 self.state['reason'] = 'finished'
 
         except Exception as e:
-            logger.critical("A critical error occurred", exc_info=True)
+            logger.critical('A critical error occurred', exc_info=True)
             raise e
         finally:
             # Gracefully exit
             if self.stop_gracefully:
-                logger.info("Gracefully stopping experiment queue due to user end signal...")
+                logger.info('Gracefully stopping experiment queue due to user end signal...')
                 self.state['reason'] = 'user'
             if self.stop_forcefully:
-                logger.info("Forcefully stopping experiment queue due to user end signal...")
+                logger.info('Forcefully stopping experiment queue due to user end signal...')
                 self.state['reason'] = 'user'
                 self.worker_manager.forcefully_stop_all_running_containers()
             else:
-                logger.info("Gracefully stopping experiment queue since last experiment started.")
+                logger.info('Gracefully stopping experiment queue since last experiment started.')
             # MongoDB.disconnect()
-            logger.info("Waiting for remaining experiments to stop...")
+            logger.info('Waiting for remaining experiments to stop...')
             self.worker_manager.wait_until_all_evaluations_are_done()
-            logger.info("BugHog has finished the evaluation!")
+            logger.info('BugHog has finished the evaluation!')
             self.state['is_running'] = False
             self.state['status'] = 'idle'
             Clients.push_info_to_all('is_running', 'state')
 
-    @staticmethod
-    def get_update_outcome_cb(search_strategy: SequenceStrategy, worker_params: WorkerParameters, sequence_config: SequenceConfiguration, checker: OutcomeChecker) -> None:
-        def cb():
-            if sequence_config.target_mech_id is not None and len(worker_params.mech_groups) == 1:
-                result = MongoDB.get_instance().get_result(worker_params.create_test_params_for(worker_params.mech_groups[0]))
-                outcome = checker.get_outcome(result)
-                search_strategy.update_outcome(worker_params.state, outcome)
-            # Just push results update to all clients. Could be more efficient, but would complicate things...
-            Clients.push_results_to_all()
-        return cb
-
     def inititialize_available_evaluation_frameworks(self):
-        self.available_evaluation_frameworks["samesite"] = SameSiteEvaluationFramework()
-        self.available_evaluation_frameworks["custom"] = CustomEvaluationFramework()
-        self.available_evaluation_frameworks["xsleaks"] = XSLeaksEvaluation()
+        self.available_evaluation_frameworks['samesite'] = SameSiteEvaluationFramework()
+        self.available_evaluation_frameworks['custom'] = CustomEvaluationFramework()
+        self.available_evaluation_frameworks['xsleaks'] = XSLeaksEvaluation()
 
     @staticmethod
-    def parse_search_strategy(sequence_config: SequenceConfiguration, state_list: list[State]):
+    def create_sequence_strategy(eval_params: EvaluationParameters) -> SequenceStrategy:
+        sequence_config = eval_params.sequence_configuration
         search_strategy = sequence_config.search_strategy
         sequence_limit = sequence_config.sequence_limit
-        if search_strategy == "bin_seq":
-            return NArySequence(state_list, 2, limit=sequence_limit)
-        if search_strategy == "bin_search":
-            return NArySearch(state_list, 2)
-        if search_strategy == "comp_search":
-            return CompositeSearch(state_list, 2, sequence_limit, NArySequence, NArySearch)
-        raise AttributeError("Unknown search strategy option '%s'" % search_strategy)
+        outcome_checker = OutcomeChecker(sequence_config)
+        state_factory = StateFactory(eval_params, outcome_checker)
+
+        if search_strategy == 'bgb_sequence':
+            strategy = BiggestGapBisectionSequence(state_factory, sequence_limit)
+        elif search_strategy == 'bgb_search':
+            strategy = BiggestGapBisectionSearch(state_factory)
+        elif search_strategy == 'comp_search':
+            strategy = CompositeSearch(state_factory, sequence_limit)
+        else:
+            raise AttributeError("Unknown search strategy option '%s'" % search_strategy)
+        return strategy
 
     def get_specific_evaluation_framework(self, evaluation_name: str) -> EvaluationFramework:
         # TODO: we always use 'custom', in which evaluation_name is a project
@@ -173,36 +141,28 @@ def get_specific_evaluation_framework(self, evaluation_name: str) -> EvaluationF
     def activate_stop_gracefully(self):
         if self.evaluation_framework:
             self.stop_gracefully = True
-            self.state = {
-                'is_running': True,
-                'reason': 'user',
-                'status': 'waiting_to_stop'
-            }
+            self.state = {'is_running': True, 'reason': 'user', 'status': 'waiting_to_stop'}
             Clients.push_info_to_all('state')
             self.evaluation_framework.stop_gracefully()
-            logger.info("Received user signal to gracefully stop.")
+            logger.info('Received user signal to gracefully stop.')
         else:
-            logger.info("Received user signal to gracefully stop, but no evaluation is running.")
+            logger.info('Received user signal to gracefully stop, but no evaluation is running.')
 
     def activate_stop_forcefully(self):
         if self.evaluation_framework:
             self.stop_forcefully = True
-            self.state = {
-                'is_running': True,
-                'reason': 'user',
-                'status': 'waiting_to_stop'
-            }
+            self.state = {'is_running': True, 'reason': 'user', 'status': 'waiting_to_stop'}
             Clients.push_info_to_all('state')
             self.evaluation_framework.stop_gracefully()
             if self.worker_manager:
                 self.worker_manager.forcefully_stop_all_running_containers()
-            logger.info("Received user signal to forcefully stop.")
+            logger.info('Received user signal to forcefully stop.')
         else:
-            logger.info("Received user signal to forcefully stop, but no evaluation is running.")
+            logger.info('Received user signal to forcefully stop, but no evaluation is running.')
 
     def stop_bughog(self):
-        logger.info("Stopping all running BugHog containers...")
+        logger.info('Stopping all running BugHog containers...')
         self.activate_stop_forcefully()
         mongodb_container.stop()
-        logger.info("Stopping BugHog core...")
+        logger.info('Stopping BugHog core...')
         exit(0)