diff --git a/src/scripts/breast_cancer_analyses/breast_cancer_analyses.py b/src/scripts/breast_cancer_analyses/breast_cancer_analyses.py index e4456a9856..0e185d12bf 100644 --- a/src/scripts/breast_cancer_analyses/breast_cancer_analyses.py +++ b/src/scripts/breast_cancer_analyses/breast_cancer_analyses.py @@ -50,8 +50,13 @@ def run_sim(service_availability): + # configure logging + log_config = { + 'filename': 'LogFile', + 'directory': outputpath, + } # Establish the simulation object and set the seed - sim = Simulation(start_date=start_date, seed=0) + sim = Simulation(start_date=start_date, seed=0, log_config=log_config) # Register the appropriate modules sim.register(demography.Demography(resourcefilepath=resourcefilepath), @@ -71,14 +76,11 @@ def run_sim(service_availability): breast_cancer.BreastCancer(resourcefilepath=resourcefilepath) ) - # Establish the logger - logfile = sim.configure_logging(filename="LogFile") - # Run the simulation sim.make_initial_population(n=popsize) sim.simulate(end_date=end_date) - return logfile + return sim.log_filepath def get_summary_stats(logfile): diff --git a/src/scripts/breast_cancer_analyses/calc_5y_survival_following_treatment.py b/src/scripts/breast_cancer_analyses/calc_5y_survival_following_treatment.py index 3a18b9e451..5d2d34daeb 100644 --- a/src/scripts/breast_cancer_analyses/calc_5y_survival_following_treatment.py +++ b/src/scripts/breast_cancer_analyses/calc_5y_survival_following_treatment.py @@ -41,8 +41,18 @@ end_date = Date(2025, 1, 1) popsize = 10000 + +# Establish the logger and look at only demography +log_config = { + 'filename': 'LogFile', + 'custom_levels': { + '*': logging.WARNING, # <-- + 'tlo.methods.demography': logging.INFO + } + } + # Establish the simulation object and set the seed -sim = Simulation(start_date=start_date, seed=0) +sim = Simulation(start_date=start_date, seed=0, log_config=log_config) # Register the appropriate modules sim.register(demography.Demography(resourcefilepath=resourcefilepath), @@ -68,19 +78,12 @@ bc_parameters["init_prop_treatment_status_breast_cancer"] = [0.0] * 4 bc_parameters["init_prob_palliative_care"] = 0.0 -# Establish the logger and look at only demography -custom_levels = {"*": logging.WARNING, # <-- - "tlo.methods.demography": logging.INFO - } -logfile = sim.configure_logging(filename="LogFile", custom_levels=custom_levels) - - # Run the simulation sim.make_initial_population(n=popsize) sim.simulate(end_date=end_date) # Read the output: -output = parse_log_file(logfile) +output = parse_log_file(sim.log_filepath) # %% Analyse the output: diff --git a/src/scripts/oesophagealcancer_analyses/calc_5y_survival_following_treatment.py b/src/scripts/oesophagealcancer_analyses/calc_5y_survival_following_treatment.py index 192b6d9fa8..e54451d516 100644 --- a/src/scripts/oesophagealcancer_analyses/calc_5y_survival_following_treatment.py +++ b/src/scripts/oesophagealcancer_analyses/calc_5y_survival_following_treatment.py @@ -40,8 +40,17 @@ end_date = Date(2080, 1, 1) popsize = 1000 +# Establish the logger and look at only demography +log_config = { + 'filename': 'LogFile', + 'custom_levels': { + '*': logging.WARNING, # <-- + 'tlo.methods.demography': logging.INFO + } +} + # Establish the simulation object and set the seed -sim = Simulation(start_date=start_date, seed=0) +sim = Simulation(start_date=start_date, seed=0, log_config=log_config) # Register the appropriate modules sim.register(demography.Demography(resourcefilepath=resourcefilepath), @@ -66,19 +75,13 @@ sim.modules['OesophagealCancer'].parameters["init_prop_treatment_status_oes_cancer"] = [0.0] * 6 sim.modules['OesophagealCancer'].parameters["init_prob_palliative_care"] = 0.0 -# Establish the logger and look at only demography -custom_levels = {"*": logging.WARNING, # <-- - "tlo.methods.demography": logging.INFO - } -logfile = sim.configure_logging(filename="LogFile", custom_levels=custom_levels) - # Run the simulation sim.make_initial_population(n=popsize) sim.simulate(end_date=end_date) # Read the output: -output = parse_log_file(logfile) +output = parse_log_file(sim.log_filepath) # %% Analyse the output: diff --git a/src/scripts/oesophagealcancer_analyses/oesophagealcancer_analyses.py b/src/scripts/oesophagealcancer_analyses/oesophagealcancer_analyses.py index 23e6c183ed..eb66364737 100644 --- a/src/scripts/oesophagealcancer_analyses/oesophagealcancer_analyses.py +++ b/src/scripts/oesophagealcancer_analyses/oesophagealcancer_analyses.py @@ -48,8 +48,12 @@ def run_sim(service_availability): + log_config = { + 'filename': 'LogFile', + 'directory': outputpath + } # Establish the simulation object and set the seed - sim = Simulation(start_date=start_date, seed=0) + sim = Simulation(start_date=start_date, seed=0, log_config=log_config) # Register the appropriate modules sim.register(demography.Demography(resourcefilepath=resourcefilepath), @@ -68,14 +72,11 @@ def run_sim(service_availability): postnatal_supervisor.PostnatalSupervisor(resourcefilepath=resourcefilepath) ) - # Establish the logger - logfile = sim.configure_logging(filename="LogFile") - # Run the simulation sim.make_initial_population(n=popsize) sim.simulate(end_date=end_date) - return logfile + return sim.log_filepath def get_summary_stats(logfile): diff --git a/src/scripts/other_adult_cancers_analyses/calc_5y_survival_following_treatment.py b/src/scripts/other_adult_cancers_analyses/calc_5y_survival_following_treatment.py index 99631447c9..a50ec7f9a6 100644 --- a/src/scripts/other_adult_cancers_analyses/calc_5y_survival_following_treatment.py +++ b/src/scripts/other_adult_cancers_analyses/calc_5y_survival_following_treatment.py @@ -41,8 +41,20 @@ end_date = Date(2080, 1, 1) popsize = 1000 +# Establish the logger and look at only demography +log_config = { + 'filename': 'LogFile', + 'directory': outputpath, + 'custom_levels': { + '*': logging.WARNING, # <-- + 'tlo.methods.demography': logging.INFO, + 'tlo.methods.other_adult_cancers': logging.INFO, + 'tlo.methods.healthsystem': logging.INFO, + } +} + # Establish the simulation object and set the seed -sim = Simulation(start_date=start_date, seed=0) +sim = Simulation(start_date=start_date, seed=0, log_config=log_config) # Register the appropriate modules sim.register(demography.Demography(resourcefilepath=resourcefilepath), @@ -66,21 +78,12 @@ # sim.modules['OtherAdultCancer'].parameters["init_prop_treatment_status_other_adult_cancer"] = [0.0] * 3 # sim.modules['OtherAdultCancer'].parameters["init_prob_palliative_care"] = 0.0 -# Establish the logger and look at only demography -custom_levels = {"*": logging.WARNING, # <-- - "tlo.methods.demography": logging.INFO, - "tlo.methods.other_adult_cancers": logging.INFO, - 'tlo.methods.healthsystem': logging.INFO, - } -logfile = sim.configure_logging(filename="LogFile", custom_levels=custom_levels) - - # Run the simulation sim.make_initial_population(n=popsize) sim.simulate(end_date=end_date) # Read the output: -output = parse_log_file(logfile) +output = parse_log_file(sim.log_filepath) # %% Analyse the output: diff --git a/src/scripts/other_adult_cancers_analyses/other_adult_cancers_analyses.py b/src/scripts/other_adult_cancers_analyses/other_adult_cancers_analyses.py index ff17364713..310541dc71 100644 --- a/src/scripts/other_adult_cancers_analyses/other_adult_cancers_analyses.py +++ b/src/scripts/other_adult_cancers_analyses/other_adult_cancers_analyses.py @@ -44,8 +44,12 @@ def run_sim(allow_hsi): + log_config = { + 'filename': 'LogFile', + 'directory': outputpath + } # Establish the simulation object and set the seed - sim = Simulation(start_date=start_date, seed=0) + sim = Simulation(start_date=start_date, seed=0, log_config=log_config) # Register the appropriate modules sim.register(demography.Demography(resourcefilepath=resourcefilepath), @@ -60,14 +64,11 @@ def run_sim(allow_hsi): other_adult_cancers.OtherAdultCancer(resourcefilepath=resourcefilepath), ) - # Establish the logger - logfile = sim.configure_logging(filename="LogFile") - # Run the simulation sim.make_initial_population(n=popsize) sim.simulate(end_date=end_date) - return logfile + return sim.log_filepath def get_summary_stats(logfile): diff --git a/src/scripts/prostate_cancer_analyses/calc_5y_survival_following_treatment.py b/src/scripts/prostate_cancer_analyses/calc_5y_survival_following_treatment.py index 026650fc43..2e56e132c0 100644 --- a/src/scripts/prostate_cancer_analyses/calc_5y_survival_following_treatment.py +++ b/src/scripts/prostate_cancer_analyses/calc_5y_survival_following_treatment.py @@ -41,8 +41,17 @@ end_date = Date(2030, 1, 1) popsize = 10000 +# Establish the logger and look at only demography +log_config = { + 'filename': 'LogFile', + 'directory': outputpath, + 'custom_levels': {"*": logging.WARNING, # <-- + "tlo.methods.demography": logging.INFO + } + } + # Establish the simulation object and set the seed -sim = Simulation(start_date=start_date, seed=0) +sim = Simulation(start_date=start_date, seed=0, log_config=log_config) # Register the appropriate modules sim.register(demography.Demography(resourcefilepath=resourcefilepath), @@ -69,19 +78,13 @@ # sim.modules['ProstateCancer'].parameters["init_prop_treatment_status_prostate_cancer"] = [0.0] * 3 # sim.modules['ProstateCancer'].parameters["init_prob_palliative_care"] = 0.0 -# Establish the logger and look at only demography -custom_levels = {"*": logging.WARNING, # <-- - "tlo.methods.demography": logging.INFO - } -logfile = sim.configure_logging(filename="LogFile", custom_levels=custom_levels) - # Run the simulation sim.make_initial_population(n=popsize) sim.simulate(end_date=end_date) # Read the output: -output = parse_log_file(logfile) +output = parse_log_file(sim.log_filepath) # %% Analyse the output: @@ -106,16 +109,17 @@ # calc % of those that were alive 5 years after starting treatment (not died of any cause): 1 - ( - len(cohort_treated.loc[cohort_treated['days_treatment_to_death'] < (5*365.25)]) / - len(cohort_treated) + len(cohort_treated.loc[cohort_treated['days_treatment_to_death'] < (5 * 365.25)]) / + len(cohort_treated) ) # 0.77 # calc % of those that had not died of prostate cancer 5 years after starting treatment (could have died of another # cause): 1 - ( - len( - cohort_treated.loc[ - (cohort_treated['cause'] == 'ProstateCancer') & (cohort_treated['days_treatment_to_death'] < (5*365.25)) - ] - ) / len(cohort_treated) -) # 0.87 + len( + cohort_treated.loc[ + (cohort_treated['cause'] == 'ProstateCancer') & ( + cohort_treated['days_treatment_to_death'] < (5 * 365.25)) + ] + ) / len(cohort_treated) +) # 0.87 diff --git a/src/scripts/prostate_cancer_analyses/prostate_cancer_analyses.py b/src/scripts/prostate_cancer_analyses/prostate_cancer_analyses.py index b4b525b0f8..0d04a93fd1 100644 --- a/src/scripts/prostate_cancer_analyses/prostate_cancer_analyses.py +++ b/src/scripts/prostate_cancer_analyses/prostate_cancer_analyses.py @@ -50,8 +50,12 @@ def run_sim(service_availability): + log_config = { + 'filename': 'LogFile', + 'directory': outputpath + } # Establish the simulation object and set the seed - sim = Simulation(start_date=start_date, seed=0) + sim = Simulation(start_date=start_date, seed=0, log_config=log_config) # Register the appropriate modules sim.register(care_of_women_during_pregnancy.CareOfWomenDuringPregnancy(resourcefilepath=resourcefilepath), @@ -71,14 +75,11 @@ def run_sim(service_availability): postnatal_supervisor.PostnatalSupervisor(resourcefilepath=resourcefilepath) ) - # Establish the logger - logfile = sim.configure_logging(filename="LogFile") - # Run the simulation sim.make_initial_population(n=popsize) sim.simulate(end_date=end_date) - return logfile + return sim.log_filepath def get_summary_stats(logfile): @@ -248,4 +249,4 @@ def get_cols_excl_none(allcols, stub): # ** 5-year survival following treatment -# See sepaerate file +# See separate file diff --git a/src/scripts/schistosomiasis/schisto_analysis.py b/src/scripts/schistosomiasis/schisto_analysis.py index d33890c34e..cfed7b3528 100644 --- a/src/scripts/schistosomiasis/schisto_analysis.py +++ b/src/scripts/schistosomiasis/schisto_analysis.py @@ -16,14 +16,24 @@ def run_simulation(popsize=10000, haem=True, mansoni=True, mda_execute=True): + outputpath = Path("./outputs") # folder for convenience of storing outputs # The resource files resourcefilepath = Path("./resources") start_date = Date(2010, 1, 1) end_date = Date(2011, 2, 1) popsize = popsize + # Sets all modules to WARNING threshold, then alters schisto to INFO + log_config = { + 'filename': 'LogFile', + 'directory': outputpath, + 'custom_levels ': {"*": logging.WARNING, + "tlo.methods.schisto": logging.INFO, + } + } + # Establish the simulation object - sim = Simulation(start_date=start_date) + sim = Simulation(start_date=start_date, log_config=log_config) # Register the appropriate modules sim.register(demography.Demography(resourcefilepath=resourcefilepath)) @@ -36,13 +46,6 @@ def run_simulation(popsize=10000, haem=True, mansoni=True, mda_execute=True): if mansoni: sim.register(schisto.Schisto_Mansoni(resourcefilepath=resourcefilepath, symptoms_and_HSI=False)) - # Sets all modules to WARNING threshold, then alters schisto to INFO - custom_levels = {"*": logging.WARNING, - "tlo.methods.schisto": logging.INFO, - } - # configure logging after registering modules with custom levels - logfile = sim.configure_logging(filename="LogFile", custom_levels=custom_levels) - # Run the simulation sim.seed_rngs(int(np.random.uniform(0, 1) * 0 + 1000)) # initialise the population @@ -51,12 +54,13 @@ def run_simulation(popsize=10000, haem=True, mansoni=True, mda_execute=True): # # start the simulation sim.simulate(end_date=end_date) # fh.flush() - output = parse_log_file(logfile) + output = parse_log_file(sim.log_filepath) return sim, output sim, output = run_simulation(popsize=10000, haem=True, mansoni=False, mda_execute=False) + # --------------------------------------------------------------------------------------------------------- # Saving the results - prevalence, mwb, dalys and parameters used # --------------------------------------------------------------------------------------------------------- @@ -286,6 +290,8 @@ def plot_prev_high_infection_per_district(infection, high_inf_distr): districts_prevalence, districts_mwb, districts_high_inf_prev = get_values_per_district('Haematobium') expected_district_prevalence = get_expected_prevalence('Haematobium') plot_prevalence_per_district('Haematobium', districts_prevalence, expected_district_prevalence) + + # plot_mwb_per_district('Haematobium', districts_mwb) # plot_prev_high_infection_per_district('Haematobium', districts_high_inf_prev) @@ -427,7 +433,6 @@ def plot_high_inf_years(): plot_prevalent_years() plot_high_inf_years() - # DALYS loger_daly = output['tlo.methods.healthburden']["DALYS"] loger_daly.drop(columns=['sex', 'YLL_Demography_Other'], inplace=True) diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index 219b1b8a6f..544ae0d68f 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -69,7 +69,7 @@ def __init__(self, *, start_date: Date, seed: int = None, log_config: dict = Non log_config = {} self._custom_log_levels = None self._log_filepath = None - self.configure_logging(**log_config) + self._configure_logging(**log_config) # random number generator seed_from = 'auto' if seed is None else 'user' @@ -81,8 +81,8 @@ def __init__(self, *, start_date: Date, seed: int = None, log_config: dict = Non ) self.rng = np.random.RandomState(np.random.MT19937(self._seed_seq)) - def configure_logging(self, filename: str = None, directory: Union[Path, str] = "./outputs", - custom_levels: Dict[str, int] = None, suppress_stdout: bool = False): + def _configure_logging(self, filename: str = None, directory: Union[Path, str] = "./outputs", + custom_levels: Dict[str, int] = None, suppress_stdout: bool = False): """Configure logging, can write logging to a logfile in addition the default of stdout. Minimum custom levels for each logger can be specified for filtering out messages diff --git a/tests/test_analysis.py b/tests/test_analysis.py index a3c4282e61..2686e431b0 100644 --- a/tests/test_analysis.py +++ b/tests/test_analysis.py @@ -88,7 +88,7 @@ def initialise_simulation(self, sim: Simulation): # At INFO level assert ( - len(output["tlo.methods.dummy"]["_metadata"]["tlo.methods.dummy"]) == 2 + len(output["tlo.methods.dummy"]["_metadata"]["tlo.methods.dummy"]) == 2 ) # should have two tables # tables should be at level INFO @@ -148,7 +148,7 @@ def initialise_simulation(self, sim): sim = Simulation( start_date=sim_start_date, seed=0, - log_config={"filename": "temp", "directory": tmpdir,}, + log_config={"filename": "temp", "directory": tmpdir, }, ) sim.register( demography.Demography(resourcefilepath=resourcefilepath), DummyModule() @@ -368,7 +368,7 @@ def test_get_parameter_functions(seed): # Check that the parameter identified exists in the simulation assert ( - name in sim.modules[module].parameters + name in sim.modules[module].parameters ), f"Parameter not recognised: {module}:{name}." # Check that the original value and the updated value are of the same type. @@ -412,9 +412,9 @@ def is_list_same_size_and_dtype(l1, l2): def test_mix_scenarios(): """Check that `mix_scenarios` works as expected.""" - d1 = {"Mod1": {"param_a": "value_in_d1", "param_b": "value_in_d1",}} + d1 = {"Mod1": {"param_a": "value_in_d1", "param_b": "value_in_d1", }} - d2 = {"Mod2": {"param_a": "value_in_d2", "param_b": "value_in_d2",}} + d2 = {"Mod2": {"param_a": "value_in_d2", "param_b": "value_in_d2", }} d3 = {"Mod1": {"param_b": "value_in_d3", "param_c": "value_in_d3"}} @@ -433,8 +433,8 @@ def test_mix_scenarios(): assert 1 == len(record) assert ( - record.list[0].message.args[0] - == "Parameter is being updated more than once: module=Mod1, parameter=param_b" + record.list[0].message.args[0] + == "Parameter is being updated more than once: module=Mod1, parameter=param_b" ) # Test the behaviour of the `mix_scenarios` taking the value in the right-most dict. @@ -460,15 +460,15 @@ def test_mix_scenarios(): "param_c": "value_in_dict3", } }, - {"Mod1": {"param_a": "value_in_dict_right_most", "param_c": "value_in_dict4",}}, - {"Mod1": {"param_c": "value_in_dict_right_most",}}, + {"Mod1": {"param_a": "value_in_dict_right_most", "param_c": "value_in_dict4", }}, + {"Mod1": {"param_c": "value_in_dict_right_most", }}, ) == { - "Mod1": { - "param_a": "value_in_dict_right_most", - "param_b": "value_in_dict_right_most", - "param_c": "value_in_dict_right_most", - } - } + "Mod1": { + "param_a": "value_in_dict_right_most", + "param_b": "value_in_dict_right_most", + "param_c": "value_in_dict_right_most", + } + } def test_improved_healthsystem_and_care_seeking_scenario_switcher(seed): @@ -538,24 +538,23 @@ def check_parameters(self) -> None: hcs = sim.modules["HealthSeekingBehaviour"].force_any_symptom_to_lead_to_healthcareseeking assert isinstance(hcs, bool) and (hcs is max_healthcare_seeking[phase_of_simulation]) - sim = Simulation(start_date=Date(2010, 1, 1), seed=seed) sim.register( *( - fullmodel(resourcefilepath=resourcefilepath) - + [ - ImprovedHealthSystemAndCareSeekingScenarioSwitcher( - resourcefilepath=resourcefilepath - ), - DummyModule(), - ] + fullmodel(resourcefilepath=resourcefilepath) + + [ + ImprovedHealthSystemAndCareSeekingScenarioSwitcher( + resourcefilepath=resourcefilepath + ), + DummyModule(), + ] ) ) # Check that the `ImprovedHealthSystemAndCareSeekingScenarioSwitcher` is the first registered module. assert ( - "ImprovedHealthSystemAndCareSeekingScenarioSwitcher" - == list(sim.modules.keys())[0] + "ImprovedHealthSystemAndCareSeekingScenarioSwitcher" + == list(sim.modules.keys())[0] ) module = sim.modules["ImprovedHealthSystemAndCareSeekingScenarioSwitcher"] @@ -586,7 +585,7 @@ def test_summarize(): names=("draw", "run"), ), index=["TimePoint0", "TimePoint1"], - data=np.array([[0, 20, 1000, 2000], [0, 20, 1000, 2000],]), + data=np.array([[0, 20, 1000, 2000], [0, 20, 1000, 2000], ]), ) results_one_draw = pd.DataFrame( @@ -637,7 +636,51 @@ def test_summarize(): pd.DataFrame( columns=pd.Index(["lower", "mean", "upper"], name="stat"), index=["TimePoint0", "TimePoint1"], - data=np.array([[0.5, 10.0, 19.5], [0.5, 10.0, 19.5],]), + data=np.array([[0.5, 10.0, 19.5], [0.5, 10.0, 19.5], ]), ), summarize(results_one_draw, collapse_columns=True), ) + + +def test_control_loggers_from_same_module_independently(seed, tmpdir): + """Check that detailed/summary loggers in the same module can configured independently.""" + + # Check that the simulation can be set-up to get only the usual demography logger and *not* the detailed + # logger, when providing the config_log information when the simulation is initialised.""" + + log_config = { + 'filename': 'temp', + 'directory': tmpdir, + 'custom_levels': { + "*": logging.WARNING, + 'tlo.methods.demography.detail': logging.WARNING, # <-- Don't explicitly turn off the detailed logger + 'tlo.methods.demography': logging.INFO, # <-- Turning on the normal logger + } + } + + def run_simulation_and_cause_one_death(sim): + """Register demography in the simulations, runs it and causes one death; return the resulting log.""" + sim.register(demography.Demography(resourcefilepath=resourcefilepath)) + sim.make_initial_population(n=100) + sim.simulate(end_date=sim.start_date) + # Cause one death to occur + sim.modules['Demography'].do_death( + individual_id=0, + originating_module=sim.modules['Demography'], + cause='Other' + ) + return parse_log_file(sim.log_filepath) + + def check_log(log): + """Check the usual `tlo.methods.demography' log is created and that check persons have died (which would be + when the detailed logger would be used).""" + assert 'tlo.methods.demography' in log.keys() + assert 1 == len(log['tlo.methods.demography']['death']) + + # Check that the detailed logger is not created. + assert 'tlo.methods.demography.detail' not in log.keys() + + # 1) Provide custom_logs argument when creating Simulation object + sim = Simulation(start_date=Date(2010, 1, 1), seed=seed, log_config=log_config) + check_log(run_simulation_and_cause_one_death(sim)) +