diff --git a/conda/meta.yaml b/conda/meta.yaml index df7b4bac..072a4bda 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -19,10 +19,10 @@ requirements: - setuptools run: - numpy - - pandas==1.5.3 + - pandas - bw2io==0.8.7 - - bw2data==3.6.6 - - wurst==0.3.3 + - bw2data + - wurst - xarray - prettytable - pycountry diff --git a/premise/__init__.py b/premise/__init__.py index fc4159ce..8b7a3b1d 100644 --- a/premise/__init__.py +++ b/premise/__init__.py @@ -1,5 +1,5 @@ __all__ = ("NewDatabase", "clear_cache", "get_regions_definition") -__version__ = (1, 6, 6) +__version__ = (1, 7, 0) from pathlib import Path diff --git a/premise/data/additional_inventories/lci-home-batteries.xlsx b/premise/data/additional_inventories/lci-home-batteries.xlsx new file mode 100644 index 00000000..f675f003 Binary files /dev/null and b/premise/data/additional_inventories/lci-home-batteries.xlsx differ diff --git a/premise/data/additional_inventories/lci-hydrogen-electrolysis.xlsx b/premise/data/additional_inventories/lci-hydrogen-electrolysis.xlsx index 79d8ca76..ec74cf18 100644 Binary files a/premise/data/additional_inventories/lci-hydrogen-electrolysis.xlsx and b/premise/data/additional_inventories/lci-hydrogen-electrolysis.xlsx differ diff --git a/premise/data/additional_inventories/migration_map.csv b/premise/data/additional_inventories/migration_map.csv index a8e35b1c..8c4b6764 100644 --- a/premise/data/additional_inventories/migration_map.csv +++ b/premise/data/additional_inventories/migration_map.csv @@ -615,4 +615,5 @@ from;to;name_from;ref_prod_from;location_from;name_to;ref_prod_to;location_to 35;39;transmission network construction, electricity, high voltage;transmission network, long-distance;CH;transmission network construction, electricity, high voltage;transmission network, electricity, high voltage;CH 36;39;transmission network construction, electricity, high voltage;transmission network, long-distance;CH;transmission network construction, electricity, high voltage;transmission network, electricity, high voltage;CH 37;39;transmission network construction, electricity, high voltage;transmission network, long-distance;CH;transmission network construction, electricity, high voltage;transmission network, electricity, high voltage;CH -38;39;transmission network construction, electricity, high voltage;transmission network, long-distance;CH;transmission network construction, electricity, high voltage;transmission network, electricity, high voltage;CH \ No newline at end of file +38;39;transmission network construction, electricity, high voltage;transmission network, long-distance;CH;transmission network construction, electricity, high voltage;transmission network, electricity, high voltage;CH +39;38;market for battery cell, Li-ion, LFP;battery cell, Li-ion, LFP;GLO;market for battery cell, Li-ion;battery cell, Li-ion;GLO \ No newline at end of file diff --git a/premise/data_collection.py b/premise/data_collection.py index 9a5279be..dcd0af0f 100644 --- a/premise/data_collection.py +++ b/premise/data_collection.py @@ -146,11 +146,7 @@ def get_gains_EU_data() -> xr.DataArray: ) gains_emi_EU["sector"] = gains_emi_EU["Sector"] + gains_emi_EU["Activity"] gains_emi_EU.drop( - [ - "Sector", - "Activity", - ], - axis=1, + ["Sector", "Activity",], axis=1, ) gains_emi_EU = gains_emi_EU[~gains_emi_EU["value"].isna()] @@ -229,9 +225,7 @@ def fix_efficiencies(data: xr.DataArray, min_year: int) -> xr.DataArray: # we correct it to 1, as we do not accept # that efficiency degrades over time data.loc[dict(year=[y for y in data.year.values if y > 2020])] = np.clip( - data.loc[dict(year=[y for y in data.year.values if y > 2020])], - 1, - None, + data.loc[dict(year=[y for y in data.year.values if y > 2020])], 1, None, ) # Inversely, if we are looking at a year prior to 2020 @@ -239,9 +233,7 @@ def fix_efficiencies(data: xr.DataArray, min_year: int) -> xr.DataArray: # we correct it to 1, as we do not accept # that efficiency in the past was higher than now data.loc[dict(year=[y for y in data.year.values if y < 2020])] = np.clip( - data.loc[dict(year=[y for y in data.year.values if y < 2020])], - None, - 1, + data.loc[dict(year=[y for y in data.year.values if y < 2020])], None, 1, ) # ensure that efficiency can not decrease over time @@ -398,9 +390,7 @@ def __init__( new_vars = flatten(new_vars) data = self.__get_iam_data( - key=key, - filedir=filepath_iam_files, - variables=new_vars, + key=key, filedir=filepath_iam_files, variables=new_vars, ) self.regions = data.region.values.tolist() @@ -412,9 +402,7 @@ def __init__( ) self.electricity_markets = self.__fetch_market_data( - data=data, - input_vars=electricity_prod_vars, - system_model=self.system_model, + data=data, input_vars=electricity_prod_vars, system_model=self.system_model, ) self.petrol_markets = self.__fetch_market_data( @@ -439,12 +427,7 @@ def __init__( input_vars={ k: v for k, v in fuel_prod_vars.items() - if any( - x in k - for x in [ - "diesel", - ] - ) + if any(x in k for x in ["diesel",]) }, system_model=self.system_model, ) @@ -471,12 +454,7 @@ def __init__( input_vars={ k: v for k, v in fuel_prod_vars.items() - if any( - x in k - for x in [ - "hydrogen", - ] - ) + if any(x in k for x in ["hydrogen",]) }, system_model=self.system_model, ) @@ -533,12 +511,7 @@ def __init__( efficiency_labels={ k: v for k, v in fuel_eff_vars.items() - if any( - x in k - for x in [ - "diesel", - ] - ) + if any(x in k for x in ["diesel",]) }, ) self.gas_efficiencies = self.get_iam_efficiencies( @@ -554,12 +527,7 @@ def __init__( efficiency_labels={ k: v for k, v in fuel_eff_vars.items() - if any( - x in k - for x in [ - "hydrogen", - ] - ) + if any(x in k for x in ["hydrogen",]) }, ) @@ -1121,7 +1089,10 @@ def get_external_data(self, datapackages): .to_xarray() ) - array.coords["year"] = [int(y) for y in array.coords["year"]] + # convert to float64 + array = array.astype(np.float64) + # convert year dim to int64 + array.coords["year"] = array.coords["year"].astype(np.int64) data[i]["production volume"] = array regions = subset["region"].unique().tolist() @@ -1163,7 +1134,10 @@ def get_external_data(self, datapackages): .mean() .to_xarray() ) - array.coords["year"] = [int(y) for y in array.coords["year"]] + # convert to float64 + array = array.astype(np.float64) + # convert year dim to int64 + array.coords["year"] = array.coords["year"].astype(np.int64) ref_years = {} @@ -1256,11 +1230,7 @@ def fetch_external_data_coal_power_plants(self): df = df.drop(columns=["fuel input"]) array = ( df.melt( - id_vars=[ - "country", - "CHP", - "fuel", - ], + id_vars=["country", "CHP", "fuel",], var_name="variable", value_name="value", ) diff --git a/premise/ecoinvent_modification.py b/premise/ecoinvent_modification.py index 092322cc..28fca55d 100644 --- a/premise/ecoinvent_modification.py +++ b/premise/ecoinvent_modification.py @@ -136,6 +136,7 @@ FILEPATH_WAVE = INVENTORY_DIR / "lci-wave_energy.xlsx" FILEPATH_FUEL_CELL = INVENTORY_DIR / "lci-fuel_cell.xlsx" FILEPATH_CSP = INVENTORY_DIR / "lci-concentrating-solar-power.xlsx" +FILEPATH_HOME_STORAGE_BATTERIES = INVENTORY_DIR / "lci-home-batteries.xlsx" config = load_constants() @@ -340,6 +341,10 @@ def check_scenarios(scenario: dict, key: bytes) -> dict: scenario["filepath"] = check_filepath(filepath) else: scenario["filepath"] = DATA_DIR / "iam_output_files" + if key is None: + raise ValueError( + "You need to provide the encryption key to decrypt the IAM output files provided by `premise`." + ) scenario["model"] = check_model_name(scenario["model"]) scenario["pathway"] = check_pathway_name( @@ -709,6 +714,7 @@ def __import_inventories(self, keep_uncertainty_data: bool = False) -> List[dict (FILEPATH_COBALT, "3.8"), (FILEPATH_GRAPHITE, "3.8"), (FILEPATH_BATTERIES, "3.8"), + (FILEPATH_HOME_STORAGE_BATTERIES, "3.9"), (FILEPATH_PHOTOVOLTAICS, "3.7"), (FILEPATH_HYDROGEN_INVENTORIES, "3.9"), (FILEPATH_HYDROGEN_SOLAR_INVENTORIES, "3.9"), @@ -726,18 +732,12 @@ def __import_inventories(self, keep_uncertainty_data: bool = False) -> List[dict (FILEPATH_SYNGAS_FROM_COAL_INVENTORIES, "3.7"), (FILEPATH_BIOFUEL_INVENTORIES, "3.7"), (FILEPATH_SYNFUEL_INVENTORIES, "3.7"), - ( - FILEPATH_SYNFUEL_FROM_FT_FROM_WOOD_GASIFICATION_INVENTORIES, - "3.7", - ), + (FILEPATH_SYNFUEL_FROM_FT_FROM_WOOD_GASIFICATION_INVENTORIES, "3.7",), ( FILEPATH_SYNFUEL_FROM_FT_FROM_WOOD_GASIFICATION_WITH_CCS_INVENTORIES, "3.7", ), - ( - FILEPATH_SYNFUEL_FROM_FT_FROM_COAL_GASIFICATION_INVENTORIES, - "3.7", - ), + (FILEPATH_SYNFUEL_FROM_FT_FROM_COAL_GASIFICATION_INVENTORIES, "3.7",), ( FILEPATH_SYNFUEL_FROM_FT_FROM_COAL_GASIFICATION_WITH_CCS_INVENTORIES, "3.7", @@ -856,12 +856,7 @@ def update_dac(self) -> None: # use multiprocessing to speed up the process with ProcessPool(processes=multiprocessing.cpu_count()) as pool: args = [ - ( - scenario, - self.version, - self.system_model, - self.modified_datasets, - ) + (scenario, self.version, self.system_model, self.modified_datasets,) for scenario in self.scenarios ] results = pool.starmap(_update_dac, args) @@ -882,12 +877,7 @@ def update_fuels(self) -> None: # use multiprocessing to speed up the process with ProcessPool(processes=multiprocessing.cpu_count()) as pool: args = [ - ( - scenario, - self.version, - self.system_model, - self.modified_datasets, - ) + (scenario, self.version, self.system_model, self.modified_datasets,) for scenario in self.scenarios ] results = pool.starmap(_update_fuels, args) @@ -908,12 +898,7 @@ def update_cement(self) -> None: # use multiprocessing to speed up the process with ProcessPool(processes=multiprocessing.cpu_count()) as pool: args = [ - ( - scenario, - self.version, - self.system_model, - self.modified_datasets, - ) + (scenario, self.version, self.system_model, self.modified_datasets,) for scenario in self.scenarios ] results = pool.starmap(_update_cement, args) @@ -934,12 +919,7 @@ def update_steel(self) -> None: # use multiprocessing to speed up the process with ProcessPool(processes=multiprocessing.cpu_count()) as pool: args = [ - ( - scenario, - self.version, - self.system_model, - self.modified_datasets, - ) + (scenario, self.version, self.system_model, self.modified_datasets,) for scenario in self.scenarios ] results = pool.starmap(_update_steel, args) @@ -1220,9 +1200,7 @@ def write_superstructure_db_to_brightway( ) write_brightway2_database( - data=self.database, - name=name, - reset_codes=True, + data=self.database, name=name, reset_codes=True, ) # generate scenario report @@ -1290,8 +1268,7 @@ def write_db_to_brightway(self, name: [str, List[str]] = None): for scen, scenario in enumerate(self.scenarios): write_brightway2_database( - scenario["database"], - name[scen], + scenario["database"], name[scen], ) # generate scenario report self.generate_scenario_report() diff --git a/premise/export.py b/premise/export.py index 8c178107..7f7b0ba0 100644 --- a/premise/export.py +++ b/premise/export.py @@ -310,12 +310,7 @@ def create_codes_and_names_of_tech_matrix(database: List[dict]): :rtype: dict """ return { - ( - i["name"], - i["reference product"], - i["unit"], - i["location"], - ): i["code"] + (i["name"], i["reference product"], i["unit"], i["location"],): i["code"] for i in database } @@ -337,10 +332,7 @@ def biosphere_flows_dictionary(version): csv_dict = {} with open(fp, encoding="utf-8") as file: - input_dict = csv.reader( - file, - delimiter=get_delimiter(filepath=fp), - ) + input_dict = csv.reader(file, delimiter=get_delimiter(filepath=fp),) for row in input_dict: csv_dict[(row[0], row[1], row[2], row[3])] = row[-1] @@ -542,9 +534,7 @@ def build_datapackage(df, inventories, list_scenarios, ei_version, name): "version": ei_version, "type": "source", }, - { - "name": "biosphere3", - }, + {"name": "biosphere3",}, ] package.descriptor["scenarios"] = [ { @@ -937,7 +927,6 @@ def prepare_db_for_export( base.database = check_amount_format(base.database) # we relink "dead" exchanges - # print("- relinking exchanges...") base.relink_datasets( excludes_datasets=["cobalt industry", "market group for electricity"], alt_names=[ @@ -950,11 +939,53 @@ def prepare_db_for_export( ], ) - # print("Done!") + for ds in base.database: + if "parameters" in ds: + if not isinstance(ds["parameters"], list): + if isinstance(ds["parameters"], dict): + ds["parameters"] = [ + {"name": k, "amount": v} for k, v in ds["parameters"].items() + ] + else: + ds["parameters"] = [ds["parameters"]] + else: + ds["parameters"] = [ + {"name": k, "amount": v} + for o in ds["parameters"] + for k, v in o.items() + ] + + for key, value in list(ds.items()): + if not value: + del ds[key] + + ds["exchanges"] = [clean_up(exc) for exc in ds["exchanges"]] return base.database, base.cache +def clean_up(exc): + """Remove keys from ``exc`` that are not in the schema.""" + + FORBIDDEN_FIELDS_TECH = [ + "categories", + ] + + FORBIDDEN_FIELDS_BIO = ["location", "product"] + + for field in list(exc.keys()): + if exc[field] is None or exc[field] == "None": + del exc[field] + continue + + if exc["type"] == "biosphere" and field in FORBIDDEN_FIELDS_BIO: + del exc[field] + if exc["type"] == "technosphere" and field in FORBIDDEN_FIELDS_TECH: + del exc[field] + + return exc + + def _prepare_database( scenario, scenario_cache, version, system_model, modified_datasets ): @@ -991,10 +1022,7 @@ class Export: """ def __init__( - self, - scenario: dict = None, - filepath: Path = None, - version: str = None, + self, scenario: dict = None, filepath: Path = None, version: str = None, ): self.db = scenario["database"] self.model = scenario["model"] @@ -1103,11 +1131,7 @@ def export_db_to_matrices(self): # Export A matrix with open(self.filepath / "A_matrix.csv", "w", encoding="utf-8") as file: - writer = csv.writer( - file, - delimiter=";", - lineterminator="\n", - ) + writer = csv.writer(file, delimiter=";", lineterminator="\n",) writer.writerow(["index of activity", "index of product", "value"]) rows = self.create_A_matrix_coordinates() for row in rows: @@ -1115,11 +1139,7 @@ def export_db_to_matrices(self): # Export A index with open(self.filepath / "A_matrix_index.csv", "w", encoding="utf-8") as file: - writer = csv.writer( - file, - delimiter=";", - lineterminator="\n", - ) + writer = csv.writer(file, delimiter=";", lineterminator="\n",) index_A = create_index_of_A_matrix(self.db) for d in index_A: data = list(d) + [index_A[d]] @@ -1129,11 +1149,7 @@ def export_db_to_matrices(self): # Export B matrix with open(self.filepath / "B_matrix.csv", "w", encoding="utf-8") as file: - writer = csv.writer( - file, - delimiter=";", - lineterminator="\n", - ) + writer = csv.writer(file, delimiter=";", lineterminator="\n",) writer.writerow(["index of activity", "index of biosphere flow", "value"]) rows = self.create_B_matrix_coordinates() for row in rows: @@ -1141,11 +1157,7 @@ def export_db_to_matrices(self): # Export B index with open(self.filepath / "B_matrix_index.csv", "w", encoding="utf-8") as file: - writer = csv.writer( - file, - delimiter=";", - lineterminator="\n", - ) + writer = csv.writer(file, delimiter=";", lineterminator="\n",) for d in index_B: data = list(d) + [index_B[d]] writer.writerow(data) diff --git a/premise/inventory_imports.py b/premise/inventory_imports.py index 83a18253..66b1cff2 100644 --- a/premise/inventory_imports.py +++ b/premise/inventory_imports.py @@ -60,10 +60,7 @@ def get_biosphere_code(version) -> dict: csv_dict = {} with open(fp, encoding="utf-8") as file: - input_dict = csv.reader( - file, - delimiter=get_delimiter(filepath=fp), - ) + input_dict = csv.reader(file, delimiter=get_delimiter(filepath=fp),) for row in input_dict: csv_dict[(row[0], row[1], row[2], row[3])] = row[4] @@ -90,8 +87,7 @@ def generate_migration_maps(origin: str, destination: str) -> Dict[str, list]: with open(FILEPATH_MIGRATION_MAP, "r", encoding="utf-8") as read_obj: csv_reader = csv.reader( - read_obj, - delimiter=get_delimiter(filepath=FILEPATH_MIGRATION_MAP), + read_obj, delimiter=get_delimiter(filepath=FILEPATH_MIGRATION_MAP), ) next(csv_reader) for row in csv_reader: @@ -511,15 +507,7 @@ def correct_product_field(self, exc: tuple) -> [str, None]: return candidate["reference product"] self.list_unlinked.append( - ( - exc[0], - exc[-1], - exc[1], - None, - exc[2], - "technosphere", - self.path.name, - ) + (exc[0], exc[-1], exc[1], None, exc[2], "technosphere", self.path.name,) ) return None @@ -692,10 +680,8 @@ def prepare_inventory(self) -> None: ) if self.system_model == "consequential": - self.import_db.data = ( - check_for_datasets_compliance_with_consequential_database( - self.import_db.data, self.consequential_blacklist - ) + self.import_db.data = check_for_datasets_compliance_with_consequential_database( + self.import_db.data, self.consequential_blacklist ) self.import_db.data = remove_categories(self.import_db.data) @@ -827,16 +813,6 @@ def load_inventory(self, path): "Incorrect filetype for inventories." "Should be either .xlsx or .csv" ) - def remove_missing_fields(self): - """ - Remove any field that does not have information. - """ - - for dataset in self.import_db.data: - for key, value in list(dataset.items()): - if not value: - del dataset[key] - def prepare_inventory(self): if str(self.version_in) != self.version_out: # if version_out is 3.9, migrate towards 3.8 first, then 3.9 @@ -855,72 +831,15 @@ def prepare_inventory(self): ) if self.system_model == "consequential": - self.import_db.data = ( - check_for_datasets_compliance_with_consequential_database( - self.import_db.data, self.consequential_blacklist - ) + self.import_db.data = check_for_datasets_compliance_with_consequential_database( + self.import_db.data, self.consequential_blacklist ) - list_missing_prod = self.search_missing_exchanges( - label="type", value="production" - ) - - if len(list_missing_prod) > 0: - print("The following datasets are missing a `production` exchange.") - print("You should fix those before proceeding further.\n") - table = PrettyTable( - ["Name", "Reference product", "Location", "Unit", "File"] - ) - for dataset in list_missing_prod: - table.add_row( - [ - dataset.get("name", "XXXX"), - dataset.get("referece product", "XXXX"), - dataset.get("location", "XXXX"), - dataset.get("unit", "XXXX"), - self.path.name, - ] - ) - - print(table) - - sys.exit() - self.import_db.data = remove_categories(self.import_db.data) - self.add_biosphere_links(delete_missing=True) - list_missing_ref = self.search_missing_field(field="name") - list_missing_ref.extend(self.search_missing_field(field="reference product")) - list_missing_ref.extend(self.search_missing_field(field="location")) - list_missing_ref.extend(self.search_missing_field(field="unit")) - - if len(list_missing_ref) > 0: - print( - "The following datasets are missing an important field " - "(`name`, `reference product`, `location` or `unit`).\n" - ) - - print("You should fix those before proceeding further.\n") - table = PrettyTable( - ["Name", "Reference product", "Location", "Unit", "File"] - ) - for dataset in list_missing_ref: - table.add_row( - [ - dataset.get("name", "XXXX"), - dataset.get("referece product", "XXXX"), - dataset.get("location", "XXXX"), - dataset.get("unit", "XXXX"), - self.path.name, - ] - ) - - print(table) - - if len(list_missing_prod) > 0 or len(list_missing_ref) > 0: - sys.exit() - - self.remove_missing_fields() + self.lower_case_technosphere_exchanges() + self.add_biosphere_links() self.add_product_field_to_exchanges() + # Check for duplicates self.check_for_already_existing_datasets() self.import_db.data = check_for_duplicate_datasets(self.import_db.data) diff --git a/premise/marginal_mixes.py b/premise/marginal_mixes.py index d9213352..72a87281 100644 --- a/premise/marginal_mixes.py +++ b/premise/marginal_mixes.py @@ -161,9 +161,7 @@ def consequential_method(data: xr.DataArray, year: int, args: dict) -> xr.DataAr weighted_slope_start: float = args.get("weighted slope start", 0.75) weighted_slope_end: float = args.get("weighted slope end", 1.0) - market_shares = xr.zeros_like( - data.interp(year=[year]), - ) + market_shares = xr.zeros_like(data.interp(year=[year]),) # Since there can be different start and end values, # we interpolate the entire data of the IAM instead @@ -197,6 +195,10 @@ def consequential_method(data: xr.DataArray, year: int, args: dict) -> xr.DataAr region=region, year=year ).sum(dim="variables") + # if shares contains only NaNs, we give its elements the value 1 + if shares.isnull().all(): + shares = xr.ones_like(shares) + time_parameters = { (False, False, False, False): { "start": year, @@ -328,31 +330,18 @@ def consequential_method(data: xr.DataArray, year: int, args: dict) -> xr.DataAr # if the capital replacement rate is not used, if isinstance(start, np.ndarray): data_start = ( - data_full.sel( - region=region, - year=start, - ) + data_full.sel(region=region, year=start,) * np.identity(start.shape[0]) ).sum(dim="variables") else: - data_start = data_full.sel( - region=region, - year=start, - ) + data_start = data_full.sel(region=region, year=start,) if isinstance(end, np.ndarray): data_end = ( - data_full.sel( - region=region, - year=end, - ) - * np.identity(end.shape[0]) + data_full.sel(region=region, year=end,) * np.identity(end.shape[0]) ).sum(dim="variables") else: - data_end = data_full.sel( - region=region, - year=end, - ) + data_end = data_full.sel(region=region, year=end,) market_shares.loc[dict(region=region)] = ( (data_end.values - data_start.values) / (end - start) @@ -409,28 +398,18 @@ def consequential_method(data: xr.DataArray, year: int, args: dict) -> xr.DataAr if measurement == 2: if isinstance(end, np.ndarray): data_end = ( - data_full.sel( - region=region, - year=end, - ) - * np.identity(end.shape[0]) + data_full.sel(region=region, year=end,) * np.identity(end.shape[0]) ).sum(dim="variables") new_end = np.zeros_like(data_full.sel(region=region)) new_end[:, :] = end[:, None] end = new_end else: - data_end = data_full.sel( - region=region, - year=end, - ) + data_end = data_full.sel(region=region, year=end,) if isinstance(start, np.ndarray): data_start = ( - data_full.sel( - region=region, - year=start, - ) + data_full.sel(region=region, year=start,) * np.identity(start.shape[0]) ).sum(dim="variables") @@ -438,10 +417,7 @@ def consequential_method(data: xr.DataArray, year: int, args: dict) -> xr.DataAr new_start[:, :] = start[:, None] start = new_start else: - data_start = data_full.sel( - region=region, - year=start, - ) + data_start = data_full.sel(region=region, year=start,) mask_end = data_full.sel(region=region).year.values[None, :] <= end mask_start = data_full.sel(region=region).year.values[None, :] >= start @@ -493,33 +469,20 @@ def consequential_method(data: xr.DataArray, year: int, args: dict) -> xr.DataAr if measurement == 3: if isinstance(end, np.ndarray): data_end = ( - data_full.sel( - region=region, - year=end, - ) - * np.identity(end.shape[0]) + data_full.sel(region=region, year=end,) * np.identity(end.shape[0]) ).sum(dim="variables") else: - data_end = data_full.sel( - region=region, - year=end, - ) + data_end = data_full.sel(region=region, year=end,) if isinstance(start, np.ndarray): data_start = ( - data_full.sel( - region=region, - year=start, - ) + data_full.sel(region=region, year=start,) * np.identity(start.shape[0]) ).sum(dim="variables") else: - data_start = data_full.sel( - region=region, - year=start, - ) + data_start = data_full.sel(region=region, year=start,) slope = (data_end.values - data_start.values) / (end - start) @@ -528,32 +491,24 @@ def consequential_method(data: xr.DataArray, year: int, args: dict) -> xr.DataAr if isinstance(short_slope_start, np.ndarray): data_short_slope_start = ( - data_full.sel( - region=region, - year=short_slope_start, - ) + data_full.sel(region=region, year=short_slope_start,) * np.identity(short_slope_start.shape[0]) ).sum(dim="variables") else: data_short_slope_start = data_full.sel( - region=region, - year=short_slope_start, + region=region, year=short_slope_start, ) if isinstance(short_slope_end, np.ndarray): data_short_slope_end = ( - data_full.sel( - region=region, - year=short_slope_end, - ) + data_full.sel(region=region, year=short_slope_end,) * np.identity(short_slope_end.shape[0]) ).sum(dim="variables") else: data_short_slope_end = data_full.sel( - region=region, - year=short_slope_end, + region=region, year=short_slope_end, ) short_slope = ( @@ -651,31 +606,18 @@ def consequential_method(data: xr.DataArray, year: int, args: dict) -> xr.DataAr if isinstance(start, np.ndarray): data_start = ( - data_full.sel( - region=region, - year=start, - ) + data_full.sel(region=region, year=start,) * np.identity(start.shape[0]) ).sum(dim="variables") else: - data_start = data_full.sel( - region=region, - year=start, - ) + data_start = data_full.sel(region=region, year=start,) if isinstance(end, np.ndarray): data_end = ( - data_full.sel( - region=region, - year=end, - ) - * np.identity(end.shape[0]) + data_full.sel(region=region, year=end,) * np.identity(end.shape[0]) ).sum(dim="variables") else: - data_end = data_full.sel( - region=region, - year=end, - ) + data_end = data_full.sel(region=region, year=end,) market_shares.loc[dict(region=region)] = ( (data_end.values - data_start.values) / (end - start) diff --git a/premise/transformation.py b/premise/transformation.py index 0ca0e279..dca178e2 100644 --- a/premise/transformation.py +++ b/premise/transformation.py @@ -6,7 +6,6 @@ """ import logging.config -import sys import uuid from collections import defaultdict from copy import deepcopy @@ -76,10 +75,7 @@ def get_suppliers_of_a_region( if exclude: filters.append(ws.doesnt_contain_any("name", exclude)) - return ws.get_many( - database, - *filters, - ) + return ws.get_many(database, *filters,) def get_shares_from_production_volume( @@ -110,12 +106,7 @@ def get_shares_from_production_volume( production_volume = max(float(exc.get("production volume", 1e-9)), 1e-9) dict_act[ - ( - act["name"], - act["location"], - act["reference product"], - act["unit"], - ) + (act["name"], act["location"], act["reference product"], act["unit"],) ] = production_volume total_production_volume += production_volume @@ -214,10 +205,13 @@ def allocate_inputs(exc, lst): if lst[0]["name"] != exc["name"]: exc["name"] = lst[0]["name"] - return [ - new_exchange(exc, obj["location"], factor / total) - for obj, factor in zip(lst, pvs) - ], [p / total for p in pvs] + return ( + [ + new_exchange(exc, obj["location"], factor / total) + for obj, factor in zip(lst, pvs) + ], + [p / total for p in pvs], + ) def filter_out_results( @@ -647,12 +641,7 @@ def fetch_proxies( self.modified_datasets[(self.model, self.scenario, self.year)][ "emptied" ].append( - ( - ds["name"], - ds["reference product"], - ds["location"], - ds["unit"], - ) + (ds["name"], ds["reference product"], ds["location"], ds["unit"],) ) # empty original datasets @@ -708,7 +697,9 @@ def empty_original_datasets( self.database, ws.equals("name", name), ws.contains("reference product", ref_prod), - ws.doesnt_contain_any("location", regions), + ws.exclude( + ws.either(*[ws.equals("location", loc) for loc in self.regions]) + ), ) for existing_ds in existing_datasets: @@ -822,8 +813,7 @@ def relink_datasets( # loop through the database # ignore datasets which name contains `name` for act in ws.get_many( - self.database, - ws.doesnt_contain_any("name", excludes_datasets), + self.database, ws.doesnt_contain_any("name", excludes_datasets), ): # and find exchanges of datasets to relink excs_to_relink = [ @@ -888,15 +878,11 @@ def relink_datasets( names_to_look_for, alternative_locations ): if ( - name_to_look_for, - exc[1], - alt_loc, - exc[-1], - ) in self.modified_datasets[ - (self.model, self.scenario, self.year) - ][ - "created" - ]: + (name_to_look_for, exc[1], alt_loc, exc[-1],) + in self.modified_datasets[ + (self.model, self.scenario, self.year) + ]["created"] + ): entry = [(name_to_look_for, exc[1], alt_loc, exc[-1], 1.0)] self.add_new_entry_to_cache( @@ -997,10 +983,7 @@ def get_carbon_capture_rate(self, loc: str, sector: str) -> float: if sector in self.iam_data.carbon_capture_rate.variables.values: rate = ( - self.iam_data.carbon_capture_rate.sel( - variables=sector, - region=loc, - ) + self.iam_data.carbon_capture_rate.sel(variables=sector, region=loc,) .interp(year=self.year) .values ) @@ -1063,17 +1046,13 @@ def create_ccs_dataset( # this corresponds to the share of biogenic CO2 # in the fossil + biogenic CO2 emissions of the plant - for exc in ws.biosphere( - ccs, - ws.equals("name", "Carbon dioxide, in air"), - ): + for exc in ws.biosphere(ccs, ws.equals("name", "Carbon dioxide, in air"),): exc["amount"] = bio_co2_stored if bio_co2_leaked > 0: # then the biogenic CO2 leaked during the capture process for exc in ws.biosphere( - ccs, - ws.equals("name", "Carbon dioxide, non-fossil"), + ccs, ws.equals("name", "Carbon dioxide, non-fossil"), ): exc["amount"] = bio_co2_leaked @@ -1112,10 +1091,7 @@ def create_ccs_dataset( self.database.append(ccs) def find_iam_efficiency_change( - self, - data: xr.DataArray, - variable: Union[str, list], - location: str, + self, data: xr.DataArray, variable: Union[str, list], location: str, ) -> float: """ Return the relative change in efficiency for `variable` in `location` @@ -1147,11 +1123,7 @@ def write_log(self, dataset, status="created"): ) def add_new_entry_to_cache( - self, - location: str, - exchange: dict, - allocated: List[dict], - shares: List[float], + self, location: str, exchange: dict, allocated: List[dict], shares: List[float], ) -> None: """ Add an entry to the cache. @@ -1185,11 +1157,7 @@ def add_new_entry_to_cache( self.cache[location][self.model][exc_key] = entry def relink_technosphere_exchanges( - self, - dataset, - exclusive=True, - biggest_first=False, - contained=True, + self, dataset, exclusive=True, biggest_first=False, contained=True, ) -> dict: """Find new technosphere providers based on the location of the dataset. Designed to be used when the dataset's location changes, or when new datasets are added. @@ -1445,12 +1413,7 @@ def relink_technosphere_exchanges( new_exchanges.append(exc) # add to cache self.add_new_entry_to_cache( - dataset["location"], - exc, - [exc], - [ - 1.0, - ], + dataset["location"], exc, [exc], [1.0,], ) # make unique list of exchanges from new_exchanges diff --git a/premise/utils.py b/premise/utils.py index 8ef0c145..56bb9115 100644 --- a/premise/utils.py +++ b/premise/utils.py @@ -281,28 +281,6 @@ def write_database(self): super().write_database() -def clean_up(exc): - """Remove keys from ``exc`` that are not in the schema.""" - - FORBIDDEN_FIELDS_TECH = [ - "categories", - ] - - FORBIDDEN_FIELDS_BIO = ["location", "product"] - - for field in list(exc.keys()): - if exc[field] is None or exc[field] == "None": - del exc[field] - continue - - if exc["type"] == "biosphere" and field in FORBIDDEN_FIELDS_BIO: - del exc[field] - if exc["type"] == "technosphere" and field in FORBIDDEN_FIELDS_TECH: - del exc[field] - - return exc - - def reset_all_codes(data): """ Re-generate all codes in each dataset of a database @@ -322,36 +300,11 @@ def reset_all_codes(data): def write_brightway2_database(data, name, reset_codes=False): # Restore parameters to Brightway2 format # which allows for uncertainty and comments - - for ds in data: - if "parameters" in ds: - if not isinstance(ds["parameters"], list): - if isinstance(ds["parameters"], dict): - ds["parameters"] = [ - {"name": k, "amount": v} for k, v in ds["parameters"].items() - ] - else: - ds["parameters"] = [ds["parameters"]] - else: - ds["parameters"] = [ - {"name": k, "amount": v} - for o in ds["parameters"] - for k, v in o.items() - ] - - for key, value in list(ds.items()): - if not value: - del ds[key] - - ds["exchanges"] = [clean_up(exc) for exc in ds["exchanges"]] - change_db_name(data, name) - if reset_codes: reset_all_codes(data) link_internal(data) check_internal_linking(data) - check_duplicate_codes(data) PremiseImporter(name, data).write_database() diff --git a/requirements.txt b/requirements.txt index 5ada41c7..c2100521 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ numpy -pandas==1.5.3 +pandas bw2io==0.8.7 -bw2data==3.6.6 -wurst==0.3.3 +bw2data +wurst xarray prettytable pycountry diff --git a/setup.py b/setup.py index 8924fbcc..762a7da8 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def package_files(directory): setup( name="premise", - version="1.6.6", + version="1.7.0", python_requires=">=3.9,<3.11", packages=packages, author="Romain Sacchi , Alois Dirnaichner , Chris Mutel " @@ -42,9 +42,9 @@ def package_files(directory): include_package_data=True, install_requires=[ "numpy", - "wurst==0.3.3", + "wurst", "bw2io==0.8.7", - "pandas==1.5.3", + "pandas", "bw2data", "xarray", "prettytable",