From 5e9648d5c6c89ee93cd642b1617104f94f269689 Mon Sep 17 00:00:00 2001 From: romainsacchi Date: Thu, 5 Oct 2023 19:26:53 +0200 Subject: [PATCH] Fix mapping with electricity techs Remove pre-calculated efficiency for duplicated datasets --- dev/test_premise.ipynb | 531 +++++++++++++++++++++++++++++--------- premise/electricity.py | 8 +- premise/transformation.py | 2 +- 3 files changed, 422 insertions(+), 119 deletions(-) diff --git a/dev/test_premise.ipynb b/dev/test_premise.ipynb index 132a137d..8a0b4b78 100644 --- a/dev/test_premise.ipynb +++ b/dev/test_premise.ipynb @@ -74,7 +74,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "premise v.(1, 7, 1)\n", + "premise v.(1, 7, 2)\n", "+------------------------------------------------------------------+\n", "| Warning |\n", "+------------------------------------------------------------------+\n", @@ -121,7 +121,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18121/18121 [00:00<00:00, 102621.38it/s]\n" + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18121/18121 [00:00<00:00, 108432.32it/s]\n" ] }, { @@ -135,7 +135,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 615644/615644 [00:26<00:00, 23614.82it/s]\n" + "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 615644/615644 [00:29<00:00, 20700.10it/s]\n" ] }, { @@ -149,7 +149,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18121/18121 [00:02<00:00, 7372.83it/s]\n" + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18121/18121 [03:45<00:00, 80.49it/s]\n" ] }, { @@ -168,7 +168,7 @@ "Cannot find cached inventories. Will create them now for next time...\n", "Importing default inventories...\n", "\n", - "Extracted 1 worksheets in 0.26 seconds\n", + "Extracted 1 worksheets in 0.27 seconds\n", "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", @@ -194,7 +194,7 @@ "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", - "Extracted 7 worksheets in 0.04 seconds\n", + "Extracted 7 worksheets in 0.03 seconds\n", "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", @@ -214,7 +214,7 @@ "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", - "Extracted 1 worksheets in 0.91 seconds\n", + "Extracted 1 worksheets in 0.46 seconds\n", "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", @@ -300,7 +300,7 @@ "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", - "Extracted 1 worksheets in 0.01 seconds\n", + "Extracted 1 worksheets in 0.02 seconds\n", "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", @@ -312,7 +312,7 @@ "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", - "Extracted 5 worksheets in 0.17 seconds\n", + "Extracted 5 worksheets in 0.79 seconds\n", "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", @@ -375,7 +375,7 @@ "a faster workflow.\n", "\n", "//////////////// IMPORTING USER-DEFINED INVENTORIES ////////////////\n", - "Extracted 1 worksheets in 0.61 seconds\n", + "Extracted 1 worksheets in 0.45 seconds\n", "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", "The following datasets to import already exist in the source database. They will not be imported\n", @@ -542,7 +542,7 @@ "output_type": "stream", "text": [ "0% [##############################] 100% | ETA: 00:00:00\n", - "Total time elapsed: 00:00:24\n" + "Total time elapsed: 00:00:26\n" ] }, { @@ -550,11 +550,11 @@ "output_type": "stream", "text": [ "Title: Writing activities to SQLite3 database:\n", - " Started: 10/05/2023 14:32:14\n", - " Finished: 10/05/2023 14:32:39\n", - " Total time elapsed: 00:00:24\n", - " CPU %: 92.30\n", - " Memory %: 21.99\n", + " Started: 10/05/2023 19:05:20\n", + " Finished: 10/05/2023 19:05:47\n", + " Total time elapsed: 00:00:26\n", + " CPU %: 93.40\n", + " Memory %: 21.91\n", "Created database: test1\n", "Database test2 already exists: it will be overwritten.\n" ] @@ -578,7 +578,7 @@ "output_type": "stream", "text": [ "0% [##############################] 100% | ETA: 00:00:00\n", - "Total time elapsed: 00:00:35\n" + "Total time elapsed: 00:00:24\n" ] }, { @@ -586,17 +586,17 @@ "output_type": "stream", "text": [ "Title: Writing activities to SQLite3 database:\n", - " Started: 10/05/2023 14:34:43\n", - " Finished: 10/05/2023 14:35:18\n", - " Total time elapsed: 00:00:35\n", - " CPU %: 76.50\n", - " Memory %: 16.55\n", + " Started: 10/05/2023 19:07:21\n", + " Finished: 10/05/2023 19:07:46\n", + " Total time elapsed: 00:00:24\n", + " CPU %: 95.80\n", + " Memory %: 21.59\n", "Created database: test2\n", "Generate scenario report.\n", "Report saved under /Users/romain/GitHub/premise/dev/export/scenario_report.\n", "Generate change report.\n", "Report saved under /Users/romain/GitHub/premise/dev.\n", - "premise v.(1, 7, 1)\n", + "premise v.(1, 7, 2)\n", "+------------------------------------------------------------------+\n", "| Warning |\n", "+------------------------------------------------------------------+\n", @@ -635,14 +635,237 @@ "NewDatabase(..., quiet=True)\n", "\n", "//////////////////// EXTRACTING SOURCE DATABASE ////////////////////\n", + "Cannot find cached database. Will create one now for next time...\n", + "Getting activity data\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 19128/19128 [00:00<00:00, 211712.46it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Adding exchange data to activities\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 621717/621717 [00:41<00:00, 14862.19it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Filling out exchange data\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 19128/19128 [00:02<00:00, 7018.74it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set missing location of datasets to global scope.\n", + "Set missing location of production exchanges to scope of dataset.\n", + "Correct missing location of technosphere exchanges.\n", + "Correct missing flow categories for biosphere exchanges\n", + "Remove empty exchanges.\n", + "Remove uncertainty data.\n", "Done!\n", "\n", "////////////////// IMPORTING DEFAULT INVENTORIES ///////////////////\n", + "Cannot find cached inventories. Will create them now for next time...\n", + "Importing default inventories...\n", "\n", - "//////////////// IMPORTING USER-DEFINED INVENTORIES ////////////////\n", - "Extracted 1 worksheets in 0.45 seconds\n", + "Extracted 1 worksheets in 0.25 seconds\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.12 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.02 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.04 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.02 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.02 seconds\n", "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 7 worksheets in 0.04 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.03 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.03 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.46 seconds\n", + "Remove uncertainty data.\n", + "The following datasets to import already exist in the source database. They will not be imported\n", + "+----------------------------------------------------+--------------------------------+----------+-------------+\n", + "| Name | Reference product | Location | File |\n", + "+----------------------------------------------------+--------------------------------+----------+-------------+\n", + "| fluorspar production, 97% purity | fluorspar, 97% purity | GLO | lci-PV.xlsx |\n", + "| metallization paste production, back side | metallization paste, back side | RER | lci-PV.xlsx |\n", + "| metallization paste production, back side, alumini | metallization paste, back side | RER | lci-PV.xlsx |\n", + "| metallization paste production, front side | metallization paste, front sid | RER | lci-PV.xlsx |\n", + "| photovoltaic module production, building-integrate | photovoltaic module, building- | RER | lci-PV.xlsx |\n", + "| photovoltaic module production, building-integrate | photovoltaic module, building- | RER | lci-PV.xlsx |\n", + "| photovoltaic mounting system production, for facad | photovoltaic mounting system, | RER | lci-PV.xlsx |\n", + "| photovoltaic mounting system production, for flat- | photovoltaic mounting system, | RER | lci-PV.xlsx |\n", + "| photovoltaic mounting system production, for slant | photovoltaic mounting system, | RER | lci-PV.xlsx |\n", + "| photovoltaic panel factory construction | photovoltaic panel factory | GLO | lci-PV.xlsx |\n", + "| polyvinylfluoride production | polyvinylfluoride | US | lci-PV.xlsx |\n", + "| polyvinylfluoride production, dispersion | polyvinylfluoride, dispersion | US | lci-PV.xlsx |\n", + "| polyvinylfluoride, film production | polyvinylfluoride, film | US | lci-PV.xlsx |\n", + "| silicon production, metallurgical grade | silicon, metallurgical grade | NO | lci-PV.xlsx |\n", + "| vinyl fluoride production | vinyl fluoride | US | lci-PV.xlsx |\n", + "| wafer factory construction | wafer factory | DE | lci-PV.xlsx |\n", + "+----------------------------------------------------+--------------------------------+----------+-------------+\n", + "Extracted 1 worksheets in 0.05 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.02 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.02 seconds\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.02 seconds\n", + "Remove uncertainty data.\n", + "The following datasets to import already exist in the source database. They will not be imported\n", + "+----------------------------------------------------+--------------------------------+----------+---------------------------------------------------+\n", + "| Name | Reference product | Location | File |\n", + "+----------------------------------------------------+--------------------------------+----------+---------------------------------------------------+\n", + "| carbon dioxide, captured at cement production plan | carbon dioxide, captured and r | RER | lci-synfuels-from-methanol-from-cement-plant.xlsx |\n", + "+----------------------------------------------------+--------------------------------+----------+---------------------------------------------------+\n", + "Extracted 1 worksheets in 0.01 seconds\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.02 seconds\n", + "Remove uncertainty data.\n", + "The following datasets to import already exist in the source database. They will not be imported\n", + "+----------------------------------------------------+----------------------+----------+-------------------------------------------+\n", + "| Name | Reference product | Location | File |\n", + "+----------------------------------------------------+----------------------+----------+-------------------------------------------+\n", + "| methanol distillation, hydrogen from coal gasifica | methanol, purified | RER | lci-synfuels-from-methanol-from-coal.xlsx |\n", + "| methanol synthesis, hydrogen from coal gasificatio | methanol, unpurified | RER | lci-synfuels-from-methanol-from-coal.xlsx |\n", + "+----------------------------------------------------+----------------------+----------+-------------------------------------------+\n", + "Extracted 1 worksheets in 0.02 seconds\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.01 seconds\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.01 seconds\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.01 seconds\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.01 seconds\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.00 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.02 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.01 seconds\n", + "Remove uncertainty data.\n", + "Extracted 5 worksheets in 0.17 seconds\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.02 seconds\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.01 seconds\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.01 seconds\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.02 seconds\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.01 seconds\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.03 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.09 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.02 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 2 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Extracted 1 worksheets in 0.03 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Remove uncertainty data.\n", + "Data cached. It is advised to restart your workflow at this point.\n", + "This allows premise to use the cached data instead, which results in\n", + "a faster workflow.\n", + "\n", + "//////////////// IMPORTING USER-DEFINED INVENTORIES ////////////////\n", + "Extracted 1 worksheets in 0.46 seconds\n", "The following datasets to import already exist in the source database. They will not be imported\n", "+----------------------------------------------------+--------------------------------+----------+-------------+\n", "| Name | Reference product | Location | File |\n", @@ -770,87 +993,56 @@ "| electricity production, photovoltaic, commercial | electricity, low voltage | GB | lci-PV.xlsx |\n", "| electricity production, photovoltaic, commercial | electricity, low voltage | US | lci-PV.xlsx |\n", "+----------------------------------------------------+--------------------------------+----------+-------------+\n", - "Applying strategy: migrate_datasets\n", - "Applying strategy: migrate_exchanges\n", "Done!\n", "\n", "/////////////////////// EXTRACTING IAM DATA ////////////////////////\n", "Done!\n", "#############################################################################################################\n", - "3.6 brightway datapackage\n", + "3.7 brightway brightway\n", "\n", "/////////////////////////// ELECTRICITY ////////////////////////////\n", "Warning: No activities found for Gas ST -- revise mapping.\n", "Warning: No activities found for Gas ST -- revise mapping.\n", "Done!\n", "\n", - "Building scenario factor file...\n", - "Number of new activities: 1568\n", - "Data package saved at /Users/romain/GitHub/premise/dev/export/datapackage/test.zip\n", - "Generate scenario report.\n", - "Report saved under /Users/romain/GitHub/premise/dev/export/scenario_report.\n", - "Generate change report.\n", - "Report saved under /Users/romain/GitHub/premise/dev.\n", - "The data package contains the following scenarios:\n", - "+-----+---------------------------+------------------------------------------------------------------------+\n", - "| No. | Scenario | Description |\n", - "+-----+---------------------------+------------------------------------------------------------------------+\n", - "| 0 | remind - SSP2-Base - 2030 | Prospective db, based on REMIND, pathway SSP2-BASE, for the year 2030. |\n", - "| 1 | remind - SSP5-Base - 2080 | Prospective db, based on REMIND, pathway SSP5-BASE, for the year 2080. |\n", - "+-----+---------------------------+------------------------------------------------------------------------+\n", - "\n", - "To unfold all scenarios, `unfold()`.\n", - "To unfold a specific scenario, `unfold(scenarios=[1,])`.\n", - "Cannot find cached database. Will create one now for next time...\n", - "Getting activity data\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18121/18121 [00:00<00:00, 257572.41it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Adding exchange data to activities\n" + "Write new database(s) to Brightway2.\n", + "Database test1 already exists: it will be overwritten.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 615644/615644 [00:36<00:00, 16646.62it/s]\n" + "Writing activities to SQLite3 database:\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Filling out exchange data\n" + "Vacuuming database \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18121/18121 [00:02<00:00, 6913.67it/s]\n" + "0% [##############################] 100% | ETA: 00:00:00\n", + "Total time elapsed: 00:00:27\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Extracting additional inventories...\n", - "Writing scenario difference file...\n", - "Generating superstructure database...\n", - "Scenario difference file exported to /Users/romain/GitHub/premise/dev/SDF ecoinvent 3.6 test.csv.csv!\n", - "\n", - "Writing superstructure database...\n", - "Database test already exists: it will be overwritten.\n" + "Title: Writing activities to SQLite3 database:\n", + " Started: 10/05/2023 19:12:28\n", + " Finished: 10/05/2023 19:12:55\n", + " Total time elapsed: 00:00:27\n", + " CPU %: 93.60\n", + " Memory %: 24.01\n", + "Created database: test1\n", + "Database test2 already exists: it will be overwritten.\n" ] }, { @@ -872,7 +1064,7 @@ "output_type": "stream", "text": [ "0% [##############################] 100% | ETA: 00:00:00\n", - "Total time elapsed: 00:00:28\n" + "Total time elapsed: 00:00:29\n" ] }, { @@ -880,13 +1072,17 @@ "output_type": "stream", "text": [ "Title: Writing activities to SQLite3 database:\n", - " Started: 10/05/2023 14:42:15\n", - " Finished: 10/05/2023 14:42:44\n", - " Total time elapsed: 00:00:28\n", - " CPU %: 89.80\n", - " Memory %: 16.05\n", - "Created database: test\n", - "premise v.(1, 7, 1)\n", + " Started: 10/05/2023 19:14:34\n", + " Finished: 10/05/2023 19:15:04\n", + " Total time elapsed: 00:00:29\n", + " CPU %: 91.20\n", + " Memory %: 21.02\n", + "Created database: test2\n", + "Generate scenario report.\n", + "Report saved under /Users/romain/GitHub/premise/dev/export/scenario_report.\n", + "Generate change report.\n", + "Report saved under /Users/romain/GitHub/premise/dev.\n", + "premise v.(1, 7, 2)\n", "+------------------------------------------------------------------+\n", "| Warning |\n", "+------------------------------------------------------------------+\n", @@ -933,7 +1129,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 19128/19128 [00:00<00:00, 175418.76it/s]\n" + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 19565/19565 [00:00<00:00, 240108.25it/s]\n" ] }, { @@ -947,7 +1143,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 621717/621717 [00:51<00:00, 11967.13it/s]\n" + "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 629959/629959 [00:58<00:00, 10694.77it/s]\n" ] }, { @@ -961,7 +1157,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 19128/19128 [00:01<00:00, 9602.34it/s]\n" + "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 19565/19565 [00:01<00:00, 9810.12it/s]\n" ] }, { @@ -980,7 +1176,9 @@ "Cannot find cached inventories. Will create them now for next time...\n", "Importing default inventories...\n", "\n", - "Extracted 1 worksheets in 0.26 seconds\n", + "Extracted 1 worksheets in 0.28 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.11 seconds\n", "Applying strategy: migrate_datasets\n", @@ -998,7 +1196,7 @@ "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", - "Extracted 1 worksheets in 0.04 seconds\n", + "Extracted 1 worksheets in 0.01 seconds\n", "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", @@ -1006,27 +1204,21 @@ "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", - "Extracted 7 worksheets in 0.04 seconds\n", - "Applying strategy: migrate_datasets\n", - "Applying strategy: migrate_exchanges\n", + "Extracted 7 worksheets in 0.03 seconds\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.03 seconds\n", - "Applying strategy: migrate_datasets\n", - "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.01 seconds\n", - "Applying strategy: migrate_datasets\n", - "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.03 seconds\n", - "Applying strategy: migrate_datasets\n", - "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.01 seconds\n", "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", - "Extracted 1 worksheets in 6.25 seconds\n", + "Extracted 1 worksheets in 0.49 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "The following datasets to import already exist in the source database. They will not be imported\n", "+----------------------------------------------------+--------------------------------+----------+-------------+\n", @@ -1049,7 +1241,7 @@ "| vinyl fluoride production | vinyl fluoride | US | lci-PV.xlsx |\n", "| wafer factory construction | wafer factory | DE | lci-PV.xlsx |\n", "+----------------------------------------------------+--------------------------------+----------+-------------+\n", - "Extracted 1 worksheets in 0.05 seconds\n", + "Extracted 1 worksheets in 0.06 seconds\n", "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", @@ -1062,8 +1254,12 @@ "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.02 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.02 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "The following datasets to import already exist in the source database. They will not be imported\n", "+----------------------------------------------------+--------------------------------+----------+---------------------------------------------------+\n", @@ -1072,8 +1268,12 @@ "| carbon dioxide, captured at cement production plan | carbon dioxide, captured and r | RER | lci-synfuels-from-methanol-from-cement-plant.xlsx |\n", "+----------------------------------------------------+--------------------------------+----------+---------------------------------------------------+\n", "Extracted 1 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.02 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "The following datasets to import already exist in the source database. They will not be imported\n", "+----------------------------------------------------+----------------------+----------+-------------------------------------------+\n", @@ -1083,14 +1283,24 @@ "| methanol synthesis, hydrogen from coal gasificatio | methanol, unpurified | RER | lci-synfuels-from-methanol-from-coal.xlsx |\n", "+----------------------------------------------------+----------------------+----------+-------------------------------------------+\n", "Extracted 1 worksheets in 0.02 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.00 seconds\n", "Applying strategy: migrate_datasets\n", @@ -1101,38 +1311,44 @@ "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 5 worksheets in 0.17 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.02 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.01 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.03 seconds\n", "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.01 seconds\n", - "Applying strategy: migrate_datasets\n", - "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.09 seconds\n", - "Applying strategy: migrate_datasets\n", - "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.01 seconds\n", - "Applying strategy: migrate_datasets\n", - "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.01 seconds\n", - "Applying strategy: migrate_datasets\n", - "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.02 seconds\n", "Applying strategy: migrate_datasets\n", @@ -1142,9 +1358,7 @@ "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", "Remove uncertainty data.\n", - "Extracted 2 worksheets in 0.01 seconds\n", - "Applying strategy: migrate_datasets\n", - "Applying strategy: migrate_exchanges\n", + "Extracted 2 worksheets in 0.02 seconds\n", "Remove uncertainty data.\n", "Extracted 1 worksheets in 0.02 seconds\n", "Applying strategy: migrate_datasets\n", @@ -1155,7 +1369,9 @@ "a faster workflow.\n", "\n", "//////////////// IMPORTING USER-DEFINED INVENTORIES ////////////////\n", - "Extracted 1 worksheets in 0.45 seconds\n", + "Extracted 1 worksheets in 0.47 seconds\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "The following datasets to import already exist in the source database. They will not be imported\n", "+----------------------------------------------------+--------------------------------+----------+-------------+\n", "| Name | Reference product | Location | File |\n", @@ -1283,16 +1499,17 @@ "| electricity production, photovoltaic, commercial | electricity, low voltage | GB | lci-PV.xlsx |\n", "| electricity production, photovoltaic, commercial | electricity, low voltage | US | lci-PV.xlsx |\n", "+----------------------------------------------------+--------------------------------+----------+-------------+\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", "Done!\n", "\n", "/////////////////////// EXTRACTING IAM DATA ////////////////////////\n", "Done!\n", "#############################################################################################################\n", - "3.7 brightway brightway\n", + "3.8 brightway brightway\n", "\n", "/////////////////////////// ELECTRICITY ////////////////////////////\n", "Warning: No activities found for Gas ST -- revise mapping.\n", - "Warning: No activities found for Gas ST -- revise mapping.\n", "Done!\n", "\n", "Write new database(s) to Brightway2.\n", @@ -1318,7 +1535,7 @@ "output_type": "stream", "text": [ "0% [##############################] 100% | ETA: 00:00:00\n", - "Total time elapsed: 00:00:38\n" + "Total time elapsed: 00:00:28\n" ] }, { @@ -1326,11 +1543,11 @@ "output_type": "stream", "text": [ "Title: Writing activities to SQLite3 database:\n", - " Started: 10/05/2023 14:47:55\n", - " Finished: 10/05/2023 14:48:33\n", - " Total time elapsed: 00:00:38\n", - " CPU %: 84.50\n", - " Memory %: 17.54\n", + " Started: 10/05/2023 19:19:46\n", + " Finished: 10/05/2023 19:20:14\n", + " Total time elapsed: 00:00:28\n", + " CPU %: 93.50\n", + " Memory %: 25.16\n", "Created database: test1\n", "Database test2 already exists: it will be overwritten.\n" ] @@ -1353,7 +1570,87 @@ "name": "stderr", "output_type": "stream", "text": [ - "0% [######################### ] 100% | ETA: 00:00:05" + "0% [##############################] 100% | ETA: 00:00:00\n", + "Total time elapsed: 00:00:33\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Title: Writing activities to SQLite3 database:\n", + " Started: 10/05/2023 19:22:02\n", + " Finished: 10/05/2023 19:22:35\n", + " Total time elapsed: 00:00:33\n", + " CPU %: 88.90\n", + " Memory %: 18.61\n", + "Created database: test2\n", + "Generate scenario report.\n", + "Report saved under /Users/romain/GitHub/premise/dev/export/scenario_report.\n", + "Generate change report.\n", + "Report saved under /Users/romain/GitHub/premise/dev.\n", + "premise v.(1, 7, 2)\n", + "+------------------------------------------------------------------+\n", + "| Warning |\n", + "+------------------------------------------------------------------+\n", + "| Because some of the scenarios can yield LCI databases |\n", + "| containing net negative emission technologies (NET), |\n", + "| it is advised to account for biogenic CO2 flows when calculating |\n", + "| Global Warming potential indicators. |\n", + "| `premise_gwp` provides characterization factors for such flows. |\n", + "| It also provides factors for hydrogen emissions to air. |\n", + "| |\n", + "| Within your bw2 project: |\n", + "| from premise_gwp import add_premise_gwp |\n", + "| add_premise_gwp() |\n", + "+------------------------------------------------------------------+\n", + "+--------------------------------+----------------------------------+\n", + "| Utils functions | Description |\n", + "+--------------------------------+----------------------------------+\n", + "| clear_cache() | Clears the cache folder. Useful |\n", + "| | when updating `premise`or |\n", + "| | encountering issues with |\n", + "| | inventories. |\n", + "+--------------------------------+----------------------------------+\n", + "| get_regions_definition(model) | Retrieves the list of countries |\n", + "| | for each region of the model. |\n", + "+--------------------------------+----------------------------------+\n", + "| ndb.NewDatabase(...) | Generates a summary of the most |\n", + "| ndb.generate_scenario_report() | important scenarios' variables. |\n", + "+--------------------------------+----------------------------------+\n", + "Keep uncertainty data?\n", + "NewDatabase(..., keep_uncertainty_data=True)\n", + "\n", + "Disable multiprocessing?\n", + "NewDatabase(..., use_multiprocessing=False)\n", + "\n", + "Hide these messages?\n", + "NewDatabase(..., quiet=True)\n", + "\n", + "//////////////////// EXTRACTING SOURCE DATABASE ////////////////////\n", + "Cannot find cached database. Will create one now for next time...\n", + "Getting activity data\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 21238/21238 [00:00<00:00, 203445.07it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Adding exchange data to activities\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 615711/674593 [00:39<00:01, 48532.27it/s]" ] } ], @@ -1392,7 +1689,7 @@ " #\"superstructure\",\n", " #\"simapro\",\n", " #\"matrices\",\n", - " \"datapackage\"\n", + " #\"datapackage\"\n", " ]:\n", "\n", " ndb = NewDatabase(\n", diff --git a/premise/electricity.py b/premise/electricity.py index 9a7526a9..34d4d88a 100644 --- a/premise/electricity.py +++ b/premise/electricity.py @@ -2016,7 +2016,7 @@ def adjust_coal_power_plant_emissions(self) -> None: } ) - # self.write_log(dataset=dataset, status="updated") + self.write_log(dataset=dataset, status="updated") def create_missing_power_plant_datasets(self) -> None: """ @@ -2031,6 +2031,11 @@ def create_missing_power_plant_datasets(self) -> None: empty_original_activity=False, ) + # if `parameters` in datasets, delete it + for ds in new_datasets.values(): + if "parameters" in ds: + del ds["parameters"] + for loc, ds in new_datasets.items(): ds["name"] = vars["proxy"]["new name"] ds["code"] = str(uuid.uuid4().hex) @@ -2048,6 +2053,7 @@ def create_missing_power_plant_datasets(self) -> None: mapping = InventorySet(self.database, model=self.model) self.powerplant_map = mapping.generate_powerplant_map() + # reverse dictionary of self.powerplant_map self.powerplant_map_rev = {} for k, v in self.powerplant_map.items(): diff --git a/premise/transformation.py b/premise/transformation.py index 7ba293ff..8f0b8771 100644 --- a/premise/transformation.py +++ b/premise/transformation.py @@ -66,7 +66,7 @@ def get_suppliers_of_a_region( """ filters = [ - ws.either(*[ws.contains("name", supplier) for supplier in names]), + ws.either(*[ws.equals("name", supplier) for supplier in names]), ws.either(*[ws.equals("location", loc) for loc in locations]), ws.contains("reference product", reference_prod), ws.equals("unit", unit),