Merge remote-tracking branch 'powerd-data/dev' into features/powerd-all

CHANGELOG.rst

@@ -765,6 +765,9 @@ Bug Fixes
 * Fix URLs of MaStR datasets
 * Fix CRS in ERA5 transformation
   `#1159 <https://github.com/openego/eGon-data/issues/1159>`_
+* Fix depricated python-operator import
+  `#183 <https://github.com/openego/powerd-data/issues/183>`_
+
 
 .. _PR #692: https://github.com/openego/eGon-data/pull/692
 .. _#343: https://github.com/openego/eGon-data/issues/343

setup.py

@@ -88,6 +88,7 @@ def read(*names, **kwargs):
         "cdsapi",
         "click<8.1",
         "entsoe-py >=0.6.2",
+        "fiona==1.9.6",
         "Flask-Session<0.6.0",
         "GeoAlchemy2",
         "geopandas>=0.10.0",
@@ -102,6 +103,7 @@ def read(*names, **kwargs):
         "omi",
         "openpyxl",
         "pandas>2.0.0",
+        "pulp<2.8.0",
         "psycopg2",
         "pyaml",
         "pypsa==0.20.1",
@@ -116,6 +118,7 @@ def read(*names, **kwargs):
         "shapely",
         "snakemake<7",
         "sqlalchemy",
+        "tabulate==0.8.0",
         "wtforms",
         "xarray",
         "xlrd",

src/egon/data/airflow/airflow.cfg

@@ -626,7 +626,7 @@ num_runs = -1
 processor_poll_interval = 1
 
 # after how much time (seconds) a new DAGs should be picked up from the filesystem
-min_file_process_interval = 0
+min_file_process_interval = 30
 
 # How often (in seconds) to scan the DAGs directory for new files. Default to 5 minutes.
 dag_dir_list_interval = 300

src/egon/data/airflow/dags/pipeline.py

@@ -41,19 +41,22 @@
 from egon.data.datasets.etrago_setup import EtragoSetup
 from egon.data.datasets.fill_etrago_gen import Egon_etrago_gen
 from egon.data.datasets.fix_ehv_subnetworks import FixEhvSubnetworks
-from egon.data.datasets.gas_areas import GasAreaseGon100RE, GasAreaseGon2035
+from egon.data.datasets.gas_areas import GasAreas
 from egon.data.datasets.gas_grid import GasNodesAndPipes
 from egon.data.datasets.gas_neighbours import GasNeighbours
 from egon.data.datasets.heat_demand import HeatDemandImport
 from egon.data.datasets.heat_demand_europe import HeatDemandEurope
 from egon.data.datasets.heat_demand_timeseries import HeatTimeSeries
 from egon.data.datasets.heat_etrago import HeatEtrago
 from egon.data.datasets.heat_etrago.hts_etrago import HtsEtragoTable
-from egon.data.datasets.heat_supply import HeatSupply
+from egon.data.datasets.heat_supply import (
+    GeothermalPotentialGermany,
+    HeatSupply,
+)
 from egon.data.datasets.heat_supply.individual_heating import (
     HeatPumps2035,
     HeatPumps2050,
-    HeatPumpsPypsaEurSec,
+    HeatPumpsPypsaEur,
 )
 from egon.data.datasets.hydrogen_etrago import (
     HydrogenBusEtrago,
@@ -78,7 +81,7 @@
 from egon.data.datasets.osmtgmod import Osmtgmod
 from egon.data.datasets.power_etrago import OpenCycleGasTurbineEtrago
 from egon.data.datasets.power_plants import PowerPlants
-from egon.data.datasets.pypsaeursec import PypsaEurSec
+from egon.data.datasets.pypsaeur import PreparePypsaEur, RunPypsaEur
 from egon.data.datasets.re_potential_areas import re_potential_area_setup
 from egon.data.datasets.renewable_feedin import RenewableFeedin
 from egon.data.datasets.saltcavern import SaltcavernData
@@ -96,6 +99,8 @@
 from egon.data.datasets.zensus import ZensusMiscellaneous, ZensusPopulation
 from egon.data.datasets.zensus_mv_grid_districts import ZensusMvGridDistricts
 from egon.data.datasets.zensus_vg250 import ZensusVg250
+from egon.data.datasets.scenario_path import CreateIntermediateScenarios
+
 # Set number of threads used by numpy and pandas
 set_numexpr_threads()
 
@@ -211,7 +216,7 @@
 
     # Download industrial gas demand
     industrial_gas_demand = IndustrialGasDemand(
-        dependencies=[scenario_parameters]
+        dependencies=[scenario_parameters, data_bundle]
     )
 
     # Extract landuse areas from the `osm` dataset
@@ -343,39 +348,56 @@
         ]
     )
 
-    # Minimum heat pump capacity for pypsa-eur-sec
-    heat_pumps_pypsa_eur_sec = HeatPumpsPypsaEurSec(
+    # Minimum heat pump capacity for pypsa-eur
+    heat_pumps_pypsa_eur = HeatPumpsPypsaEur(
         dependencies=[
             cts_demand_buildings,
             DistrictHeatingAreas,
             heat_time_series,
         ]
     )
 
-    # run pypsa-eur-sec
-    run_pypsaeursec = PypsaEurSec(
+    prepare_pypsa_eur = PreparePypsaEur(
+        dependencies=[
+            weather_data,
+            data_bundle,
+        ]
+    )
+
+
+    geothermal_potential_germany = GeothermalPotentialGermany(
+        dependencies=[
+            data_bundle,
+            district_heating_areas,
+        ]
+        )
+
+    # Deal with electrical neighbours
+    foreign_lines = ElectricalNeighbours(
+        dependencies=[prepare_pypsa_eur, tyndp_data, osmtgmod, fix_subnetworks]
+    )
+
+    # run pypsa-eur
+    run_pypsaeur = RunPypsaEur(
         dependencies=[
+            prepare_pypsa_eur,
             weather_data,
             hd_abroad,
             osmtgmod,
             setup_etrago,
             data_bundle,
             electrical_load_etrago,
             heat_time_series,
-            heat_pumps_pypsa_eur_sec,
+            geothermal_potential_germany,
+            foreign_lines,
         ]
     )
 
-    # Deal with electrical neighbours
-    foreign_lines = ElectricalNeighbours(
-        dependencies=[run_pypsaeursec, tyndp_data]
-    )
-
     # Import NEP (Netzentwicklungsplan) data
     scenario_capacities = ScenarioCapacities(
         dependencies=[
             data_bundle,
-            run_pypsaeursec,
+            run_pypsaeur,
             setup,
             vg250,
             zensus_population,
@@ -390,6 +412,7 @@
             osmtgmod,
             scenario_parameters,
             tasks["etrago_setup.create-tables"],
+            run_pypsaeur,
         ]
     )
 
@@ -403,17 +426,17 @@
     )
 
     # Create gas voronoi eGon2035
-    create_gas_polygons_egon2035 = GasAreaseGon2035(
+    create_gas_polygons = GasAreas(
         dependencies=[setup_etrago, insert_hydrogen_buses, vg250]
     )
 
     # Insert hydrogen grid
     insert_h2_grid = HydrogenGridEtrago(
         dependencies=[
-            create_gas_polygons_egon2035,
+            create_gas_polygons,
             gas_grid_insert_data,
             insert_hydrogen_buses,
-            run_pypsaeursec,
+            run_pypsaeur,
         ]
     )
 
@@ -432,47 +455,41 @@
         dependencies=[h2_infrastructure, insert_power_to_h2_installations]
     )
 
-    # Create gas voronoi eGon100RE
-    create_gas_polygons_egon100RE = GasAreaseGon100RE(
-        dependencies=[create_gas_polygons_egon2035, insert_h2_grid, vg250]
-    )
-
     # Gas abroad
     gas_abroad_insert_data = GasNeighbours(
         dependencies=[
             gas_grid_insert_data,
-            run_pypsaeursec,
+            prepare_pypsa_eur,
             foreign_lines,
             insert_hydrogen_buses,
-            create_gas_polygons_egon100RE,
+            run_pypsaeur,
         ]
     )
 
     # Import gas production
     gas_production_insert_data = CH4Production(
-        dependencies=[create_gas_polygons_egon2035]
+        dependencies=[create_gas_polygons]
     )
 
     # Import CH4 storages
     insert_data_ch4_storages = CH4Storages(
-        dependencies=[create_gas_polygons_egon2035]
+        dependencies=[create_gas_polygons]
     )
 
     # Assign industrial gas demand eGon2035
     IndustrialGasDemandeGon2035(
-        dependencies=[create_gas_polygons_egon2035, industrial_gas_demand]
+        dependencies=[create_gas_polygons, industrial_gas_demand]
     )
 
     # Assign industrial gas demand eGon100RE
     IndustrialGasDemandeGon100RE(
-        dependencies=[create_gas_polygons_egon100RE, industrial_gas_demand]
+        dependencies=[create_gas_polygons, industrial_gas_demand, run_pypsaeur,]
     )
 
     # CHP locations
     chp = Chp(
         dependencies=[
-            create_gas_polygons_egon100RE,
-            create_gas_polygons_egon2035,
+            create_gas_polygons,
             demand_curves_industry,
             district_heating_areas,
             industrial_sites,
@@ -504,7 +521,7 @@
     )
 
     create_ocgt = OpenCycleGasTurbineEtrago(
-        dependencies=[create_gas_polygons_egon2035, power_plants]
+        dependencies=[create_gas_polygons, power_plants]
     )
 
     # Fill eTraGo generators tables
@@ -519,6 +536,7 @@
             data_bundle,
             district_heating_areas,
             zensus_mv_grid_districts,
+            geothermal_potential_germany,
         ]
     )
 
@@ -569,7 +587,7 @@
 
     # eMobility: heavy duty transport
     heavy_duty_transport = HeavyDutyTransport(
-        dependencies=[vg250, setup_etrago, create_gas_polygons_egon2035]
+        dependencies=[vg250, setup_etrago, create_gas_polygons]
     )
 
     # Heat pump disaggregation for eGon2035
@@ -579,7 +597,7 @@
             DistrictHeatingAreas,
             heat_supply,
             heat_time_series,
-            heat_pumps_pypsa_eur_sec,
+            heat_pumps_pypsa_eur,
             power_plants,
         ]
     )
@@ -598,8 +616,8 @@
     # Heat pump disaggregation for eGon100RE
     heat_pumps_2050 = HeatPumps2050(
         dependencies=[
-            run_pypsaeursec,
-            heat_pumps_pypsa_eur_sec,
+            run_pypsaeur,
+            heat_pumps_pypsa_eur,
             heat_supply,
         ]
     )
@@ -623,7 +641,6 @@
             insert_H2_storage,
             insert_power_to_h2_installations,
             insert_h2_to_ch4_grid_links,
-            create_gas_polygons_egon100RE,
             gas_production_insert_data,
             insert_data_ch4_storages,
         ]
@@ -657,6 +674,18 @@
         ]
     )
 
+    # Create intermediate scenarios based on status2019 and eGon100RE
+    create_intemediate_scenarios = CreateIntermediateScenarios(
+        dependencies=[
+            storage_etrago,
+            hts_etrago_table,
+            fill_etrago_generators,
+            household_electricity_demand_annual,
+            cts_demand_buildings,
+            emobility_mit,
+        ]
+    )
+
     # ########## Keep this dataset at the end
     # Sanity Checks
     sanity_checks = SanityChecks(