From 049b18566a3ed7e7f7dbc63cfb617cb5a4e97554 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 7 Sep 2022 10:47:56 +0200
Subject: [PATCH 01/16] Update the pre-commit configuration

Done simply via `pre-commit autoupdate`. Should fix all pre-commit
issues.
---
 .pre-commit-config.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index f16383106..7dda57867 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -5,21 +5,21 @@
 exclude: '^(\.tox|ci/templates|\.bumpversion\.cfg)(/|$)'
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v4.3.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: debug-statements
   - repo: https://github.com/timothycrosley/isort
-    rev: 5.8.0
+    rev: 5.10.1
     hooks:
       - id: isort
   - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.0
+    rev: 3.9.2
     hooks:
       - id: flake8
   - repo: https://github.com/psf/black
-    rev: 20.8b1
+    rev: 22.8.0
     hooks:
       - id: black
         language_version: python3 # Should be a command that runs python3.6+

From 3ad23ba75c6269f18f7d8e7299ee45f201ee27e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 7 Sep 2022 10:51:24 +0200
Subject: [PATCH 02/16] Tell flake8 that this bare `except` is OK

Otherwise it would complain with "E722 do not use bare 'except'".
---
 src/egon/data/db.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/egon/data/db.py b/src/egon/data/db.py
index 962e4ff61..b97ae620d 100755
--- a/src/egon/data/db.py
+++ b/src/egon/data/db.py
@@ -128,7 +128,7 @@ def session_scope():
     try:
         yield session
         session.commit()
-    except:
+    except:  # noqa: It's OK because the exception gets re-raised immediately.
         session.rollback()
         raise
     finally:

From da7703b350fc17bcce2eca4e38a0f6739081ed51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 7 Sep 2022 10:53:38 +0200
Subject: [PATCH 03/16] Use (parts of) a UUID4 as the `next_etrago_id`

I did a few experiments and using six bytes of the UUID would've been OK
but risky, but using seven bytes didn't lead to collisions for up to
around 1.6e+10 UUIDs in around 3e+5 contiguous ranges with random
lengths from 1e+4 to 1e+5. Using eight bytes would mean the risk of
generating integers which are to big for PostgreSQL's BigInt datatype,
so seven bytes is the best compromise between simplicity and collision
mitigation.
---
 src/egon/data/db.py | 19 ++-----------------
 1 file changed, 2 insertions(+), 17 deletions(-)

diff --git a/src/egon/data/db.py b/src/egon/data/db.py
index b97ae620d..d2395b7f4 100755
--- a/src/egon/data/db.py
+++ b/src/egon/data/db.py
@@ -2,6 +2,7 @@
 import codecs
 import functools
 import time
+import uuid
 
 from psycopg2.errors import DeadlockDetected, UniqueViolation
 from sqlalchemy import create_engine, text
@@ -240,23 +241,7 @@ def next_etrago_id(component):
     :func:`check_db_unique_violation` instead.
     """
 
-    if component == "transformer":
-        id_column = "trafo_id"
-    else:
-        id_column = f"{component}_id"
-
-    max_id = select_dataframe(
-        f"""
-        SELECT MAX({id_column}) FROM grid.egon_etrago_{component}
-        """
-    )["max"][0]
-
-    if max_id:
-        next_id = max_id + 1
-    else:
-        next_id = 1
-
-    return next_id
+    return int(uuid.uuid4().bytes[0:7].hex(), base=16)
 
 
 def check_db_unique_violation(func):

From 0cb29fc067a97fe630b49950ad44cda7f04e8693 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 7 Sep 2022 15:19:15 +0200
Subject: [PATCH 04/16] Avoid a bare except

This is due to a "flake8" complaint, but it also makes the code more
explicit. The intention is to provide an alternative, should the
`dataset_2040[node]` lookup fail. Explicitly catching only the
`KeyError` caused by this avoids potentially hiding other errors.
There's still a bug in this code, as explained in issue #926, but that's
something which has to be fixed separately.
---
 src/egon/data/datasets/electrical_neighbours.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/egon/data/datasets/electrical_neighbours.py b/src/egon/data/datasets/electrical_neighbours.py
index 0d104769a..a3a2490e3 100755
--- a/src/egon/data/datasets/electrical_neighbours.py
+++ b/src/egon/data/datasets/electrical_neighbours.py
@@ -1249,7 +1249,7 @@ def tyndp_demand():
 
             for node in nodes:
                 data_2040 = dataset_2040[node][2011] + data_2040
-        except:
+        except KeyError:
             data_2040 = data_2030
 
         # According to the NEP, data for 2030 and 2040 is linear interpolated

From 7876b1fbef42d56aca5ac6a397b8852a702a86fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 7 Sep 2022 16:41:37 +0200
Subject: [PATCH 05/16] Fix style and clean up the code

Most of these are done automatically via "isort" or "black". The
exceptions are:

  - fixing the line length of SQL strings and comments,
  - removing trailing whitespace in SQL strings and
  - removing the unused local variable `sources`.

These things had to be done manually to ensure they really don't change
the semantics of the code.
---
 src/egon/data/datasets/chp_etrago.py          |  9 ++++---
 .../data/datasets/electrical_neighbours.py    | 25 +++++++++++--------
 .../data/datasets/heat_etrago/__init__.py     | 24 +++++++++---------
 3 files changed, 33 insertions(+), 25 deletions(-)

diff --git a/src/egon/data/datasets/chp_etrago.py b/src/egon/data/datasets/chp_etrago.py
index fb30dcb6e..594890a66 100644
--- a/src/egon/data/datasets/chp_etrago.py
+++ b/src/egon/data/datasets/chp_etrago.py
@@ -37,6 +37,8 @@ def insert():
 
     targets = config.datasets()["chp_etrago"]["targets"]
 
+    source_schema = f"{sources['etrago_buses']['schema']}"
+    source_table = f"{sources['etrago_buses']['table']}"
     db.execute_sql(
         f"""
         DELETE FROM {targets['link']['schema']}.{targets['link']['table']}
@@ -44,19 +46,20 @@ def insert():
         AND scn_name = 'eGon2035'
         AND bus0 IN
         (SELECT bus_id
-         FROM {sources['etrago_buses']['schema']}.{sources['etrago_buses']['table']}
+         FROM {source_schema}.{source_table}
          WHERE scn_name = 'eGon2035'
          AND country = 'DE')
         AND bus1 IN
         (SELECT bus_id
-         FROM {sources['etrago_buses']['schema']}.{sources['etrago_buses']['table']}
+         FROM {source_schema}.{source_table}
          WHERE scn_name = 'eGon2035'
          AND country = 'DE')
         """
     )
     db.execute_sql(
         f"""
-        DELETE FROM {targets['generator']['schema']}.{targets['generator']['table']}
+        DELETE FROM
+            {targets['generator']['schema']}.{targets['generator']['table']}
         WHERE carrier LIKE '%%CHP%%'
         AND scn_name = 'eGon2035'
         """
diff --git a/src/egon/data/datasets/electrical_neighbours.py b/src/egon/data/datasets/electrical_neighbours.py
index a3a2490e3..7c6aeeb85 100755
--- a/src/egon/data/datasets/electrical_neighbours.py
+++ b/src/egon/data/datasets/electrical_neighbours.py
@@ -3,16 +3,16 @@
 
 import zipfile
 
-import geopandas as gpd
-import pandas as pd
 from shapely.geometry import LineString
 from sqlalchemy.orm import sessionmaker
+import geopandas as gpd
+import pandas as pd
 
-import egon.data.datasets.etrago_setup as etrago
-import egon.data.datasets.scenario_parameters.parameters as scenario_parameters
 from egon.data import config, db
 from egon.data.datasets import Dataset
 from egon.data.datasets.scenario_parameters import get_sector_parameters
+import egon.data.datasets.etrago_setup as etrago
+import egon.data.datasets.scenario_parameters.parameters as scenario_parameters
 
 
 class ElectricalNeighbours(Dataset):
@@ -1038,7 +1038,8 @@ def insert_storage(capacities):
     # Delete existing data
     db.execute_sql(
         f"""
-        DELETE FROM {targets['storage']['schema']}.{targets['storage']['table']}
+        DELETE FROM
+            {targets['storage']['schema']}.{targets['storage']['table']}
         WHERE bus IN (
             SELECT bus_id FROM
             {targets['buses']['schema']}.{targets['buses']['table']}
@@ -1048,7 +1049,8 @@ def insert_storage(capacities):
         """
     )
 
-    # Add missing information suitable for eTraGo selected from scenario_parameter table
+    # Add missing information suitable for eTraGo selected from
+    # scenario_parameter table
     parameters_pumped_hydro = scenario_parameters.electricity("eGon2035")[
         "efficiency"
     ]["pumped_hydro"]
@@ -1074,9 +1076,12 @@ def insert_storage(capacities):
     )
 
     # Add columns for additional parameters to df
-    store["dispatch"], store["store"], store["standing_loss"], store[
-        "max_hours"
-    ] = (None, None, None, None)
+    (
+        store["dispatch"],
+        store["store"],
+        store["standing_loss"],
+        store["max_hours"],
+    ) = (None, None, None, None)
 
     # Insert carrier specific parameters
 
@@ -1129,7 +1134,7 @@ def get_map_buses():
         "SE01": "SE02",
         "SE03": "SE02",
         "SE04": "SE02",
-        "RU":   "RU00",
+        "RU": "RU00",
     }
 
 
diff --git a/src/egon/data/datasets/heat_etrago/__init__.py b/src/egon/data/datasets/heat_etrago/__init__.py
index d0beba45f..0f188c80a 100644
--- a/src/egon/data/datasets/heat_etrago/__init__.py
+++ b/src/egon/data/datasets/heat_etrago/__init__.py
@@ -1,14 +1,15 @@
 """The central module containing all code dealing with heat sector in etrago
 """
-import pandas as pd
 import geopandas as gpd
-from egon.data import db, config
+import pandas as pd
+
+from egon.data import config, db
+from egon.data.datasets import Dataset
+from egon.data.datasets.etrago_setup import link_geom_from_buses
 from egon.data.datasets.heat_etrago.power_to_heat import (
     insert_central_power_to_heat,
     insert_individual_power_to_heat,
 )
-from egon.data.datasets import Dataset
-from egon.data.datasets.etrago_setup import link_geom_from_buses
 from egon.data.datasets.scenario_parameters import get_sector_parameters
 
 
@@ -70,7 +71,7 @@ def insert_buses(carrier, scenario):
             FROM {sources['mv_grids']['schema']}.
             {sources['mv_grids']['table']}
             WHERE bus_id IN (
-                SELECT bus_id FROM 
+                SELECT bus_id FROM
                 demand.egon_etrago_timeseries_individual_heating
                 WHERE scenario = '{scenario}')
             """
@@ -95,7 +96,6 @@ def insert_buses(carrier, scenario):
 
 def insert_store(scenario, carrier):
 
-    sources = config.datasets()["etrago_heat"]["sources"]
     targets = config.datasets()["etrago_heat"]["targets"]
 
     db.execute_sql(
@@ -274,8 +274,8 @@ def insert_central_direct_heat(scenario="eGon2035"):
         {targets['heat_generators']['table']}
         WHERE carrier IN ('solar_thermal_collector', 'geo_thermal')
         AND scn_name = '{scenario}'
-        AND bus IN 
-        (SELECT bus_id 
+        AND bus IN
+        (SELECT bus_id
          FROM {targets['heat_buses']['schema']}.
          {targets['heat_buses']['table']}
          WHERE scn_name = '{scenario}'
@@ -492,14 +492,14 @@ def insert_rural_gas_boilers(scenario="eGon2035"):
         {targets['heat_links']['table']}
         WHERE carrier  = 'rural_gas_boiler'
         AND scn_name = '{scenario}'
-        AND bus0 IN 
-        (SELECT bus_id 
+        AND bus0 IN
+        (SELECT bus_id
          FROM {targets['heat_buses']['schema']}.
          {targets['heat_buses']['table']}
          WHERE scn_name = '{scenario}'
          AND country = 'DE')
-        AND bus1 IN 
-        (SELECT bus_id 
+        AND bus1 IN
+        (SELECT bus_id
          FROM {targets['heat_buses']['schema']}.
          {targets['heat_buses']['table']}
          WHERE scn_name = '{scenario}'

From 81bca5f5bba60d53a3a1e11451210e4666d06652 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 7 Sep 2022 17:49:06 +0200
Subject: [PATCH 06/16] Reformat "emobility/**/model_timeseries.py"

There's some interesting stuff in here. First, some of the lines, the
ones within the `# fmt: on`/`# fmt: off` blocks, could not be formatted
by "black" due to psf/black#510. Second, I had to wrestle with "isort"
quite a bit because it kept formatting the imports to be too long so I
had to resort to the `# noqa: E501` solution. Last but not least, if an
import is only used in a docstring, "flake8" doesn't seem to recognize
it as used.
---
 .../model_timeseries.py                       | 31 ++++++++++++++-----
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/src/egon/data/datasets/emobility/motorized_individual_travel/model_timeseries.py b/src/egon/data/datasets/emobility/motorized_individual_travel/model_timeseries.py
index 75b86fc78..b310cbaa7 100644
--- a/src/egon/data/datasets/emobility/motorized_individual_travel/model_timeseries.py
+++ b/src/egon/data/datasets/emobility/motorized_individual_travel/model_timeseries.py
@@ -37,7 +37,10 @@
 import pandas as pd
 
 from egon.data import db
-from egon.data.datasets.emobility.motorized_individual_travel.db_classes import (
+from egon.data.datasets.emobility.motorized_individual_travel import (  # noqa: F401, E501 (Used, but only in a docstring. Also, "isort" forces the long line by moving the comment to the import line.)
+    MotorizedIndividualTravel,
+)
+from egon.data.datasets.emobility.motorized_individual_travel.db_classes import (  # noqa: E501 (Can't do anything about the long module hierarchy.)
     EgonEvMvGridDistrict,
     EgonEvPool,
     EgonEvTrip,
@@ -93,7 +96,7 @@ def data_preprocessing(
     # charging capacity in MVA
     ev_data_df = ev_data_df.assign(
         charging_capacity_grid_MW=(
-            ev_data_df.charging_capacity_grid / 10 ** 3
+            ev_data_df.charging_capacity_grid / 10**3
         ),
         minimum_charging_time=(
             ev_data_df.charging_demand
@@ -619,7 +622,11 @@ def write_link(scenario_name: str) -> None:
                         carrier="BEV charger",
                         efficiency=float(run_config.eta_cp),
                         p_nom=(
-                            load_time_series_df.simultaneous_plugged_in_charging_capacity.max()
+                            # fmt: off
+                            load_time_series_df
+                            .simultaneous_plugged_in_charging_capacity
+                            .max()
+                            # fmt: on
                         ),
                         p_nom_extendable=False,
                         p_nom_min=0,
@@ -641,7 +648,11 @@ def write_link(scenario_name: str) -> None:
                         temp_id=1,
                         p_min_pu=None,
                         p_max_pu=(
-                            hourly_load_time_series_df.ev_availability.to_list()
+                            # fmt: off
+                            hourly_load_time_series_df
+                            .ev_availability
+                            .to_list()
+                            # fmt: on
                         ),
                     )
                 )
@@ -744,7 +755,11 @@ def write_load(
                 scenario_name=scenario_name,
                 connection_bus_id=emob_bus_id,
                 load_ts=(
-                    hourly_load_time_series_df.driving_load_time_series.to_list()
+                    # fmt: off
+                    hourly_load_time_series_df
+                    .driving_load_time_series
+                    .to_list()
+                    # fmt: on
                 ),
             )
         else:
@@ -840,9 +855,9 @@ def write_to_file():
 
     # Write to database: regular and noflex scenario
     write_to_db(write_noflex_model=False)
-    print('    Writing flex scenario...')
+    print("    Writing flex scenario...")
     if write_noflex_model is True:
-        print('    Writing noflex scenario...')
+        print("    Writing noflex scenario...")
         write_to_db(write_noflex_model=True)
 
     # Export to working dir if requested
@@ -930,7 +945,7 @@ def generate_model_data_bunch(scenario_name: str, bunch: range) -> None:
         Note: `bunch` is NOT a list of grid districts but is used for slicing
         the ordered list (by bus_id) of grid districts! This is used for
         parallelization. See
-        :meth:`egon.data.datasets.emobility.motorized_individual_travel.MotorizedIndividualTravel.generate_model_data_tasks`
+        :meth:`MotorizedIndividualTravel.generate_model_data_tasks`
     """
     # Get list of grid districts / substations for this bunch
     mvgd_bus_ids = load_grid_district_ids().iloc[bunch]

From 91af5d2d7b0245e71e778cf83caed586dd5d3b0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 7 Sep 2022 18:31:13 +0200
Subject: [PATCH 07/16] Stop incrementing `next_etrago_id` results

Since the IDs are randomly generated now, there's no reason to increment
them anymore. Both variants are (roughly) equally likely to collide with
IDs already in use.
---
 src/egon/data/datasets/electrical_neighbours.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/egon/data/datasets/electrical_neighbours.py b/src/egon/data/datasets/electrical_neighbours.py
index 7c6aeeb85..af3a6c786 100755
--- a/src/egon/data/datasets/electrical_neighbours.py
+++ b/src/egon/data/datasets/electrical_neighbours.py
@@ -158,7 +158,7 @@ def buses(scenario, sources, targets):
 
     central_buses = central_buses_egon100(sources)
 
-    next_bus_id = db.next_etrago_id("bus") + 1
+    next_bus_id = db.next_etrago_id("bus")
 
     # if in test mode, add bus in center of Germany
     if config.settings()["egon-data"]["--dataset-boundary"] != "Everything":
@@ -620,7 +620,7 @@ def foreign_dc_lines(scenario, sources, targets, central_buses):
             pd.DataFrame(
                 index=[1],
                 data={
-                    "link_id": db.next_etrago_id("link") + 1,
+                    "link_id": db.next_etrago_id("link"),
                     "bus0": converter_bentwisch,
                     "bus1": central_buses[
                         (central_buses.country == "DK")

From cdc322084888ee682a5ddb6a0f9fc4aa08be9f5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 7 Sep 2022 18:47:17 +0200
Subject: [PATCH 08/16] Fix various style issues in "datasets/gas_grid.py"

Mostly overly long lines but also one missing space after an inline
comment (`#`) sign and one f-string without placeholders.
---
 src/egon/data/datasets/gas_grid.py | 34 ++++++++++++++++++++----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/egon/data/datasets/gas_grid.py b/src/egon/data/datasets/gas_grid.py
index ddee6d96b..39cd42fc4 100755
--- a/src/egon/data/datasets/gas_grid.py
+++ b/src/egon/data/datasets/gas_grid.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-The central module containing all code dealing with importing data from SciGRID_gas IGGIELGN data
+The central module containing all code dealing with importing data from
+SciGRID_gas IGGIELGN data.
 """
 from pathlib import Path
 from urllib.request import urlretrieve
@@ -55,7 +56,7 @@ def download_SciGRID_gas_data():
         "Productions",
         "Storages",
         "LNGs",
-    ]  #'Compressors'
+    ]  # 'Compressors'
     files = []
     for i in components:
         files.append("data/" + basename + "_" + i + ".csv")
@@ -141,7 +142,9 @@ def insert_CH4_nodes_list(gas_nodes_list):
 
     gas_nodes_list = gas_nodes_list[
         gas_nodes_list["country_code"].str.match("DE")
-    ]  # To eventually replace with a test if the nodes are in the german boundaries.
+    ]
+    # To eventually replace with a test whether the nodes are in the
+    # german boundaries.
 
     # Cut data to federal state if in testmode
     NUTS1 = []
@@ -175,7 +178,8 @@ def insert_CH4_nodes_list(gas_nodes_list):
             gas_nodes_list["NUTS1"].isin([map_states[boundary], np.nan])
         ]
 
-        # A completer avec nodes related to pipelines which have an end in the selected area et evt deplacer ds define_gas_nodes_list
+        # A completer avec nodes related to pipelines which have an end
+        # in the selected area et evt deplacer ds define_gas_nodes_list
 
     # Add missing columns
     c = {"scn_name": "eGon2035", "carrier": "CH4"}
@@ -217,7 +221,10 @@ def insert_CH4_nodes_list(gas_nodes_list):
 
 
 def insert_gas_buses_abroad(scn_name="eGon2035"):
-    """Insert central gas buses in foreign countries to db, same buses than the foreign AC buses
+    """Insert central gas buses in foreign countries to db.
+
+    These are the same buses as the foreign AC buses.
+
     Parameters
     ----------
     scn_name : str
@@ -226,7 +233,8 @@ def insert_gas_buses_abroad(scn_name="eGon2035"):
     Returns
     -------
     gdf_abroad_buses : dataframe
-        Dataframe containing the gas in the neighbouring countries and one in the center of Germany in test mode
+        Dataframe containing the gas in the neighbouring countries and
+        one in the center of Germany in test mode
     """
     # Select sources and targets from dataset configuration
     sources = config.datasets()["electrical_neighbours"]["sources"]
@@ -646,8 +654,10 @@ def insert_gas_pipeline_list(
 
     # Insert data to db
     db.execute_sql(
-        f"""DELETE FROM grid.egon_etrago_link WHERE "carrier" = '{main_gas_carrier}' AND
-           scn_name = '{scn_name}';
+        f"""DELETE FROM
+                grid.egon_etrago_link
+            WHERE "carrier" = '{main_gas_carrier}'
+            AND scn_name = '{scn_name}';
         """
     )
 
@@ -698,11 +708,13 @@ def remove_isolated_gas_buses():
         AND scn_name = 'eGon2035'
         AND country = 'DE'
         AND "bus_id" NOT IN
-            (SELECT bus0 FROM {targets['links']['schema']}.{targets['links']['table']}
+            (SELECT bus0 FROM
+                {targets['links']['schema']}.{targets['links']['table']}
             WHERE scn_name = 'eGon2035'
             AND carrier = 'CH4')
         AND "bus_id" NOT IN
-            (SELECT bus1 FROM {targets['links']['schema']}.{targets['links']['table']}
+            (SELECT bus1 FROM
+                {targets['links']['schema']}.{targets['links']['table']}
             WHERE scn_name = 'eGon2035'
             AND carrier = 'CH4');
     """
@@ -738,7 +750,7 @@ def insert_gas_data_eGon100RE():
     # get CH4 pipelines and modify their nominal capacity with the
     # retrofitting factor
     gdf = db.select_geodataframe(
-        f"""
+        """
         SELECT * FROM grid.egon_etrago_link
         WHERE carrier = 'CH4' AND scn_name = 'eGon2035' AND
         bus0 IN (

From ed6e4562caef2eb638c89ae8670b1f5d203b5687 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 7 Sep 2022 18:50:40 +0200
Subject: [PATCH 09/16] Remove superfluous casts to `int`

The result of `next_etrago_id` is now always an `int` so no need to cast
it.
---
 src/egon/data/datasets/electrical_neighbours.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/egon/data/datasets/electrical_neighbours.py b/src/egon/data/datasets/electrical_neighbours.py
index af3a6c786..463f848bf 100755
--- a/src/egon/data/datasets/electrical_neighbours.py
+++ b/src/egon/data/datasets/electrical_neighbours.py
@@ -946,7 +946,7 @@ def insert_generators(capacities):
     for i, row in gen.iterrows():
         entry = etrago.EgonPfHvGenerator(
             scn_name="eGon2035",
-            generator_id=int(db.next_etrago_id("generator")),
+            generator_id=db.next_etrago_id("generator"),
             bus=row.bus,
             carrier=row.carrier,
             p_nom=row.cap_2035,
@@ -1098,7 +1098,7 @@ def insert_storage(capacities):
     for i, row in store.iterrows():
         entry = etrago.EgonPfHvStorage(
             scn_name="eGon2035",
-            storage_id=int(db.next_etrago_id("storage")),
+            storage_id=db.next_etrago_id("storage"),
             bus=row.bus,
             max_hours=row.max_hours,
             efficiency_store=row.store,
@@ -1262,14 +1262,14 @@ def tyndp_demand():
 
         entry = etrago.EgonPfHvLoad(
             scn_name="eGon2035",
-            load_id=int(load_id),
+            load_id=load_id,
             carrier="AC",
             bus=int(buses.bus[bus]),
         )
 
         entry_ts = etrago.EgonPfHvLoadTimeseries(
             scn_name="eGon2035",
-            load_id=int(load_id),
+            load_id=load_id,
             temp_id=1,
             p_set=list(data_2035.values),
         )

From 4ad629d73f05ae940d4768d3b54860963b7cbcd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 7 Sep 2022 18:52:03 +0200
Subject: [PATCH 10/16] Fix import order and a trailing space

---
 src/egon/data/datasets/electricity_demand_etrago.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/egon/data/datasets/electricity_demand_etrago.py b/src/egon/data/datasets/electricity_demand_etrago.py
index 65b50a2a0..b2c4ab260 100644
--- a/src/egon/data/datasets/electricity_demand_etrago.py
+++ b/src/egon/data/datasets/electricity_demand_etrago.py
@@ -4,12 +4,13 @@
 """
 from datetime import datetime
 from pathlib import Path
-
 import os
-import egon.data.config
+
 import pandas as pd
+
 from egon.data import db
 from egon.data.datasets import Dataset
+import egon.data.config
 
 
 def demands_per_bus(scenario):
@@ -183,7 +184,7 @@ def export_to_db():
             WHERE scn_name = '{scenario}'
             AND carrier = 'AC'
             AND bus IN (
-                SELECT bus_id FROM 
+                SELECT bus_id FROM
                 {sources['etrago_buses']['schema']}.
                 {sources['etrago_buses']['table']}
                 WHERE country = 'DE'

From 7c1a2ac74f2b2c9c74900b95d94f62e2c622b3ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 7 Sep 2022 18:53:00 +0200
Subject: [PATCH 11/16] Move `next_etrago_id` calls closer to use sites

At least where possible. I also took the liberty of removing comments
like `# Select next id value` because these just restate the name of the
function in other words so one could argue that they should fall into a
category of comments which should be avoided according to [PEP8][0].

[0]: https://peps.python.org/pep-0008/#inline-comments
---
 src/egon/data/datasets/electrical_neighbours.py     |  3 +--
 src/egon/data/datasets/electricity_demand_etrago.py |  4 +---
 src/egon/data/datasets/gas_grid.py                  | 13 ++++---------
 src/egon/data/datasets/power_plants/pv_rooftop.py   |  4 +---
 4 files changed, 7 insertions(+), 17 deletions(-)

diff --git a/src/egon/data/datasets/electrical_neighbours.py b/src/egon/data/datasets/electrical_neighbours.py
index 463f848bf..b65f023e0 100755
--- a/src/egon/data/datasets/electrical_neighbours.py
+++ b/src/egon/data/datasets/electrical_neighbours.py
@@ -1241,8 +1241,6 @@ def tyndp_demand():
         if bus in map_series.values:
             nodes.extend(list(map_series[map_series == bus].index.values))
 
-        load_id = db.next_etrago_id("load")
-
         # Some etrago bus_ids represent multiple TYNDP nodes,
         # in this cases the loads are summed
         data_2030 = pd.Series(index=range(8760), data=0.0)
@@ -1260,6 +1258,7 @@ def tyndp_demand():
         # According to the NEP, data for 2030 and 2040 is linear interpolated
         data_2035 = ((data_2030 + data_2040) / 2)[:8760]
 
+        load_id = db.next_etrago_id("load")
         entry = etrago.EgonPfHvLoad(
             scn_name="eGon2035",
             load_id=load_id,
diff --git a/src/egon/data/datasets/electricity_demand_etrago.py b/src/egon/data/datasets/electricity_demand_etrago.py
index b2c4ab260..f258d150e 100644
--- a/src/egon/data/datasets/electricity_demand_etrago.py
+++ b/src/egon/data/datasets/electricity_demand_etrago.py
@@ -226,14 +226,12 @@ def export_to_db():
             columns=["scn_name", "load_id", "temp_id", "p_set", "q_set"]
         )
 
-        # Choose next unused load_id
-        next_load_id = db.next_etrago_id("load")
-
         # Insert values into load df
         load.bus = curves.bus
         load.scn_name = scenario
         load.sign = -1
         load.carrier = "AC"
+        next_load_id = db.next_etrago_id("load")
         load.load_id = range(next_load_id, next_load_id + len(load))
         load.p_set = curves.p_set
 
diff --git a/src/egon/data/datasets/gas_grid.py b/src/egon/data/datasets/gas_grid.py
index 39cd42fc4..bfeb17e63 100755
--- a/src/egon/data/datasets/gas_grid.py
+++ b/src/egon/data/datasets/gas_grid.py
@@ -77,9 +77,6 @@ def define_gas_nodes_list():
         Dataframe containing the gas nodes (Europe)
 
     """
-    # Select next id value
-    new_id = db.next_etrago_id("bus")
-
     target_file = (
         Path(".") / "datasets" / "gas_data" / "data" / "IGGIELGN_Nodes.csv"
     )
@@ -101,6 +98,7 @@ def define_gas_nodes_list():
 
     gas_nodes_list = gas_nodes_list.rename(columns={"lat": "y", "long": "x"})
 
+    new_id = db.next_etrago_id("bus")
     gas_nodes_list["bus_id"] = range(new_id, new_id + len(gas_nodes_list))
     gas_nodes_list = gas_nodes_list.set_index("id")
 
@@ -256,9 +254,6 @@ def insert_gas_buses_abroad(scn_name="eGon2035"):
     gdf_abroad_buses = central_buses_egon100(sources)
     gdf_abroad_buses = gdf_abroad_buses.drop_duplicates(subset=["country"])
 
-    # Select next id value
-    new_id = db.next_etrago_id("bus")
-
     gdf_abroad_buses = gdf_abroad_buses.drop(
         columns=[
             "v_nom",
@@ -270,6 +265,8 @@ def insert_gas_buses_abroad(scn_name="eGon2035"):
     )
     gdf_abroad_buses["scn_name"] = "eGon2035"
     gdf_abroad_buses["carrier"] = main_gas_carrier
+
+    new_id = db.next_etrago_id("bus")
     gdf_abroad_buses["bus_id"] = range(new_id, new_id + len(gdf_abroad_buses))
 
     # Add central bus in Russia
@@ -346,9 +343,6 @@ def insert_gas_pipeline_list(
 
     engine = db.engine()
 
-    # Select next id value
-    new_id = db.next_etrago_id("link")
-
     classifiaction_file = (
         Path(".")
         / "data_bundle_egon_data"
@@ -388,6 +382,7 @@ def insert_gas_pipeline_list(
         ~gas_pipelines_list["id"].str.match("EntsoG_Map__ST_195")
     ]
 
+    new_id = db.next_etrago_id("link")
     gas_pipelines_list["link_id"] = range(
         new_id, new_id + len(gas_pipelines_list)
     )
diff --git a/src/egon/data/datasets/power_plants/pv_rooftop.py b/src/egon/data/datasets/power_plants/pv_rooftop.py
index c50d9b79a..c0a693551 100644
--- a/src/egon/data/datasets/power_plants/pv_rooftop.py
+++ b/src/egon/data/datasets/power_plants/pv_rooftop.py
@@ -135,10 +135,8 @@ def pv_rooftop_per_mv_grid_and_scenario(scenario, level):
 
         capacities = demand["share_country"].mul(target)
 
-    # Select next id value
-    new_id = db.next_etrago_id("generator")
-
     # Store data in dataframe
+    new_id = db.next_etrago_id("generator")
     pv_rooftop = pd.DataFrame(
         data={
             "scn_name": scenario,

From 99c2b15aa17f046af56229e53594436c51dd77d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 7 Sep 2022 18:59:17 +0200
Subject: [PATCH 12/16] Remove unused local variables

As requested by "flake8".
---
 src/egon/data/datasets/fill_etrago_gen.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/egon/data/datasets/fill_etrago_gen.py b/src/egon/data/datasets/fill_etrago_gen.py
index 68d07ad7f..9da16fe34 100644
--- a/src/egon/data/datasets/fill_etrago_gen.py
+++ b/src/egon/data/datasets/fill_etrago_gen.py
@@ -48,11 +48,11 @@ def fill_etrago_generators():
 
     etrago_pp = add_marginal_costs(etrago_pp)
 
-    etrago_gen_table = fill_etrago_gen_table(
+    fill_etrago_gen_table(
         etrago_pp2=etrago_pp, etrago_gen_orig=etrago_gen_orig, cfg=cfg, con=con
     )
 
-    etrago_gen_time_table = fill_etrago_gen_time_table(
+    fill_etrago_gen_time_table(
         etrago_pp=etrago_pp,
         power_plants=power_plants,
         renew_feedin=renew_feedin,

From a223112c895caf045f9fc70ac8985060a91a4d06 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 7 Sep 2022 18:59:46 +0200
Subject: [PATCH 13/16] Rename `max_id` to `new_id`

Since we now generate IDs randomly, they are no longer "guaranteed" to
be the current maximum so the old name is misleading.
---
 src/egon/data/datasets/fill_etrago_gen.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/egon/data/datasets/fill_etrago_gen.py b/src/egon/data/datasets/fill_etrago_gen.py
index 9da16fe34..3da5f0bb8 100644
--- a/src/egon/data/datasets/fill_etrago_gen.py
+++ b/src/egon/data/datasets/fill_etrago_gen.py
@@ -81,8 +81,8 @@ def group_power_plants(power_plants, renew_feedin, etrago_gen_orig, cfg):
     )
     etrago_pp = etrago_pp.reset_index(drop=True)
 
-    max_id = db.next_etrago_id("generator")
-    etrago_pp["generator_id"] = list(range(max_id, max_id + len(etrago_pp)))
+    new_id = db.next_etrago_id("generator")
+    etrago_pp["generator_id"] = list(range(new_id, new_id + len(etrago_pp)))
     etrago_pp.set_index("generator_id", inplace=True)
 
     return etrago_pp

From 085448a55a713e1e9fbad14dae075995bd2a03b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 7 Sep 2022 19:02:23 +0200
Subject: [PATCH 14/16] Remove empty lines

The respective code blocks kind of belong together and having them
closer to each other makes this a little bit more obvious. See also the
[PEP8 note][0] on using "blank lines [..] sparingly".

[0]: https://peps.python.org/pep-0008/#blank-lines
---
 src/egon/data/datasets/electrical_neighbours.py | 1 -
 src/egon/data/datasets/heat_etrago/__init__.py  | 1 -
 2 files changed, 2 deletions(-)

diff --git a/src/egon/data/datasets/electrical_neighbours.py b/src/egon/data/datasets/electrical_neighbours.py
index b65f023e0..3640813ed 100755
--- a/src/egon/data/datasets/electrical_neighbours.py
+++ b/src/egon/data/datasets/electrical_neighbours.py
@@ -1265,7 +1265,6 @@ def tyndp_demand():
             carrier="AC",
             bus=int(buses.bus[bus]),
         )
-
         entry_ts = etrago.EgonPfHvLoadTimeseries(
             scn_name="eGon2035",
             load_id=load_id,
diff --git a/src/egon/data/datasets/heat_etrago/__init__.py b/src/egon/data/datasets/heat_etrago/__init__.py
index 0f188c80a..daa4d50c1 100644
--- a/src/egon/data/datasets/heat_etrago/__init__.py
+++ b/src/egon/data/datasets/heat_etrago/__init__.py
@@ -325,7 +325,6 @@ def insert_central_direct_heat(scenario="eGon2035"):
     )
 
     new_id = db.next_etrago_id("generator")
-
     generator = pd.DataFrame(
         data={
             "scn_name": scenario,

From 5593c82c9afad921f944e60841dcbc4ecb9412d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 7 Sep 2022 19:09:29 +0200
Subject: [PATCH 15/16] Fix potentially dangerous duplicate calls

Calling `next_etrago_id` two times in "close" succession did not
guarantee getting the same results even before switching to random IDs.
This was due to concurrency. Now that we're using random IDs, these two
calls are basically guaranteed to yield wildly different results (which
is the point of the new implementation), so this pattern is very
dangerous and will nearly always be wrong.
Fortunately it is easily fixed by saving the result of the first call
and using it in place of both calls.
---
 src/egon/data/datasets/chp_etrago.py          | 31 +++++++------------
 .../data/datasets/electrical_neighbours.py    |  5 ++-
 .../data/datasets/heat_etrago/__init__.py     | 20 +++++++-----
 3 files changed, 26 insertions(+), 30 deletions(-)

diff --git a/src/egon/data/datasets/chp_etrago.py b/src/egon/data/datasets/chp_etrago.py
index 594890a66..43b226aa7 100644
--- a/src/egon/data/datasets/chp_etrago.py
+++ b/src/egon/data/datasets/chp_etrago.py
@@ -108,9 +108,8 @@ def insert():
         "eGon2035",
     )
     # Set index
-    chp_el["link_id"] = range(
-        db.next_etrago_id("link"), len(chp_el) + db.next_etrago_id("link")
-    )
+    new_id = db.next_etrago_id("link")
+    chp_el["link_id"] = range(new_id, new_id + len(chp_el))
 
     # Add marginal cost which is only VOM in case of gas chp
     chp_el["marginal_cost"] = get_sector_parameters("gas", "eGon2035")[
@@ -140,9 +139,8 @@ def insert():
         "eGon2035",
     )
 
-    chp_heat["link_id"] = range(
-        db.next_etrago_id("link"), len(chp_heat) + db.next_etrago_id("link")
-    )
+    new_id = db.next_etrago_id("link")
+    chp_heat["link_id"] = range(new_id, new_id + len(chp_heat))
 
     chp_heat.to_postgis(
         targets["link"]["table"],
@@ -165,10 +163,8 @@ def insert():
         },
     )
 
-    chp_el_gen["generator_id"] = range(
-        db.next_etrago_id("generator"),
-        len(chp_el_gen) + db.next_etrago_id("generator"),
-    )
+    new_id = db.next_etrago_id("generator")
+    chp_el_gen["generator_id"] = range(new_id, new_id + len(chp_el_gen))
 
     # Add marginal cost
     chp_el_gen["marginal_cost"] = get_sector_parameters(
@@ -193,10 +189,8 @@ def insert():
         },
     )
 
-    chp_heat_gen["generator_id"] = range(
-        db.next_etrago_id("generator"),
-        len(chp_heat_gen) + db.next_etrago_id("generator"),
-    )
+    new_id = db.next_etrago_id("generator")
+    chp_heat_gen["generator_id"] = range(new_id, new_id + len(chp_heat_gen))
 
     chp_heat_gen.to_sql(
         targets["generator"]["table"],
@@ -238,9 +232,8 @@ def insert():
         "eGon2035",
     )
 
-    chp_el_ind["link_id"] = range(
-        db.next_etrago_id("link"), len(chp_el_ind) + db.next_etrago_id("link")
-    )
+    new_id = db.next_etrago_id("link")
+    chp_el_ind["link_id"] = range(new_id, new_id + len(chp_el_ind))
 
     # Add marginal cost which is only VOM in case of gas chp
     chp_el_ind["marginal_cost"] = get_sector_parameters("gas", "eGon2035")[
@@ -267,9 +260,9 @@ def insert():
         },
     )
 
+    new_id = db.next_etrago_id("generator")
     chp_el_ind_gen["generator_id"] = range(
-        db.next_etrago_id("generator"),
-        len(chp_el_ind_gen) + db.next_etrago_id("generator"),
+        new_id, new_id + len(chp_el_ind_gen)
     )
 
     # Add marginal cost
diff --git a/src/egon/data/datasets/electrical_neighbours.py b/src/egon/data/datasets/electrical_neighbours.py
index 3640813ed..4f0086d6e 100755
--- a/src/egon/data/datasets/electrical_neighbours.py
+++ b/src/egon/data/datasets/electrical_neighbours.py
@@ -332,9 +332,8 @@ def cross_border_lines(scenario, sources, targets, central_buses):
 
     if config.settings()["egon-data"]["--dataset-boundary"] == "Everything":
         new_lines = new_lines[~new_lines.country.isnull()]
-    new_lines.line_id = range(
-        db.next_etrago_id("line"), db.next_etrago_id("line") + len(new_lines)
-    )
+    new_id = db.next_etrago_id("line")
+    new_lines.line_id = range(new_id, new_id + len(new_lines))
 
     # Set bus in center of foreogn countries as bus1
     for i, row in new_lines.iterrows():
diff --git a/src/egon/data/datasets/heat_etrago/__init__.py b/src/egon/data/datasets/heat_etrago/__init__.py
index daa4d50c1..640abaae6 100644
--- a/src/egon/data/datasets/heat_etrago/__init__.py
+++ b/src/egon/data/datasets/heat_etrago/__init__.py
@@ -156,9 +156,10 @@ def insert_store(scenario, carrier):
 
     water_tank_bus = dh_bus.copy()
     water_tank_bus.carrier = carrier + "_store"
+    water_tank_bus_base_id = db.next_etrago_id("bus")
     water_tank_bus.bus_id = range(
-        db.next_etrago_id("bus"),
-        db.next_etrago_id("bus") + len(water_tank_bus),
+        water_tank_bus_base_id,
+        water_tank_bus_base_id + len(water_tank_bus),
     )
 
     water_tank_bus.to_postgis(
@@ -169,6 +170,7 @@ def insert_store(scenario, carrier):
         index=False,
     )
 
+    water_tank_charger_base_id = db.next_etrago_id("link")
     water_tank_charger = pd.DataFrame(
         data={
             "scn_name": scenario,
@@ -180,8 +182,8 @@ def insert_store(scenario, carrier):
             ]["water_tank_charger"],
             "p_nom_extendable": True,
             "link_id": range(
-                db.next_etrago_id("link"),
-                db.next_etrago_id("link") + len(water_tank_bus),
+                water_tank_charger_base_id,
+                water_tank_charger_base_id + len(water_tank_bus),
             ),
         }
     )
@@ -194,6 +196,7 @@ def insert_store(scenario, carrier):
         index=False,
     )
 
+    water_tank_discharger_base_id = db.next_etrago_id("link")
     water_tank_discharger = pd.DataFrame(
         data={
             "scn_name": scenario,
@@ -205,8 +208,8 @@ def insert_store(scenario, carrier):
             ]["water_tank_discharger"],
             "p_nom_extendable": True,
             "link_id": range(
-                db.next_etrago_id("link"),
-                db.next_etrago_id("link") + len(water_tank_bus),
+                water_tank_discharger_base_id,
+                water_tank_discharger_base_id + len(water_tank_bus),
             ),
         }
     )
@@ -219,6 +222,7 @@ def insert_store(scenario, carrier):
         index=False,
     )
 
+    water_tank_store_base_id = db.next_etrago_id("store")
     water_tank_store = pd.DataFrame(
         data={
             "scn_name": scenario,
@@ -232,8 +236,8 @@ def insert_store(scenario, carrier):
             ],
             "e_nom_extendable": True,
             "store_id": range(
-                db.next_etrago_id("store"),
-                db.next_etrago_id("store") + len(water_tank_bus),
+                water_tank_store_base_id,
+                water_tank_store_base_id + len(water_tank_bus),
             ),
         }
     )

From c194284a7f577c2b4d620ff7cc1e08e7657486b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 7 Sep 2022 21:19:46 +0200
Subject: [PATCH 16/16] Fix circular import

Turns out that it's not possible to import `MotorizedIndividualTravel`
because it leads to a circular import. Solution:

```
import egon.data.datasets.emobility.motorized_individual_travel
```

instead and pull `MotorizedIndividualTravel` out of it at use time.
---
 .../motorized_individual_travel/model_timeseries.py         | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/egon/data/datasets/emobility/motorized_individual_travel/model_timeseries.py b/src/egon/data/datasets/emobility/motorized_individual_travel/model_timeseries.py
index b310cbaa7..c7cff6922 100644
--- a/src/egon/data/datasets/emobility/motorized_individual_travel/model_timeseries.py
+++ b/src/egon/data/datasets/emobility/motorized_individual_travel/model_timeseries.py
@@ -37,9 +37,6 @@
 import pandas as pd
 
 from egon.data import db
-from egon.data.datasets.emobility.motorized_individual_travel import (  # noqa: F401, E501 (Used, but only in a docstring. Also, "isort" forces the long line by moving the comment to the import line.)
-    MotorizedIndividualTravel,
-)
 from egon.data.datasets.emobility.motorized_individual_travel.db_classes import (  # noqa: E501 (Can't do anything about the long module hierarchy.)
     EgonEvMvGridDistrict,
     EgonEvPool,
@@ -62,6 +59,7 @@
     EgonPfHvStoreTimeseries,
 )
 from egon.data.datasets.mv_grid_districts import MvGridDistricts
+import egon.data.datasets.emobility.motorized_individual_travel as emoit  # noqa: F401, E501
 
 # from egon.data.datasets.scenario_parameters import get_sector_parameters
 
@@ -945,7 +943,7 @@ def generate_model_data_bunch(scenario_name: str, bunch: range) -> None:
         Note: `bunch` is NOT a list of grid districts but is used for slicing
         the ordered list (by bus_id) of grid districts! This is used for
         parallelization. See
-        :meth:`MotorizedIndividualTravel.generate_model_data_tasks`
+        :meth:`emoit.MotorizedIndividualTravel.generate_model_data_tasks`
     """
     # Get list of grid districts / substations for this bunch
     mvgd_bus_ids = load_grid_district_ids().iloc[bunch]