Merge pull request #84 from worldbank/feature/ntl-metadata

Feature/ntl-metadata
worldbank · Nov 26, 2024 · 7839577 · 7839577
2 parents f79af23 + bcf7098
commit 7839577
Show file tree

Hide file tree

Showing 40 changed files with 7,377 additions and 5,187 deletions.
diff --git a/notebooks/MP_SCRIPTS/Combine_CSV_into_parquet.ipynb b/notebooks/MP_SCRIPTS/Combine_CSV_into_parquet.ipynb
diff --git a/notebooks/MP_SCRIPTS/Population_demographics/Combine_CSV_into_parquet.ipynb b/notebooks/MP_SCRIPTS/Population_demographics/Combine_CSV_into_parquet.ipynb
diff --git a/notebooks/MP_SCRIPTS/Population_demographics/TEST_zonal_stats_data.ipynb b/notebooks/MP_SCRIPTS/Population_demographics/TEST_zonal_stats_data.ipynb
diff --git a/notebooks/MP_SCRIPTS/Population_demographics/readme.md b/notebooks/MP_SCRIPTS/Population_demographics/readme.md
diff --git a/notebooks/MP_SCRIPTS/zonal_pop_by_gender.py → ...ation_demographics/zonal_pop_by_gender.py b/notebooks/MP_SCRIPTS/zonal_pop_by_gender.py → ...ation_demographics/zonal_pop_by_gender.py
diff --git a/notebooks/MP_SCRIPTS/TEST_zonal_stats_data.ipynb b/notebooks/MP_SCRIPTS/TEST_zonal_stats_data.ipynb
diff --git a/notebooks/MP_SCRIPTS/global_f_0_2020_1km_zonal.csv b/notebooks/MP_SCRIPTS/global_f_0_2020_1km_zonal.csv
diff --git a/notebooks/MP_SCRIPTS/testing.ipynb b/notebooks/MP_SCRIPTS/testing.ipynb
diff --git a/notebooks/MP_SCRIPTS/zonal_urbanization.py b/notebooks/MP_SCRIPTS/zonal_urbanization.py
@@ -22,11 +22,10 @@
 AWS_SESSION_TOKEN = os.getenv("AWS_SESSION_TOKEN")
 
 if __name__ == "__main__":
-    multiprocess = True
     verbose = True
-    run_urban = True
+    run_urban = False
+    run_urban_pop = True
     run_urban_pop = False
-
     tPrint("Starting")
     h3_level = 6
     data_prefix = "Urbanization"
@@ -35,6 +34,17 @@
     # Urbanization layers
     unq_urban = [11, 12, 13, 21, 22, 23, 30]
     ghsl_folder = "/home/public/Data/GLOBAL/GHSL/"
+
+    ghs_smod = os.path.join(
+        ghsl_folder, "SMOD", "GHS_SMOD_E2020_GLOBE_R2023A_54009_1000_V1_0.tif"
+    )
+    ghs_pop = os.path.join(
+        ghsl_folder, "Pop", "GHS_POP_E2020_GLOBE_R2023A_54009_100_V1_0.tif"
+    )
+
+    # h3_0_list = h3_helper.generate_lvl0_lists(h3_level, return_gdf=True, buffer0=False)
+    # if verbose:
+
     ghs_smod = os.path.join(
         ghsl_folder, "SMOD", "GHS_SMOD_E2020_GLOBE_R2023A_54009_1000_V1_0.tif"
     )
@@ -44,6 +54,7 @@
 
     # h3_0_list = h3_helper.generate_lvl0_lists(h3_level, return_gdf=True, buffer0=False)
     # if verbose:
+
     #    tPrint("H3_0 list generated")
 
     h3_1_list = h3_helper.generate_lvl1_lists(
@@ -55,6 +66,22 @@
     urban_pop_args = []
     urban_args = []
     for h3_1_key, cur_gdf in h3_1_list.items():
+        if run_urban_pop:
+            # Set up mp arguments for urban population
+            pop_filename = "GHS_POP_2020_Urban_Breakdown.csv"
+            pop_out_s3_key = f"Space2Stats/h3_stats_data/GLOBAL/{data_prefix_pop}/{h3_1_key}/{pop_filename}"
+            pop_full_path = os.path.join("s3://", AWS_S3_BUCKET, pop_out_s3_key)
+            urban_pop_args.append(
+                [cur_gdf, "shape_id", ghs_pop, ghs_smod, pop_full_path, unq_urban]
+            )
+        if run_urban:
+            # set up mp arguments for urban summary
+            urban_filename = "GHS_SMOD_2020.csv"
+            urban_out_s3_key = f"Space2Stats/h3_stats_data/GLOBAL/{data_prefix}/{h3_1_key}/{urban_filename}"
+            urban_full_path = os.path.join("s3://", AWS_S3_BUCKET, urban_out_s3_key)
+            urban_args.append(
+                [cur_gdf, "shape_id", ghs_smod, unq_urban, urban_full_path]
+            )
         # Set up mp arguments for urban population
         pop_filename = "GHS_POP_2020_Urban_Breakdown.csv"
         pop_out_s3_key = f"Space2Stats/h3_stats_data/GLOBAL/{data_prefix_pop}/{h3_1_key}/{pop_filename}"
@@ -78,41 +105,69 @@
         if multiprocess:
             with multiprocessing.Pool(processes=min([70, len(urban_args)])) as pool:
                 results = pool.starmap(global_zonal.zonal_stats_categories, urban_args)
+            tPrint(f"Finished urban calculations: {len(results)}")
+            for combo in results:
+                out_file = list(combo.keys())[0]
+                res = combo[out_file]
+                res.to_csv(
+                    out_file,
+                    storage_options={
+                        "key": AWS_ACCESS_KEY_ID,
+                        "secret": AWS_SECRET_ACCESS_KEY,
+                        "token": AWS_SESSION_TOKEN,
+                    },
+                )
         else:
-            for a in arg_list:
-                results = run_zonal(*a)
-        for combo in results:
-            out_file = list(combo.keys())[0]
-            res = combo[out_file]
-            res.to_csv(
-                f"s3://{AWS_S3_BUCKET}/{out_file}",
-                index=False,
-                storage_options={
-                    "key": AWS_ACCESS_KEY_ID,
-                    "secret": AWS_SECRET_ACCESS_KEY,
-                    "token": AWS_SESSION_TOKEN,
-                },
-            )
+            for a in urban_args:
+                results = global_zonal.zonal_stats_categories(*a)
+                out_file = list(results.keys())[0]
+                res = combo[out_file]
+                res.to_csv(
+                    out_file,
+                    storage_options={
+                        "key": AWS_ACCESS_KEY_ID,
+                        "secret": AWS_SECRET_ACCESS_KEY,
+                        "token": AWS_SESSION_TOKEN,
+                    },
+                )
+                tPrint(f"Finished {out_file}")
 
     if run_urban_pop:
+        tPrint(
+            f"Running calculations on urban population: {len(urban_pop_args)} processes"
+        )
         # Run multi processing on urban_pop_calculations
         if multiprocess:
-            with multiprocessing.Pool(processes=min([70, len(urban_pop_args)])) as pool:
+            with multiprocessing.Pool(processes=min([40, len(urban_pop_args)])) as pool:
                 results = pool.starmap(
                     global_zonal.zonal_stats_categorical, urban_pop_args
                 )
+            tPrint(f"Finished multiprocessing urban pop calculations: {len(results)}")
+            for combo in results:
+                out_file = list(combo.keys())[0]
+                res = combo[out_file]
+                res.to_csv(
+                    out_file,
+                    storage_options={
+                        "key": AWS_ACCESS_KEY_ID,
+                        "secret": AWS_SECRET_ACCESS_KEY,
+                        "token": AWS_SESSION_TOKEN,
+                    },
+                )
         else:
-            for a in arg_list:
-                results = run_zonal(*a)
-        for combo in results:
-            out_file = list(combo.keys())[0]
-            res = combo[out_file]
-            res.to_csv(
-                f"s3://{AWS_S3_BUCKET}/{out_file}",
-                index=False,
-                storage_options={
-                    "key": AWS_ACCESS_KEY_ID,
-                    "secret": AWS_SECRET_ACCESS_KEY,
-                    "token": AWS_SESSION_TOKEN,
-                },
-            )
+            for a in urban_pop_args:
+                combo = global_zonal.zonal_stats_categorical(
+                    *a, verbose=verbose, minVal=0
+                )
+                out_file = list(combo.keys())[0]
+                tPrint(f"Completed {out_file}")
+                res = combo[out_file]
+                res.to_csv(
+                    out_file,
+                    storage_options={
+                        "key": AWS_ACCESS_KEY_ID,
+                        "secret": AWS_SECRET_ACCESS_KEY,
+                        "token": AWS_SESSION_TOKEN,
+                    },
+                )
+    tPrint("Finished")
diff --git a/notebooks/Evaluate_Hex_Zonal_Effect.ipynb → ...ploration/Evaluate_Hex_Zonal_Effect.ipynb b/notebooks/Evaluate_Hex_Zonal_Effect.ipynb → ...ploration/Evaluate_Hex_Zonal_Effect.ipynb
diff --git a/notebooks/Explore_H3_Admin_Size.ipynb → ...a_exploration/Explore_H3_Admin_Size.ipynb b/notebooks/Explore_H3_Admin_Size.ipynb → ...a_exploration/Explore_H3_Admin_Size.ipynb
diff --git a/notebooks/H3_Generate_AWS.ipynb → ...ks/data_exploration/H3_Generate_AWS.ipynb b/notebooks/H3_Generate_AWS.ipynb → ...ks/data_exploration/H3_Generate_AWS.ipynb
diff --git a/notebooks/H3_zonal_stats_AWS.ipynb → ...data_exploration/H3_zonal_stats_AWS.ipynb b/notebooks/H3_zonal_stats_AWS.ipynb → ...data_exploration/H3_zonal_stats_AWS.ipynb
diff --git a/notebooks/Intersect_Points_H3.ipynb → ...ata_exploration/Intersect_Points_H3.ipynb b/notebooks/Intersect_Points_H3.ipynb → ...ata_exploration/Intersect_Points_H3.ipynb
diff --git a/...ks/Space2Stats_Comparing_H3_WBAdmin.ipynb → ...on/Space2Stats_Comparing_H3_WBAdmin.ipynb b/...ks/Space2Stats_Comparing_H3_WBAdmin.ipynb → ...on/Space2Stats_Comparing_H3_WBAdmin.ipynb
diff --git a/notebooks/Space2Stats_RunZonal.ipynb → ...ta_exploration/Space2Stats_RunZonal.ipynb b/notebooks/Space2Stats_RunZonal.ipynb → ...ta_exploration/Space2Stats_RunZonal.ipynb
diff --git a/notebooks/Space2Stats_RunZonal_HTI.ipynb → ...xploration/Space2Stats_RunZonal_HTI.ipynb b/notebooks/Space2Stats_RunZonal_HTI.ipynb → ...xploration/Space2Stats_RunZonal_HTI.ipynb
diff --git a/.../Space2Stats_RunZonal_Scorecard_Pop.ipynb → .../Space2Stats_RunZonal_Scorecard_Pop.ipynb b/.../Space2Stats_RunZonal_Scorecard_Pop.ipynb → .../Space2Stats_RunZonal_Scorecard_Pop.ipynb
diff --git a/notebooks/ZONAL_TESTING.ipynb → ...ooks/data_exploration/ZONAL_TESTING.ipynb b/notebooks/ZONAL_TESTING.ipynb → ...ooks/data_exploration/ZONAL_TESTING.ipynb
diff --git a/notebooks/h3-food-security.ipynb → ...s/data_exploration/h3-food-security.ipynb b/notebooks/h3-food-security.ipynb → ...s/data_exploration/h3-food-security.ipynb
diff --git a/notebooks/space2stats_api_adm_example.ipynb → ...oration/space2stats_api_adm_example.ipynb b/notebooks/space2stats_api_adm_example.ipynb → ...oration/space2stats_api_adm_example.ipynb
diff --git a/notebooks/visualize_nyc_sample.ipynb → ...ta_exploration/visualize_nyc_sample.ipynb b/notebooks/visualize_nyc_sample.ipynb → ...ta_exploration/visualize_nyc_sample.ipynb