upgrade setuptools

aliavni · May 28, 2024 · 8e5bba1 · 8e5bba1
1 parent 83c496a
commit 8e5bba1
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 5 deletions.
diff --git a/airflow/dags/airbnb_job.py b/airflow/dags/airbnb_job.py
@@ -4,13 +4,16 @@
 import argparse
 
 
-parser = argparse.ArgumentParser()
-parser.add_argument("--url", dest="url", type=str, help="Inside Airbnb data url")
-args = parser.parse_args()
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--url", dest="url", type=str, help="Inside Airbnb data url")
+    return parser.parse_args()
+
 
 spark = SparkSession.builder.appName("airbnb").getOrCreate()
 sc = spark.sparkContext
 
+args = get_args()
 df = spark.createDataFrame(pd.read_csv(args.url))
 
 print(f"# of rows: {df.count()}")

diff --git a/docker/airflow/requirements_spark.txt b/docker/airflow/requirements_spark.txt
@@ -1,3 +1,3 @@
 # requirements should be the same as `docker/spark/requirements.txt`
 pandas==2.2.2
-setuptools==65.5.0
+setuptools==70.0.0
diff --git a/docker/spark/requirements.txt b/docker/spark/requirements.txt
@@ -1,3 +1,3 @@
 # requirements should be the same as `docker/airflow/requirements_spark.txt`
 pandas==2.2.2
-setuptools==65.5.0
+setuptools==70.0.0