Upgrade Airflow (#16)

aliavni · Aug 19, 2024 · b8feb16 · b8feb16
1 parent 876daa7
commit b8feb16
Show file tree

Hide file tree

Showing 7 changed files with 18 additions and 16 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -11,13 +11,13 @@ repos:
       - id: check-added-large-files
       - id: requirements-txt-fixer
   - repo: https://github.com/PyCQA/bandit
-    rev: 1.7.8
+    rev: 1.7.9
     hooks:
       - id: bandit
         args: ["-c", "pyproject.toml"]
         additional_dependencies: ["bandit[toml]"]
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.5
+    rev: v0.6.1
     hooks:
       - id: ruff
         types_or: [ python, pyi, jupyter ]
@@ -37,6 +37,6 @@ repos:
         files:
           ^README\.md$
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.39.0
+    rev: v0.41.0
     hooks:
     - id: markdownlint
diff --git a/Dockerfile b/Dockerfile
@@ -1,8 +1,9 @@
 # syntax=docker/dockerfile:1
-FROM python:3.12.2
+FROM python:3.12.5
 
 COPY requirements.txt /tmp/requirements.txt
 COPY docker/jupyter/requirements.txt /tmp/docker/jupyter/requirements.txt
+COPY docker/airflow/requirements.txt /tmp/docker/airflow/requirements.txt
 
 RUN --mount=type=cache,mode=0755,target=/root/.cache \
     pip install -r /tmp/requirements.txt

diff --git a/docker/airflow/Dockerfile b/docker/airflow/Dockerfile
@@ -1,4 +1,4 @@
-FROM apache/airflow:2.9.1-python3.11
+FROM apache/airflow:2.10.0-python3.12
 
 USER root
 RUN apt-get update && apt-get install -y curl wget vim

diff --git a/docker/airflow/requirements.txt b/docker/airflow/requirements.txt
@@ -1,9 +1,9 @@
--c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.1/constraints-3.11.txt
-apache-airflow-providers-amazon==8.20.0
-apache-airflow-providers-apache-spark==4.7.2
-apache-airflow-providers-mongo==4.0.0
-apache-airflow-providers-slack==8.6.2
+-c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.0/constraints-3.12.txt
+apache-airflow-providers-amazon==8.27.0
+apache-airflow-providers-apache-spark==4.9.0
+apache-airflow-providers-mongo==4.1.2
+apache-airflow-providers-slack==8.8.0
 delta-spark==3.2.0
-deltalake==0.17.3
-polars==0.20.31
+deltalake==0.18.2
+polars==1.5.0
 pyspark==3.5.1
diff --git a/docker/jupyter/requirements.txt b/docker/jupyter/requirements.txt
@@ -6,8 +6,8 @@ grpcio
 grpcio-status
 jupyterlab==4.2.4
 kafka-python==2.0.2
-pandas==2.2.2
-polars==0.20.31
+pandas==2.1.4
+polars==1.5.0
 protobuf
-pymongo==4.7.3
+pymongo==4.8.0
 pyspark==3.5.1
diff --git a/requirements.txt b/requirements.txt
@@ -1,3 +1,4 @@
+-r docker/airflow/requirements.txt
 -r docker/jupyter/requirements.txt
 
 psycopg2-binary==2.9.9
diff --git a/requirements_dev.txt b/requirements_dev.txt
@@ -2,5 +2,5 @@
 
 ipdb==0.13.13
 pre-commit==3.8.0
-pytest==8.2.2
+pytest==8.3.2
 pytest-cov==5.0.0