docker-compose.yaml

name: docker

x-common:
  &common
  networks:
    - backend

x-airflow-common:
  &airflow-common
  build: ./docker/airflow
  env_file:
    - .env
  volumes:
    - ./air/dags:/opt/airflow/dags
    - ./air/logs:/opt/airflow/logs
    - ./air/config:/opt/airflow/config
    - ./air/plugins:/opt/airflow/plugins
  user: "${AIRFLOW_UID:-50000}:0"
  depends_on:
    &airflow-common-depends-on
    redis:
      condition: service_healthy
    db:
      condition: service_healthy
  networks:
    - backend

services:
  db:
    <<: *common
    container_name: docker-postgres
    image: postgres:16
    ports:
      - 5433:5432
    restart: always
    env_file:
      - .env
    volumes:
      - pgdata:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -d postgres -U postgres"]
      interval: 10s
      timeout: 5s
      retries: 5

  python:
    <<: *common
    container_name: python
    build:
      context: .
    depends_on:
      db:
        condition: service_healthy
    env_file:
      - .env

  trino:
    <<: *common
    container_name: trino
    ports:
      - "8080:8080"
    image: "trinodb/trino:444"
    volumes:
      - ./docker/trino/etc:/usr/lib/trino/etc:ro
      - ./docker/trino/catalog:/etc/trino/catalog
    environment:
      - CATALOG_MANAGEMENT=dynamic

  spark-master:
    <<: *common
    container_name: spark-master
    build: ./docker/spark
    ports:
      - "4040:4040"
      - "6066:6066"
      - "7077:7077"
      - "9090:8080"
      - "9870:9870"
      - "10000:10000"
    volumes:
       - ./spark-apps:/opt/spark-apps
       - ./data/spark-data:/opt/spark-data
    environment:
      - SPARK_LOCAL_IP=spark-master
      - SPARK_WORKLOAD=master
    profiles:
      - spark
  spark-worker-a:
    <<: *common
    build: ./docker/spark
    container_name: spark-worker-a
    deploy:
      resources:
        limits:
          memory: 10GB
    env_file:
      - .env
    ports:
      - "10091:8080"
      - "7001:7000"
    depends_on:
      - spark-master
    environment:
      - SPARK_WORKLOAD=worker
      - SPARK_LOCAL_IP=spark-worker-a
    volumes:
       - ./spark-apps:/opt/spark-apps
       - ./data/spark-data:/opt/spark-data
    profiles:
      - spark
  spark-worker-b:
    <<: *common
    build: ./docker/spark
    container_name: spark-worker-b
    deploy:
      resources:
        limits:
          memory: 10GB
    ports:
      - "10092:8080"
      - "7002:7000"
    depends_on:
      - spark-master
    environment:
      - SPARK_WORKLOAD=worker
      - SPARK_LOCAL_IP=spark-worker-b
    volumes:
       - ./spark-apps:/opt/spark-apps
       - ./data/spark-data:/opt/spark-data
    profiles:
      - spark

  jupyterlab:
    <<: *common
    container_name: jupyterlab
    build: ./docker/jupyter
    environment:
      - JUPYTER_PORT=8089
    volumes:
      - ./notebooks:$LAB_HOME
      - ./data:$LAB_HOME/data
    env_file:
      - .env
    ports:
      - "8089:8089"

  scylla-1:
    <<: *common
    build: ./docker/scylladb
    container_name: scylla-1
    restart: always
    command: --seeds=scylla-1,scylla-2,scylla-3 --smp 1 --memory 750M --overprovisioned 1
    volumes:
      - ./data/scylla:/var/lib/scylla/data
      - ./docker/scylladb/scylla.yaml:/etc/scylla/scylla.yaml
      - ./docker/scylladb/cassandra-rackdc.properties.dc1:/etc/scylla/cassandra-rackdc.properties
    ports:
      - "19042:19042"
    profiles:
      - scylla

  scylla-2:
    <<: *common
    build: ./docker/scylladb
    container_name: scylla-2
    restart: always
    volumes:
      - ./data/scylla:/var/lib/scylla
      - ./docker/scylladb/scylla.yaml:/etc/scylla/scylla.yaml
      - ./docker/scylladb/cassandra-rackdc.properties.dc1:/etc/scylla/cassandra-rackdc.properties
    command: --seeds=scylla-1,scylla-2,scylla-3 --smp 1 --memory 750M --overprovisioned 1
    profiles:
      - scylla

  scylla-3:
    <<: *common
    build: ./docker/scylladb
    container_name: scylla-3
    restart: always
    volumes:
      - ./data/scylla:/var/lib/scylla
      - ./docker/scylladb/scylla.yaml:/etc/scylla/scylla.yaml
      - ./docker/scylladb/cassandra-rackdc.properties.dc1:/etc/scylla/cassandra-rackdc.properties
    command: --seeds=scylla-1,scylla-2,scylla-3 --smp 1 --memory 750M --overprovisioned 1
    profiles:
      - scylla

  kafka:
    <<: *common
    container_name: kafka
    build: ./docker/kafka
    volumes:
      - ./data/kafka:/bitnami/kafka
    env_file:
      - .env
    ports:
      - '9092:9092'
      - '9093:9093'
      - '9094:9094'

  redis:
    <<: *common
    container_name: redis
    # Redis is limited to 7.2-bookworm due to licencing change
    # https://redis.io/blog/redis-adopts-dual-source-available-licensing/
    image: redis:7.2-bookworm
    ports:
      - "6379:6379"
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 10s
      timeout: 30s
      retries: 50
      start_period: 30s
    restart: always

  airflow-webserver:
    <<: *airflow-common
    command: webserver
    ports:
      - "8081:8080"
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 30s
    restart: always
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-scheduler:
    <<: *airflow-common
    command: scheduler
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:8974/health"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 30s
    restart: always
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-worker:
    <<: *airflow-common
    command: celery worker
    ports:
      - "8793:8793"
    deploy:
      resources:
        limits:
          memory: 10GB
    healthcheck:
      # yamllint disable rule:line-length
      test:
        - "CMD-SHELL"
        - 'celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}" || celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 30s
    environment:
      # Required to handle warm shutdown of the celery workers properly
      # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation
      DUMB_INIT_SETSID: "0"
    restart: always
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-triggerer:
    <<: *airflow-common
    command: triggerer
    ports:
      - "8794:8794"
    healthcheck:
      test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"']
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 30s
    restart: always
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-init:
    <<: *airflow-common
    entrypoint: /bin/bash
    # yamllint disable rule:line-length
    command:
      - -c
      - |
        if [[ -z "${AIRFLOW_UID}" ]]; then
          echo
          echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m"
          echo "If you are on Linux, you SHOULD follow the instructions below to set "
          echo "AIRFLOW_UID environment variable, otherwise files will be owned by root."
          echo "For other operating systems you can get rid of the warning with manually created .env file:"
          echo "    See: https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#setting-the-right-airflow-user"
          echo
        fi
        one_meg=1048576
        mem_available=$$(($$(getconf _PHYS_PAGES) * $$(getconf PAGE_SIZE) / one_meg))
        cpus_available=$$(grep -cE 'cpu[0-9]+' /proc/stat)
        disk_available=$$(df / | tail -1 | awk '{print $$4}')
        warning_resources="false"
        if (( mem_available < 4000 )) ; then
          echo
          echo -e "\033[1;33mWARNING!!!: Not enough memory available for Docker.\e[0m"
          echo "At least 4GB of memory required. You have $$(numfmt --to iec $$((mem_available * one_meg)))"
          echo
          warning_resources="true"
        fi
        if (( cpus_available < 2 )); then
          echo
          echo -e "\033[1;33mWARNING!!!: Not enough CPUS available for Docker.\e[0m"
          echo "At least 2 CPUs recommended. You have $${cpus_available}"
          echo
          warning_resources="true"
        fi
        if (( disk_available < one_meg * 10 )); then
          echo
          echo -e "\033[1;33mWARNING!!!: Not enough Disk space available for Docker.\e[0m"
          echo "At least 10 GBs recommended. You have $$(numfmt --to iec $$((disk_available * 1024 )))"
          echo
          warning_resources="true"
        fi
        if [[ $${warning_resources} == "true" ]]; then
          echo
          echo -e "\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\e[0m"
          echo "Please follow the instructions to increase amount of resources available:"
          echo "   https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#before-you-begin"
          echo
        fi
        mkdir -p /sources/logs /sources/dags /sources/plugins
        chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins}
        exec /entrypoint airflow version
    # yamllint enable rule:line-length
    environment:
      _AIRFLOW_DB_MIGRATE: 'true'
      _AIRFLOW_WWW_USER_CREATE: 'true'
      _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
      _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}
      _PIP_ADDITIONAL_REQUIREMENTS: ''
    user: "0:0"
    volumes:
      - ${AIRFLOW_PROJ_DIR:-.}:/sources

  airflow-cli:
    <<: *airflow-common
    profiles:
      - debug
    environment:
      CONNECTION_CHECK_MAX_COUNT: "0"
    # Workaround for entrypoint issue. See: https://github.com/apache/airflow/issues/16252
    command:
      - bash
      - -c
      - airflow

  # You can enable flower by adding "--profile flower" option e.g. docker-compose --profile flower up
  # or by explicitly targeted on the command line e.g. docker-compose up flower.
  # See: https://docs.docker.com/compose/profiles/
  flower:
    <<: *airflow-common
    command: celery flower
    profiles:
      - flower
    ports:
      - "5555:5555"
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:5555/"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 30s
    restart: always
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  mongo:
    <<: *common
    container_name: mongo
    build: ./docker/mongo
    ports:
      - "27017:27017"
    volumes:
      - ./data/mongo:/data/db
    healthcheck:
      test: ["CMD-SHELL", "mongo --eval 'db.adminCommand(\"ping\")'"]
      interval: 10s
      timeout: 5s
      retries: 5
    restart: always


volumes:
  pgdata:

networks:
  backend: