version: "3.8"

# This should give a high level idea of the approach. 
# The complete solution is too complex and involves multiple internal microservices.
# I have tried to capture core things to consider while some else needs to achieve Docker swarm based auto scalout of workers.

services:
  webserver:
    image: customized/airflow:prod
    environment:
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql://username:password@db-host:5432/airflow-db
      - AIRFLOW__WEBSERVER__BASE_URL=http://localhost:8080/airflow
      - AIRFLOW__CELERY__DEFAULT_QUEUE=${HOSTNAME:-airflow}
      - AIRFLOW__CELERY__BROKER_URL=pyamqp://username:pass@rabbitmq:5672/
      - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://username:password@db-host:5432/airflow-db
    volumes:
      - workflow_logs:/opt/airflow/logs
      - /var/run/docker.sock:/var/run/docker.sock
      - ${DOCKER_BIN_PATH}:/usr/bin/docker
    ports:
      - 8080
    command: ["webserver", "-p", "8080"]
    networks:
      - myNetwork
    deploy:
      placement:
        constraints:
          - "node.role==manager"
  
  scheduler:
    image: customized/airflow:prod
    environment:
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql://username:password@db-host:5432/airflow-db
      - AIRFLOW__CELERY__BROKER_URL=pyamqp://username:pass@rabbitmq:5672/
      - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://username:password@db-host:5432/airflow-db
      - AIRFLOW__CELERY__DEFAULT_QUEUE=${HOSTNAME:-airflow}
    volumes:
      - workflow_logs:/opt/airflow/logs
    command: ["scheduler"]
    networks:
      - myNetwork
    deploy:
      placement:
        constraints:
          - "node.role==manager"
  
  worker:
    image: customized/airflow:prod
    environment:
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql://username:password@db-host:5432/airflow-db
      - AIRFLOW__CELERY__BROKER_URL=pyamqp://username:pass@rabbitmq:5672/
      - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://username:password@db-host:5432/airflow-db
      - AIRFLOW__CELERY__DEFAULT_QUEUE=${HOSTNAME:-airflow}
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ${DOCKER_BIN_PATH}:/usr/bin/docker
    ports:
      - 8793
    hostname: '{{.Node.Hostname}}'
    command: worker
    networks:
      - myNetwork
    deploy:
      replicas: 5
      placement:
        max_replicas_per_node: 1
        constraints:
          - "node.labels.type==celery-worker"