Skip to content

Instantly share code, notes, and snippets.

@kysnm

kysnm/airflow.cfg

Last active December 17, 2021 00:02
Show Gist options
  • Save kysnm/1fe8c23ec9cefd17c3a8289e15e888db to your computer and use it in GitHub Desktop.
Save kysnm/1fe8c23ec9cefd17c3a8289e15e888db to your computer and use it in GitHub Desktop.
Download ZIP
Raw
airflow.cfg
[core]
airflow_home = /etc/airflow
dags_folder = /home/airflow/gcs/dags
base_log_folder = /home/airflow/gcs/logs
plugins_folder = /home/airflow/gcs/plugins
remote_logging = True
remote_log_conn_id = google_cloud_default
remote_base_log_folder = gs://asia-northeast1-settings-te-xxxxxxxx-bucket/logs
executor = CeleryExecutor
dags_are_paused_at_creation = False
load_examples = False
donot_pickle = True
dagbag_import_timeout = 30
fernet_key = W5yfP4Cj6cuwOEcJteu3sd3-xxxxxxxxxxxxxxxxxxx=
parallelism = 30
dag_concurrency = 15
max_active_runs_per_dag = 15
enable_xcom_pickling = False
[webserver]
web_server_host = 0.0.0.0
web_server_port = 8080
secret_key = temporary_key
workers = 2
worker_class = sync
expose_config = True
web_server_name = settings-test
[celery]
celery_app_name = airflow.executors.celery_executor
worker_concurrency = 6
worker_log_server_port = 8793
broker_url = redis://airflow-redis-service:6379/0
result_backend = redis://airflow-redis-service:6379/0
flower_port = 5555
default_queue = default
ssl_active = False
[email]
email_backend = airflow.contrib.utils.sendgrid.send_email
[smtp]
smtp_host = localhost
smtp_starttls = True
smtp_ssl = False
smtp_user = airflow
smtp_port = 25
smtp_password = airflow
smtp_mail_from = [email protected]
[scheduler]
dag_dir_list_interval = 100
[kubernetes]
in_cluster = True
Raw
daemonset.yaml
apiVersion: v1
items:
- apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
creationTimestamp: "2019-02-27T01:27:14Z"
generation: 1
name: composer-fluentd-daemon
namespace: default
resourceVersion: "985"
selfLink: /apis/extensions/v1beta1/namespaces/default/daemonsets/composer-fluentd-daemon
uid: d03578bb-3a2e-11e9-8976-42010a9200db
spec:
revisionHistoryLimit: 10
selector:
matchLabels:
name: composer-fluentd
template:
metadata:
creationTimestamp: null
labels:
name: composer-fluentd
spec:
containers:
- env:
- name: FLUENTD_ARGS
value: --no-supervisor -q
- name: COMPOSER_ENVIRONMENT
value: settings-test
image: gcr.io/cloud-airflow-releaser/composer-fluentd:cloud_composer_service_2019-02-05-RC4
imagePullPolicy: IfNotPresent
livenessProbe:
exec:
command:
- /bin/sh
- -c
- |
LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-300}; STUCK_THRESHOLD_SECONDS=${STUCK_THRESHOLD_SECONDS:-900}; if [ ! -e /var/log/fluentd-buffers ]; then
exit 1;
fi; LAST_MODIFIED_DATE=`stat /var/log/fluentd-buffers | grep Modify | sed -r "s/Modify: (.*)/\1/"`; LAST_MODIFIED_TIMESTAMP=`date -d "$LAST_MODIFIED_DATE" +%s`; if [ `date +%s` -gt `expr $LAST_MODIFIED_TIMESTAMP + $STUCK_THRESHOLD_SECONDS` ]; then
rm -rf /var/log/fluentd-buffers;
exit 1;
fi; if [ `date +%s` -gt `expr $LAST_MODIFIED_TIMESTAMP + $LIVENESS_THRESHOLD_SECONDS` ]; then
exit 1;
fi;
failureThreshold: 3
initialDelaySeconds: 300
periodSeconds: 60
successThreshold: 1
timeoutSeconds: 1
name: composer-fluentd
resources: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /var/log
name: varlog
- mountPath: /var/lib/docker/containers
name: varlibdockercontainers
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 30
volumes:
- hostPath:
path: /var/log
type: ""
name: varlog
- hostPath:
path: /var/lib/docker/containers
type: ""
name: varlibdockercontainers
templateGeneration: 1
updateStrategy:
rollingUpdate:
maxUnavailable: 1
type: RollingUpdate
status:
currentNumberScheduled: 3
desiredNumberScheduled: 3
numberAvailable: 3
numberMisscheduled: 0
numberReady: 3
observedGeneration: 1
updatedNumberScheduled: 3
kind: List
metadata:
resourceVersion: ""
selfLink: ""
Raw
deployment.yaml
apiVersion: v1
items:
- apiVersion: extensions/v1beta1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "1"
creationTimestamp: "2019-02-26T10:08:16Z"
generation: 1
labels:
run: airflow-scheduler
name: airflow-scheduler
namespace: default
resourceVersion: "1632"
selfLink: /apis/extensions/v1beta1/namespaces/default/deployments/airflow-scheduler
uid: 6f0307e9-39ae-11e9-bf53-xxxxxxxxxxxx
spec:
progressDeadlineSeconds: 2147483647
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
run: airflow-scheduler
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 1
type: RollingUpdate
template:
metadata:
creationTimestamp: null
labels:
run: airflow-scheduler
spec:
containers:
- args:
- scheduler
env:
- name: CLOUDSDK_METRICS_ENVIRONMENT
value: 1.10.1-composer
- name: GCS_BUCKET
value: asia-northeast1-settings-te-xxxxxxxx-bucket
- name: AIRFLOW_HOME
value: /etc/airflow
- name: DAGS_FOLDER
value: /home/airflow/gcs/dags
- name: SQL_DATABASE
value: airflow-db
- name: SQL_USER
value: root
- name: SQL_PASSWORD
- name: AIRFLOW__WEBSERVER__BASE_URL
value: https://nf2d2247d4fdcdaed-tp.appspot.com
- name: AIRFLOW__CORE__SQL_ALCHEMY_CONN
value: mysql+mysqldb://$(SQL_USER):$(SQL_PASSWORD)@airflow-sqlproxy-service/$(SQL_DATABASE)
- name: GCP_PROJECT
value: xxxxxxxxx
- name: COMPOSER_LOCATION
value: asia-northeast1
- name: COMPOSER_PYTHON_VERSION
value: "2"
- name: COMPOSER_ENVIRONMENT
value: settings-test
- name: GKE_CLUSTER_NAME
value: asia-northeast1-settings-te-xxxxxxxx-gke
- name: CONTAINER_NAME
value: airflow-scheduler
image: gcr.io/cloud-airflow-releaser/airflow-worker-scheduler-1.10.1:cloud_composer_service_2019-02-05-RC4
imagePullPolicy: IfNotPresent
livenessProbe:
exec:
command:
- python
- /var/local/scheduler_checker.py
failureThreshold: 3
initialDelaySeconds: 120
periodSeconds: 60
successThreshold: 1
timeoutSeconds: 1
name: airflow-scheduler
resources: {}
securityContext:
privileged: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /etc/airflow/airflow_cfg
name: airflow-config
- mountPath: /home/airflow/gcs
name: gcsdir
- args:
- /home/airflow/gcs
env:
- name: GCS_BUCKET
value: asia-northeast1-settings-te-xxxxxxxx-bucket
image: gcr.io/cloud-airflow-releaser/gcs-syncd:cloud_composer_service_2019-02-05-RC4
imagePullPolicy: IfNotPresent
name: gcs-syncd
resources:
requests:
cpu: 10m
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /home/airflow/gcs
name: gcsdir
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 30
volumes:
- configMap:
defaultMode: 420
name: airflow-configmap
name: airflow-config
- emptyDir: {}
name: gcsdir
status:
availableReplicas: 1
conditions:
- lastTransitionTime: "2019-02-26T10:08:16Z"
lastUpdateTime: "2019-02-26T10:08:16Z"
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: "True"
type: Available
observedGeneration: 1
readyReplicas: 1
replicas: 1
updatedReplicas: 1
- apiVersion: extensions/v1beta1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "1"
creationTimestamp: "2019-02-26T10:08:16Z"
generation: 1
labels:
run: airflow-sqlproxy
name: airflow-sqlproxy
namespace: default
resourceVersion: "1636"
selfLink: /apis/extensions/v1beta1/namespaces/default/deployments/airflow-sqlproxy
uid: 6f0fd545-39ae-11e9-bf53-xxxxxxxxxxxx
spec:
progressDeadlineSeconds: 2147483647
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
run: airflow-sqlproxy
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 1
type: RollingUpdate
template:
metadata:
creationTimestamp: null
labels:
run: airflow-sqlproxy
spec:
containers:
- command:
- /cloud_sql_proxy
- -instances=nf2d2247d4fdcdaed-tp:asia-northeast1:asia-northeast1-settings-te-xxxxxxxx-sql=tcp:0.0.0.0:3306
image: gcr.io/cloudsql-docker/gce-proxy:1.09
imagePullPolicy: IfNotPresent
livenessProbe:
exec:
command:
- /bin/sh
- -c
- netstat -tlnp | grep -i cloud_sql_proxy
failureThreshold: 3
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: airflow-sqlproxy
ports:
- containerPort: 3306
protocol: TCP
resources: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /etc/ssl/certs
name: ssl-certs
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 30
volumes:
- hostPath:
path: /etc/ssl/certs
type: ""
name: ssl-certs
status:
availableReplicas: 1
conditions:
- lastTransitionTime: "2019-02-26T10:08:16Z"
lastUpdateTime: "2019-02-26T10:08:16Z"
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: "True"
type: Available
observedGeneration: 1
readyReplicas: 1
replicas: 1
updatedReplicas: 1
- apiVersion: extensions/v1beta1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "1"
creationTimestamp: "2019-02-26T10:08:16Z"
generation: 1
labels:
run: airflow-worker
name: airflow-worker
namespace: default
resourceVersion: "1698"
selfLink: /apis/extensions/v1beta1/namespaces/default/deployments/airflow-worker
uid: 6f196e83-39ae-11e9-bf53-xxxxxxxxxxxx
spec:
progressDeadlineSeconds: 2147483647
replicas: 3
revisionHistoryLimit: 10
selector:
matchLabels:
run: airflow-worker
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 1
type: RollingUpdate
template:
metadata:
creationTimestamp: null
labels:
run: airflow-worker
spec:
containers:
- args:
- worker
env:
- name: CLOUDSDK_METRICS_ENVIRONMENT
value: 1.10.1-composer
- name: GCS_BUCKET
value: asia-northeast1-settings-te-xxxxxxxx-bucket
- name: GCP_PROJECT
value: xxxxxxxxx
- name: COMPOSER_LOCATION
value: asia-northeast1
- name: COMPOSER_GKE_ZONE
value: asia-northeast1-c
- name: COMPOSER_GKE_NAME
value: asia-northeast1-settings-te-xxxxxxxx-gke
- name: COMPOSER_PYTHON_VERSION
value: "2"
- name: COMPOSER_ENVIRONMENT
value: settings-test
- name: AIRFLOW_HOME
value: /etc/airflow
- name: DAGS_FOLDER
value: /home/airflow/gcs/dags
- name: SQL_DATABASE
value: airflow-db
- name: SQL_USER
value: root
- name: SQL_PASSWORD
- name: AIRFLOW__WEBSERVER__BASE_URL
value: https://nf2d2247d4fdcdaed-tp.appspot.com
- name: AIRFLOW__CORE__SQL_ALCHEMY_CONN
value: mysql+mysqldb://$(SQL_USER):$(SQL_PASSWORD)@airflow-sqlproxy-service/$(SQL_DATABASE)
- name: C_FORCE_ROOT
value: "TRUE"
image: gcr.io/cloud-airflow-releaser/airflow-worker-scheduler-1.10.1:cloud_composer_service_2019-02-05-RC4
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- pkill ForkPoolWorker
- ;
- kill
- "1"
livenessProbe:
exec:
command:
- python
- /var/local/worker_checker.py
failureThreshold: 3
initialDelaySeconds: 120
periodSeconds: 60
successThreshold: 1
timeoutSeconds: 1
name: airflow-worker
ports:
- containerPort: 8793
protocol: TCP
resources: {}
securityContext:
privileged: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /etc/airflow/airflow_cfg
name: airflow-config
- mountPath: /home/airflow/gcs
name: gcsdir
- args:
- /home/airflow/gcs
env:
- name: GCS_BUCKET
value: asia-northeast1-settings-te-xxxxxxxx-bucket
image: gcr.io/cloud-airflow-releaser/gcs-syncd:cloud_composer_service_2019-02-05-RC4
imagePullPolicy: IfNotPresent
name: gcs-syncd
resources:
requests:
cpu: 10m
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /home/airflow/gcs
name: gcsdir
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 30
volumes:
- configMap:
defaultMode: 420
name: airflow-configmap
name: airflow-config
- emptyDir: {}
name: gcsdir
status:
availableReplicas: 3
conditions:
- lastTransitionTime: "2019-02-26T10:10:30Z"
lastUpdateTime: "2019-02-26T10:10:30Z"
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: "True"
type: Available
observedGeneration: 1
readyReplicas: 3
replicas: 3
updatedReplicas: 3
kind: List
metadata:
resourceVersion: ""
selfLink: ""
Raw
service.yaml
apiVersion: v1
items:
- apiVersion: v1
kind: Service
metadata:
creationTimestamp: "2019-02-26T10:08:16Z"
labels:
run: airflow-redis
name: airflow-redis-service
namespace: default
resourceVersion: "1264"
selfLink: /api/v1/namespaces/default/services/airflow-redis-service
uid: 6f326dea-39ae-11e9-bf53-xxxxxxxxxxxx
spec:
clusterIP: xxx.xxx.xxx.xxx
ports:
- port: 6379
protocol: TCP
targetPort: 6379
selector:
run: airflow-redis
sessionAffinity: None
type: ClusterIP
status:
loadBalancer: {}
- apiVersion: v1
kind: Service
metadata:
creationTimestamp: "2019-02-26T10:08:16Z"
labels:
run: airflow-sqlproxy
name: airflow-sqlproxy-service
namespace: default
resourceVersion: "1278"
selfLink: /api/v1/namespaces/default/services/airflow-sqlproxy-service
uid: 6f454453-39ae-11e9-bf53-xxxxxxxxxxxx
spec:
clusterIP: xxx.xxx.xxx.xxx
ports:
- port: 3306
protocol: TCP
targetPort: 3306
selector:
run: airflow-sqlproxy
sessionAffinity: None
type: ClusterIP
status:
loadBalancer: {}
- apiVersion: v1
kind: Service
metadata:
creationTimestamp: "2019-02-26T10:01:05Z"
labels:
component: apiserver
provider: kubernetes
name: kubernetes
namespace: default
resourceVersion: "6"
selfLink: /api/v1/namespaces/default/services/kubernetes
uid: 6e81176c-39ad-11e9-9eef-42010a920014
spec:
clusterIP: xxx.xxx.xxx.xxx
ports:
- name: https
port: 443
protocol: TCP
targetPort: 443
sessionAffinity: None
type: ClusterIP
status:
loadBalancer: {}
kind: List
metadata:
resourceVersion: ""
selfLink: ""
Raw
statefulset.yaml
apiVersion: v1
items:
- apiVersion: apps/v1
kind: StatefulSet
metadata:
creationTimestamp: "2019-02-27T02:32:11Z"
generation: 1
labels:
run: airflow-redis
name: airflow-redis
namespace: default
resourceVersion: "1643"
selfLink: /apis/apps/v1/namespaces/default/statefulsets/airflow-redis
uid: e2904c7f-3a37-11e9-b2b8-42010a92013a
spec:
podManagementPolicy: OrderedReady
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
run: airflow-redis
serviceName: airflow-redis-service
template:
metadata:
creationTimestamp: null
labels:
run: airflow-redis
spec:
containers:
- command:
- redis-server
- --
- save
- "60"
- "1"
image: gcr.io/cloud-airflow-releaser/redis-5.0.3:cloud_composer_service_2019-02-05-RC4
imagePullPolicy: IfNotPresent
name: airflow-redis
ports:
- containerPort: 6379
protocol: TCP
resources: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /data
name: airflow-redis-volume
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 60
updateStrategy:
type: RollingUpdate
volumeClaimTemplates:
- metadata:
creationTimestamp: null
name: airflow-redis-volume
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2Gi
status:
phase: Pending
status:
collisionCount: 0
currentReplicas: 1
currentRevision: airflow-redis-55bb5cc5cf
observedGeneration: 1
readyReplicas: 1
replicas: 1
updateRevision: airflow-redis-55bb5cc5cf
updatedReplicas: 1
kind: List
metadata:
resourceVersion: ""
selfLink: ""
@kysnm
Copy link
Author

kysnm commented Feb 27, 2019

  • 以下のコマンドで出力
kubectl get Deployment -o yaml > deployment.yml
kubectl get Service -o yaml > service.yml
kubectl get Daemonset -o yaml > daemonset.yml
kubectl get Statefulset -o yaml > statefullset.yml

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment