Skip to content

Instantly share code, notes, and snippets.

@duyet

duyet/airflow-dag-example.py

Last active February 25, 2020 17:12
Show Gist options
  • Save duyet/30b94f3b02b64b6dd4abc071d6665a86 to your computer and use it in GitHub Desktop.
Save duyet/30b94f3b02b64b6dd4abc071d6665a86 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. duyet renamed this gist Feb 25, 2020. 1 changed file with 1 addition and 6 deletions.
    7 changes: 1 addition & 6 deletions airflow-dag.py → airflow-dag-example.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,13 +1,8 @@
    import logging as logger
    from datetime import datetime

    import boto3
    import pandas as pd

    from airflow.models import DAG, Variable
    from airflow.models import DAG
    from airflow.operators.dummy_operator import DummyOperator
    from airflow.operators.python_operator import PythonOperator
    from utils import update_context

    DAG_OWNER = '[email protected]'
    DAG_ID = 'dag_name'
  2. duyet revised this gist Feb 25, 2020. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions airflow-dag.py
    View file
    Original file line number Diff line number Diff line change
    @@ -10,12 +10,12 @@
    from utils import update_context

    DAG_OWNER = '[email protected]'
    DAG_ID = 'cronjob_load_phone_model'
    DAG_ID = 'dag_name'
    SCHEDULE_INTERVAL = '@weekly'

    default_args = {
    'owner': DAG_OWNER,
    'start_date': datetime(2019, 10, 1),
    'start_date': datetime(2020, 1, 1),
    'executor_config': {
    'KubernetesExecutor': {
    'request_memory': '512Mi',
  3. duyet created this gist Feb 25, 2020.
    50 changes: 50 additions & 0 deletions airflow-dag.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,50 @@
    import logging as logger
    from datetime import datetime

    import boto3
    import pandas as pd

    from airflow.models import DAG, Variable
    from airflow.operators.dummy_operator import DummyOperator
    from airflow.operators.python_operator import PythonOperator
    from utils import update_context

    DAG_OWNER = '[email protected]'
    DAG_ID = 'cronjob_load_phone_model'
    SCHEDULE_INTERVAL = '@weekly'

    default_args = {
    'owner': DAG_OWNER,
    'start_date': datetime(2019, 10, 1),
    'executor_config': {
    'KubernetesExecutor': {
    'request_memory': '512Mi',
    'limit_memory': '1Gi',
    'request_cpu': '500m',
    'limit_cpu': '1000m'
    }
    },
    'retries': 3,
    }


    def python_task_callable(**kwargs):
    pass


    def create_dag():
    dag = DAG(DAG_ID, default_args=default_args,
    schedule_interval=SCHEDULE_INTERVAL)

    start = DummyOperator(task_id='start', dag=dag)
    python_task = PythonOperator(task_id='python_task',
    provide_context=True,
    python_callable=python_task_callable,
    dag=dag)
    complete = DummyOperator(task_id='complete', dag=dag)

    start >> python_task >> complete

    return dag

    globals()[DAG_ID] = create_dag()