我想更改dag_concurrency
特定Airflow DAG 的参数.似乎有一个全局dag_concurrency
参数airflow.cfg
但可以为不同的DAG设置不同的值吗?
我尝试在SSHExecuteOperator
任务中的DAG代码中添加并发参数,但并发值仍显示DAG详细信息中的标准参数(16).
from airflow import DAG
from datetime import datetime, timedelta
from airflow.contrib.hooks.ssh_hook import SSHHook
from airflow.contrib.operators.ssh_execute_operator import SSHExecuteOperator
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': datetime.now(),
'email': ['exceptions@airflow.com'],
'email_on_failure': True,
'retries': 0
}
#server must be changed to point to the correct environment
sshHookEtl = SSHHook(conn_id='SSH__airflow@myserver')
with DAG(
'ed_data_quality_20min-v1.6.6',
default_args=default_args,
schedule_interval="0,20,40 * * * *",
dagrun_timeout=timedelta(hours=24)) as dag:
(
dag
>> SSHExecuteOperator(
task_id='run_remote_ed_data_quality_20min',
bash_command='bash /opt/scripts/shell/EXEC_ED_DATA_QUALITY_20MIN.sh ',
ssh_hook=sshHookEtl,
retries=0,
concurrency=1,
dag=dag)
)
这是DAG的详细信息