airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fo...@apache.org
Subject [44/50] incubator-airflow git commit: [AIRFLOW-2411] add dataproc_jars to templated_fields
Date Fri, 04 May 2018 07:02:40 GMT
[AIRFLOW-2411] add dataproc_jars to templated_fields

This commit makes it possible to use jinja
templates when passing
JAR file URIs to the DataProc operators that
require JAR files,
specifically the DataProc Hive, Pig, SparkSql,
Spark, Hadoop and
PySpark operators.

Closes #3305 from mchalek/template-dataproc-jars


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/c5fa8cd4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/c5fa8cd4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/c5fa8cd4

Branch: refs/heads/v1-10-test
Commit: c5fa8cd411ca67889eafd109988d2472ccfbba10
Parents: 6b3f6ce
Author: Kevin McHale <mchalek@gmail.com>
Authored: Thu May 3 22:55:10 2018 +0200
Committer: Fokko Driesprong <fokkodriesprong@godatadriven.com>
Committed: Thu May 3 22:55:10 2018 +0200

----------------------------------------------------------------------
 airflow/contrib/operators/dataproc_operator.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/c5fa8cd4/airflow/contrib/operators/dataproc_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/dataproc_operator.py b/airflow/contrib/operators/dataproc_operator.py
index 56ebb91..ad0aa09 100644
--- a/airflow/contrib/operators/dataproc_operator.py
+++ b/airflow/contrib/operators/dataproc_operator.py
@@ -481,7 +481,7 @@ class DataProcPigOperator(BaseOperator):
     :param region: The specified region where the dataproc cluster is created.
     :type region: string
     """
-    template_fields = ['query', 'variables', 'job_name', 'cluster_name']
+    template_fields = ['query', 'variables', 'job_name', 'cluster_name', 'dataproc_jars']
     template_ext = ('.pg', '.pig',)
     ui_color = '#0273d4'
 
@@ -561,7 +561,7 @@ class DataProcHiveOperator(BaseOperator):
     :param region: The specified region where the dataproc cluster is created.
     :type region: string
     """
-    template_fields = ['query', 'variables', 'job_name', 'cluster_name']
+    template_fields = ['query', 'variables', 'job_name', 'cluster_name', 'dataproc_jars']
     template_ext = ('.q',)
     ui_color = '#0273d4'
 
@@ -642,7 +642,7 @@ class DataProcSparkSqlOperator(BaseOperator):
     :param region: The specified region where the dataproc cluster is created.
     :type region: string
     """
-    template_fields = ['query', 'variables', 'job_name', 'cluster_name']
+    template_fields = ['query', 'variables', 'job_name', 'cluster_name', 'dataproc_jars']
     template_ext = ('.q',)
     ui_color = '#0273d4'
 
@@ -731,7 +731,7 @@ class DataProcSparkOperator(BaseOperator):
     :type region: string
     """
 
-    template_fields = ['arguments', 'job_name', 'cluster_name']
+    template_fields = ['arguments', 'job_name', 'cluster_name', 'dataproc_jars']
     ui_color = '#0273d4'
 
     @apply_defaults
@@ -821,7 +821,7 @@ class DataProcHadoopOperator(BaseOperator):
     :type region: string
     """
 
-    template_fields = ['arguments', 'job_name', 'cluster_name']
+    template_fields = ['arguments', 'job_name', 'cluster_name', 'dataproc_jars']
     ui_color = '#0273d4'
 
     @apply_defaults
@@ -911,7 +911,7 @@ class DataProcPySparkOperator(BaseOperator):
     :type region: string
     """
 
-    template_fields = ['arguments', 'job_name', 'cluster_name']
+    template_fields = ['arguments', 'job_name', 'cluster_name', 'dataproc_jars']
     ui_color = '#0273d4'
 
     @staticmethod


Mime
View raw message