airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From criccom...@apache.org
Subject [1/3] incubator-airflow git commit: [AIRFLOW-119] Fix Template not found error and default tags for Qubole operator
Date Fri, 20 May 2016 15:06:05 GMT
Repository: incubator-airflow
Updated Branches:
  refs/heads/master ccfc4c8c1 -> 5898de186


[AIRFLOW-119] Fix Template not found error and default tags for Qubole operator


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/d08c1332
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/d08c1332
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/d08c1332

Branch: refs/heads/master
Commit: d08c1332a2c7b03b7c89486ba5a03b55fbf1cd9f
Parents: 6f4696b
Author: Sumit Maheshwari <sumitm@qubole.com>
Authored: Tue May 17 10:45:44 2016 +0530
Committer: Sumit Maheshwari <sumitm@qubole.com>
Committed: Tue May 17 10:45:44 2016 +0530

----------------------------------------------------------------------
 airflow/contrib/hooks/qubole_hook.py         | 10 +++++++---
 airflow/contrib/operators/qubole_operator.py |  2 +-
 2 files changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d08c1332/airflow/contrib/hooks/qubole_hook.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/hooks/qubole_hook.py b/airflow/contrib/hooks/qubole_hook.py
index 7308b64..7de6a14 100755
--- a/airflow/contrib/hooks/qubole_hook.py
+++ b/airflow/contrib/hooks/qubole_hook.py
@@ -62,12 +62,11 @@ class QuboleHook(BaseHook):
         self.task_id = kwargs['task_id']
         self.dag_id = kwargs['dag'].dag_id
         self.kwargs = kwargs
-        self.args = self.create_cmd_args()
         self.cls = COMMAND_CLASSES[self.kwargs['command_type']]
         self.cmd = None
 
     def execute(self, context):
-        args = self.cls.parse(self.args)
+        args = self.cls.parse(self.create_cmd_args(context))
         self.cmd = self.cls.create(**args)
         context['task_instance'].xcom_push(key='qbol_cmd_id', value=self.cmd.id)
         logging.info("Qubole command created with Id: {0} and Status: {1}".format(str(self.cmd.id),
self.cmd.status))
@@ -143,10 +142,11 @@ class QuboleHook(BaseHook):
             cmd_id = ti.xcom_pull(key="qbol_cmd_id", task_ids=self.task_id)
         Command.get_jobs_id(self.cls, cmd_id)
 
-    def create_cmd_args(self):
+    def create_cmd_args(self, context):
         args = []
         cmd_type = self.kwargs['command_type']
         inplace_args = None
+        tags = set([self.dag_id, self.task_id, context['run_id']])
 
         for k,v in self.kwargs.items():
             if k in COMMAND_ARGS[cmd_type]:
@@ -154,12 +154,16 @@ class QuboleHook(BaseHook):
                     args.append("--{0}={1}".format(k.replace('_', '-'),v))
                 elif k in POSITIONAL_ARGS:
                     inplace_args = v
+                elif k == 'tags':
+                    tags.add(v)
                 else:
                     args.append("--{0}={1}".format(k,v))
 
             if k == 'notify' and v is True:
                 args.append("--notify")
 
+        args.append("--tags={0}".format(','.join(tags)))
+
         if inplace_args is not None:
             if cmd_type == 'hadoopcmd':
                 args += inplace_args.split(' ', 1)

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d08c1332/airflow/contrib/operators/qubole_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/qubole_operator.py b/airflow/contrib/operators/qubole_operator.py
index 2ed94e1..1cec673 100755
--- a/airflow/contrib/operators/qubole_operator.py
+++ b/airflow/contrib/operators/qubole_operator.py
@@ -95,7 +95,7 @@ class QuboleOperator(BaseOperator):
     """
 
     template_fields = ('query', 'script_location', 'sub_command', 'script', 'files', 'archives',
'program', 'cmdline', 'sql', 'where_clause', 'extract_query', 'boundary_query', 'macros',
'tags', 'name')
-    template_ext = ('.hql', '.sql', '.sh', '.bash', '.pig')
+    template_ext = ('.txt')
     ui_color = '#3064A1'
     ui_fgcolor = '#fff'
 


Mime
View raw message