airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maximebeauche...@apache.org
Subject [5/7] incubator-airflow-site git commit: New version of the docs
Date Thu, 11 Aug 2016 22:59:44 GMT
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/airflow/operators/sensors.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/sensors.html b/_modules/airflow/operators/sensors.html
index 65a0587..3b716d3 100644
--- a/_modules/airflow/operators/sensors.html
+++ b/_modules/airflow/operators/sensors.html
@@ -155,6 +155,8 @@
 <span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
 <span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="nb">basestring</span>
+
 <span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlparse</span>
@@ -229,6 +231,7 @@
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;sql&#39;</span><span class="p">,)</span>
     <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.hql&#39;</span><span class="p">,</span> <span class="s1">&#39;.sql&#39;</span><span class="p">,)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#7c7287&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn_id</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
@@ -272,6 +275,7 @@
 <span class="sd">    :type mysql_conn_id: str</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;partition_name&#39;</span><span class="p">,</span> <span class="s1">&#39;table&#39;</span><span class="p">,</span> <span class="s1">&#39;schema&#39;</span><span class="p">)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#8da7be&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
@@ -327,6 +331,7 @@
 <span class="sd">        or execution_date_fn can be passed to ExternalTaskSensor, but not both.</span>
 <span class="sd">    :type execution_date_fn: callable</span>
 <span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#19647e&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
@@ -382,10 +387,11 @@
 
 <span class="sd">    :param partition_names: List of fully qualified names of the</span>
 <span class="sd">        partitions to wait for. A fully qualified name is of the</span>
-<span class="sd">        form schema.table/pk1=pv1/pk2=pv2, for example,</span>
+<span class="sd">        form ``schema.table/pk1=pv1/pk2=pv2``, for example,</span>
 <span class="sd">        default.users/ds=2016-01-01. This is passed as is to the metastore</span>
-<span class="sd">        Thrift client &quot;get_partitions_by_name&quot; method. Note that</span>
-<span class="sd">        you cannot use logical operators as in HivePartitionSensor.</span>
+<span class="sd">        Thrift client ``get_partitions_by_name`` method. Note that</span>
+<span class="sd">        you cannot use logical or comparison operators as in</span>
+<span class="sd">        HivePartitionSensor.</span>
 <span class="sd">    :type partition_names: list of strings</span>
 <span class="sd">    :param metastore_conn_id: reference to the metastore thrift service</span>
 <span class="sd">        connection id</span>
@@ -393,6 +399,7 @@
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;partition_names&#39;</span><span class="p">,</span> <span class="p">)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#8d99ae&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
@@ -408,9 +415,6 @@
         <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">partition_names</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
             <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;partition_names must be an array of strings&#39;</span><span class="p">)</span>
 
-        <span class="k">for</span> <span class="n">partition_name</span> <span class="ow">in</span> <span class="n">partition_names</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">parse_partition_name</span><span class="p">(</span><span class="n">partition_name</span><span class="p">)</span>
-
         <span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span> <span class="o">=</span> <span class="n">metastore_conn_id</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span> <span class="o">=</span> <span class="n">partition_names</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">=</span> <span class="mi">0</span>
@@ -452,7 +456,7 @@
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Waits for a partition to show up in Hive.</span>
 
-<span class="sd">    Note: Because @partition supports general logical operators, it</span>
+<span class="sd">    Note: Because ``partition`` supports general logical operators, it</span>
 <span class="sd">    can be inefficient. Consider using NamedHivePartitionSensor instead if</span>
 <span class="sd">    you don&#39;t need the full flexibility of HivePartitionSensor.</span>
 
@@ -460,15 +464,16 @@
 <span class="sd">        notation (my_database.my_table)</span>
 <span class="sd">    :type table: string</span>
 <span class="sd">    :param partition: The partition clause to wait for. This is passed as</span>
-<span class="sd">        is to the metastore Thrift client &quot;get_partitions_by_filter&quot; method,</span>
-<span class="sd">        and apparently supports SQL like notation as in `ds=&#39;2015-01-01&#39;</span>
-<span class="sd">        AND type=&#39;value&#39;` and &gt; &lt; sings as in &quot;ds&gt;=2015-01-01&quot;</span>
+<span class="sd">        is to the metastore Thrift client ``get_partitions_by_filter`` method,</span>
+<span class="sd">        and apparently supports SQL like notation as in ``ds=&#39;2015-01-01&#39;</span>
+<span class="sd">        AND type=&#39;value&#39;`` and comparison operators as in ``&quot;ds&gt;=2015-01-01&quot;``</span>
 <span class="sd">    :type partition: string</span>
 <span class="sd">    :param metastore_conn_id: reference to the metastore thrift service</span>
 <span class="sd">        connection id</span>
 <span class="sd">    :type metastore_conn_id: str</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;schema&#39;</span><span class="p">,</span> <span class="s1">&#39;table&#39;</span><span class="p">,</span> <span class="s1">&#39;partition&#39;</span><span class="p">,)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#2b2d42&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
@@ -505,6 +510,7 @@
 <span class="sd">    Waits for a file or folder to land in HDFS</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;filepath&#39;</span><span class="p">,)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#4d9de0&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/bash_operator.html
----------------------------------------------------------------------
diff --git a/_modules/bash_operator.html b/_modules/bash_operator.html
index d38a3f8..9877b4a 100644
--- a/_modules/bash_operator.html
+++ b/_modules/bash_operator.html
@@ -170,6 +170,9 @@
 <span class="sd">    :param bash_command: The command, set of commands or reference to a</span>
 <span class="sd">        bash script (must be &#39;.sh&#39;) to be executed.</span>
 <span class="sd">    :type bash_command: string</span>
+<span class="sd">    :param xcom_push: If xcom_push is True, the last line written to stdout</span>
+<span class="sd">        will also be pushed to an XCom when the bash command completes.</span>
+<span class="sd">    :type xcom_push: bool</span>
 <span class="sd">    :param env: If env is not None, it must be a mapping that defines the</span>
 <span class="sd">        environment variables for the new process; these are used instead</span>
 <span class="sd">        of inheriting the current process environment, which is the default</span>
@@ -189,10 +192,7 @@
             <span class="n">env</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">output_encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">,</span>
             <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
-        <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        If xcom_push is True, the last line written to stdout will also</span>
-<span class="sd">        be pushed to an XCom when the bash command completes.</span>
-<span class="sd">        &quot;&quot;&quot;</span>
+
         <span class="nb">super</span><span class="p">(</span><span class="n">BashOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">bash_command</span> <span class="o">=</span> <span class="n">bash_command</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">env</span> <span class="o">=</span> <span class="n">env</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/bigquery_hook.html
----------------------------------------------------------------------
diff --git a/_modules/bigquery_hook.html b/_modules/bigquery_hook.html
index 9478587..1f8170a 100644
--- a/_modules/bigquery_hook.html
+++ b/_modules/bigquery_hook.html
@@ -157,6 +157,9 @@
 <span class="sd">implementation for BigQuery.</span>
 <span class="sd">&quot;&quot;&quot;</span>
 
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">range</span>
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="nb">basestring</span>
+
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">time</span>
 
@@ -440,10 +443,10 @@
                 <span class="bp">self</span><span class="o">.</span><span class="n">_split_project_dataset_table_input</span><span class="p">(</span>
                     <span class="s1">&#39;source_project_dataset_table&#39;</span><span class="p">,</span> <span class="n">source_project_dataset_table</span><span class="p">)</span>
             <span class="n">source_project_dataset_tables_fixup</span><span class="o">.</span><span class="n">append</span><span class="p">({</span>
-                    <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="n">source_project</span><span class="p">,</span>
-                    <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">source_dataset</span><span class="p">,</span>
-                    <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">source_table</span>
-                <span class="p">})</span>
+                <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="n">source_project</span><span class="p">,</span>
+                <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">source_dataset</span><span class="p">,</span>
+                <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">source_table</span>
+            <span class="p">})</span>
 
         <span class="k">assert</span> <span class="mi">3</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">destination_project_dataset_table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)),</span> <span class="p">(</span>
             <span class="s1">&#39;Expected destination_project_dataset_table in the format of &#39;</span>
@@ -652,29 +655,34 @@
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="c1"># check to see if the table exists</span>
         <span class="n">table_id</span> <span class="o">=</span> <span class="n">table_resource</span><span class="p">[</span><span class="s1">&#39;tableReference&#39;</span><span class="p">][</span><span class="s1">&#39;tableId&#39;</span><span class="p">]</span>
-        <span class="n">table_exists</span> <span class="o">=</span> <span class="bp">False</span>
         <span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
         <span class="n">tables_list_resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
                                                       <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
-        <span class="k">if</span> <span class="s1">&#39;tables&#39;</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="p">:</span>
-            <span class="k">for</span> <span class="n">table</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="p">[</span><span class="s1">&#39;tables&#39;</span><span class="p">]:</span>
+        <span class="k">while</span> <span class="bp">True</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">table</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;tables&#39;</span><span class="p">,</span> <span class="p">[]):</span>
                 <span class="k">if</span> <span class="n">table</span><span class="p">[</span><span class="s1">&#39;tableReference&#39;</span><span class="p">][</span><span class="s1">&#39;tableId&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="n">table_id</span><span class="p">:</span>
-                    <span class="n">table_exists</span> <span class="o">=</span> <span class="bp">True</span>
-                    <span class="k">break</span>
-
-        <span class="c1"># do update if table exists</span>
-        <span class="k">if</span> <span class="n">table_exists</span><span class="p">:</span>
-            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> exists, updating.&#39;</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
-            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
-                                                <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
-                                                <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">,</span>
-                                                <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
-        <span class="c1"># do insert if table does not exist</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> does not exist. creating.&#39;</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
-            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
-                                                <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
-                                                <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+                    <span class="c1"># found the table, do update</span>
+                    <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> exists, updating.&#39;</span><span class="p">,</span>
+                                 <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+                    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+                                                        <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+                                                        <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">,</span>
+                                                        <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="c1"># If there is a next page, we need to check the next page.</span>
+            <span class="k">if</span> <span class="s1">&#39;nextPageToken&#39;</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="p">:</span>
+                <span class="n">tables_list_resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span>\
+                    <span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+                          <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+                          <span class="n">pageToken</span><span class="o">=</span><span class="n">tables_list_resp</span><span class="p">[</span><span class="s1">&#39;nextPageToken&#39;</span><span class="p">])</span>\
+                    <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="c1"># If there is no next page, then the table doesn&#39;t exist.</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="c1"># do insert</span>
+                <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> does not exist. creating.&#39;</span><span class="p">,</span>
+                             <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+                <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+                                                    <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+                                                    <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
 
     <span class="k">def</span> <span class="nf">run_grant_dataset_view_access</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                                       <span class="n">source_dataset</span><span class="p">,</span>
@@ -851,7 +859,7 @@
         <span class="k">if</span> <span class="n">size</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
             <span class="n">size</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">arraysize</span>
         <span class="n">result</span> <span class="o">=</span> <span class="p">[]</span>
-        <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">xrange</span><span class="p">(</span><span class="n">size</span><span class="p">):</span>
+        <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">size</span><span class="p">):</span>
             <span class="n">one</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span>
             <span class="k">if</span> <span class="n">one</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
                 <span class="k">break</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/cloudant_hook.html
----------------------------------------------------------------------
diff --git a/_modules/cloudant_hook.html b/_modules/cloudant_hook.html
index b238080..b7747b4 100644
--- a/_modules/cloudant_hook.html
+++ b/_modules/cloudant_hook.html
@@ -150,9 +150,11 @@
 <span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
 <span class="c1"># See the License for the specific language governing permissions and</span>
 <span class="c1"># limitations under the License.</span>
-<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="nb">unicode</span>
 
 <span class="kn">import</span> <span class="nn">cloudant</span>
+<span class="kn">import</span> <span class="nn">logging</span>
 
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
 <span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/dagrun_operator.html
----------------------------------------------------------------------
diff --git a/_modules/dagrun_operator.html b/_modules/dagrun_operator.html
index c6725bb..1d147e0 100644
--- a/_modules/dagrun_operator.html
+++ b/_modules/dagrun_operator.html
@@ -185,6 +185,7 @@
     <span class="n">template_fields</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>
     <span class="n">template_ext</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>
     <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#ffefeb&#39;</span>
+
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
             <span class="bp">self</span><span class="p">,</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/dbapi_hook.html
----------------------------------------------------------------------
diff --git a/_modules/dbapi_hook.html b/_modules/dbapi_hook.html
index e608a23..604dddd 100644
--- a/_modules/dbapi_hook.html
+++ b/_modules/dbapi_hook.html
@@ -195,9 +195,8 @@
             <span class="n">username</span><span class="o">=</span><span class="n">db</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
             <span class="n">schema</span><span class="o">=</span><span class="n">db</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span></div>
 
-
 <div class="viewcode-block" id="DbApiHook.get_pandas_df"><a class="viewcode-back" href="../code.html#airflow.hooks.DbApiHook.get_pandas_df">[docs]</a>    <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
-        <span class="sd">&#39;&#39;&#39;</span>
+        <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Executes the sql and returns a pandas dataframe</span>
 
 <span class="sd">        :param sql: the sql statement to be executed (str) or a list of</span>
@@ -205,7 +204,7 @@
 <span class="sd">        :type sql: str or list</span>
 <span class="sd">        :param parameters: The parameters to render the SQL query with.</span>
 <span class="sd">        :type parameters: mapping or iterable</span>
-<span class="sd">        &#39;&#39;&#39;</span>
+<span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&lt;</span> <span class="mi">3</span><span class="p">:</span>
             <span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
         <span class="kn">import</span> <span class="nn">pandas.io.sql</span> <span class="kn">as</span> <span class="nn">psql</span>
@@ -215,7 +214,7 @@
         <span class="k">return</span> <span class="n">df</span></div>
 
 <div class="viewcode-block" id="DbApiHook.get_records"><a class="viewcode-back" href="../code.html#airflow.hooks.DbApiHook.get_records">[docs]</a>    <span class="k">def</span> <span class="nf">get_records</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
-        <span class="sd">&#39;&#39;&#39;</span>
+        <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Executes the sql and returns a set of records.</span>
 
 <span class="sd">        :param sql: the sql statement to be executed (str) or a list of</span>
@@ -223,7 +222,7 @@
 <span class="sd">        :type sql: str or list</span>
 <span class="sd">        :param parameters: The parameters to render the SQL query with.</span>
 <span class="sd">        :type parameters: mapping or iterable</span>
-<span class="sd">        &#39;&#39;&#39;</span>
+<span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&lt;</span> <span class="mi">3</span><span class="p">:</span>
             <span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
@@ -238,7 +237,7 @@
         <span class="k">return</span> <span class="n">rows</span></div>
 
 <div class="viewcode-block" id="DbApiHook.get_first"><a class="viewcode-back" href="../code.html#airflow.hooks.DbApiHook.get_first">[docs]</a>    <span class="k">def</span> <span class="nf">get_first</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
-        <span class="sd">&#39;&#39;&#39;</span>
+        <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Executes the sql and returns the first resulting row.</span>
 
 <span class="sd">        :param sql: the sql statement to be executed (str) or a list of</span>
@@ -246,7 +245,7 @@
 <span class="sd">        :type sql: str or list</span>
 <span class="sd">        :param parameters: The parameters to render the SQL query with.</span>
 <span class="sd">        :type parameters: mapping or iterable</span>
-<span class="sd">        &#39;&#39;&#39;</span>
+<span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&lt;</span> <span class="mi">3</span><span class="p">:</span>
             <span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
@@ -280,7 +279,7 @@
             <span class="n">sql</span> <span class="o">=</span> <span class="p">[</span><span class="n">sql</span><span class="p">]</span>
 
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">supports_autocommit</span><span class="p">:</span>
-           <span class="bp">self</span><span class="o">.</span><span class="n">set_autocommit</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">set_autocommit</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">)</span>
 
         <span class="n">cur</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
         <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">sql</span><span class="p">:</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/druid_hook.html
----------------------------------------------------------------------
diff --git a/_modules/druid_hook.html b/_modules/druid_hook.html
index f5b2a4d..20ffb1d 100644
--- a/_modules/druid_hook.html
+++ b/_modules/druid_hook.html
@@ -165,6 +165,7 @@
 <span class="n">LOAD_CHECK_INTERVAL</span> <span class="o">=</span> <span class="mi">5</span>
 <span class="n">DEFAULT_TARGET_PARTITION_SIZE</span> <span class="o">=</span> <span class="mi">5000000</span>
 
+
 <span class="k">class</span> <span class="nc">AirflowDruidLoadException</span><span class="p">(</span><span class="n">AirflowException</span><span class="p">):</span>
     <span class="k">pass</span>
 
@@ -260,10 +261,10 @@
                         <span class="s2">&quot;mapreduce.map.output.compress&quot;</span><span class="p">:</span> <span class="s2">&quot;false&quot;</span><span class="p">,</span>
                         <span class="s2">&quot;mapreduce.output.fileoutputformat.compress&quot;</span><span class="p">:</span> <span class="s2">&quot;false&quot;</span><span class="p">,</span>
                     <span class="p">},</span>
-                    <span class="s2">&quot;partitionsSpec&quot;</span> <span class="p">:</span> <span class="p">{</span>
-                        <span class="s2">&quot;type&quot;</span> <span class="p">:</span> <span class="s2">&quot;hashed&quot;</span><span class="p">,</span>
-                        <span class="s2">&quot;targetPartitionSize&quot;</span> <span class="p">:</span> <span class="n">target_partition_size</span><span class="p">,</span>
-                        <span class="s2">&quot;numShards&quot;</span> <span class="p">:</span> <span class="n">num_shards</span><span class="p">,</span>
+                    <span class="s2">&quot;partitionsSpec&quot;</span><span class="p">:</span> <span class="p">{</span>
+                        <span class="s2">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;hashed&quot;</span><span class="p">,</span>
+                        <span class="s2">&quot;targetPartitionSize&quot;</span><span class="p">:</span> <span class="n">target_partition_size</span><span class="p">,</span>
+                        <span class="s2">&quot;numShards&quot;</span><span class="p">:</span> <span class="n">num_shards</span><span class="p">,</span>
                     <span class="p">},</span>
                 <span class="p">},</span>
                 <span class="s2">&quot;ioConfig&quot;</span><span class="p">:</span> <span class="p">{</span>
@@ -303,8 +304,9 @@
             <span class="n">intervals</span><span class="p">,</span> <span class="n">num_shards</span><span class="p">,</span> <span class="n">target_partition_size</span><span class="p">,</span> <span class="n">metric_spec</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">hadoop_dependency_coordinates</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        load data to druid from hdfs</span>
-<span class="sd">        :params ts_dim: The column name to use as a timestamp</span>
-<span class="sd">        :params metric_spec: A list of dictionaries</span>
+
+<span class="sd">        :param ts_dim: The column name to use as a timestamp</span>
+<span class="sd">        :param metric_spec: A list of dictionaries</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">task_id</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">send_ingest_query</span><span class="p">(</span>
             <span class="n">datasource</span><span class="p">,</span> <span class="n">static_path</span><span class="p">,</span> <span class="n">ts_dim</span><span class="p">,</span> <span class="n">columns</span><span class="p">,</span> <span class="n">metric_spec</span><span class="p">,</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/email_operator.html
----------------------------------------------------------------------
diff --git a/_modules/email_operator.html b/_modules/email_operator.html
index 6bb4434..4d91901 100644
--- a/_modules/email_operator.html
+++ b/_modules/email_operator.html
@@ -169,6 +169,10 @@
 <span class="sd">    :type html_content: string</span>
 <span class="sd">    :param files: file names to attach in email</span>
 <span class="sd">    :type files: list</span>
+<span class="sd">    :param cc: list of recipients to be added in CC field</span>
+<span class="sd">    :type cc: list or string (comma or semicolon delimited)</span>
+<span class="sd">    :param bcc: list of recipients to be added in BCC field</span>
+<span class="sd">    :type bcc: list or string (comma or semicolon delimited)</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;subject&#39;</span><span class="p">,</span> <span class="s1">&#39;html_content&#39;</span><span class="p">)</span>
@@ -182,15 +186,19 @@
             <span class="n">subject</span><span class="p">,</span>
             <span class="n">html_content</span><span class="p">,</span>
             <span class="n">files</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">cc</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">bcc</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
         <span class="nb">super</span><span class="p">(</span><span class="n">EmailOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">to</span> <span class="o">=</span> <span class="n">to</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">subject</span> <span class="o">=</span> <span class="n">subject</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">html_content</span> <span class="o">=</span> <span class="n">html_content</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">files</span> <span class="o">=</span> <span class="n">files</span> <span class="ow">or</span> <span class="p">[]</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cc</span> <span class="o">=</span> <span class="n">cc</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">bcc</span> <span class="o">=</span> <span class="n">bcc</span>
 
     <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">send_email</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">subject</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">html_content</span><span class="p">,</span> <span class="n">files</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">)</span></div>
+        <span class="n">send_email</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">subject</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">html_content</span><span class="p">,</span> <span class="n">files</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">,</span> <span class="n">cc</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="p">,</span> <span class="n">bcc</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="p">)</span></div>
 </pre></div>
 
            </div>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/ftp_hook.html
----------------------------------------------------------------------
diff --git a/_modules/ftp_hook.html b/_modules/ftp_hook.html
index 19a7498..eed5b97 100644
--- a/_modules/ftp_hook.html
+++ b/_modules/ftp_hook.html
@@ -352,6 +352,16 @@
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
         <span class="n">conn</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div>
 
+<div class="viewcode-block" id="FTPHook.rename"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.FTPHook.rename">[docs]</a>    <span class="k">def</span> <span class="nf">rename</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">from_name</span><span class="p">,</span> <span class="n">to_name</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Rename a file.</span>
+
+<span class="sd">        :param from_name: rename file from name</span>
+<span class="sd">        :param to_name: rename file to name</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">conn</span><span class="o">.</span><span class="n">rename</span><span class="p">(</span><span class="n">from_name</span><span class="p">,</span> <span class="n">to_name</span><span class="p">)</span></div>
+
     <span class="k">def</span> <span class="nf">get_mod_time</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
         <span class="n">ftp_mdtm</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">sendcmd</span><span class="p">(</span><span class="s1">&#39;MDTM &#39;</span> <span class="o">+</span> <span class="n">path</span><span class="p">)</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/hive_hooks.html
----------------------------------------------------------------------
diff --git a/_modules/hive_hooks.html b/_modules/hive_hooks.html
index e25ec97..926b54c 100644
--- a/_modules/hive_hooks.html
+++ b/_modules/hive_hooks.html
@@ -156,19 +156,25 @@
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">zip</span>
 <span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="nb">basestring</span>
 
+<span class="kn">import</span> <span class="nn">collections</span>
 <span class="kn">import</span> <span class="nn">unicodecsv</span> <span class="kn">as</span> <span class="nn">csv</span>
+<span class="kn">import</span> <span class="nn">itertools</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">re</span>
 <span class="kn">import</span> <span class="nn">subprocess</span>
+<span class="kn">import</span> <span class="nn">time</span>
 <span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span>
 <span class="kn">import</span> <span class="nn">hive_metastore</span>
 
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
 <span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.helpers</span> <span class="kn">import</span> <span class="n">as_flattened_list</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.file</span> <span class="kn">import</span> <span class="n">TemporaryDirectory</span>
 <span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">configuration</span>
 <span class="kn">import</span> <span class="nn">airflow.security.utils</span> <span class="kn">as</span> <span class="nn">utils</span>
 
+<span class="n">HIVE_QUEUE_PRIORITIES</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;VERY_HIGH&#39;</span><span class="p">,</span> <span class="s1">&#39;HIGH&#39;</span><span class="p">,</span> <span class="s1">&#39;NORMAL&#39;</span><span class="p">,</span> <span class="s1">&#39;LOW&#39;</span><span class="p">,</span> <span class="s1">&#39;VERY_LOW&#39;</span><span class="p">]</span>
+
 
 <div class="viewcode-block" id="HiveCliHook"><a class="viewcode-back" href="../code.html#airflow.hooks.HiveCliHook">[docs]</a><span class="k">class</span> <span class="nc">HiveCliHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
 
@@ -186,12 +192,24 @@
 
 <span class="sd">    The extra connection parameter ``auth`` gets passed as in the ``jdbc``</span>
 <span class="sd">    connection string as is.</span>
+
+<span class="sd">    :param mapred_queue: queue used by the Hadoop Scheduler (Capacity or Fair)</span>
+<span class="sd">    :type  mapred_queue: string</span>
+<span class="sd">    :param mapred_queue_priority: priority within the job queue.</span>
+<span class="sd">        Possible settings include: VERY_HIGH, HIGH, NORMAL, LOW, VERY_LOW</span>
+<span class="sd">    :type  mapred_queue_priority: string</span>
+<span class="sd">    :param mapred_job_name: This name will appear in the jobtracker.</span>
+<span class="sd">        This can make monitoring easier.</span>
+<span class="sd">    :type  mapred_job_name: string</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
             <span class="bp">self</span><span class="p">,</span>
             <span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="s2">&quot;hive_cli_default&quot;</span><span class="p">,</span>
-            <span class="n">run_as</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+            <span class="n">run_as</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">mapred_queue</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">mapred_queue_priority</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">mapred_job_name</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_params</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;hive_cli_params&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">use_beeline</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;use_beeline&#39;</span><span class="p">,</span> <span class="bp">False</span><span class="p">)</span>
@@ -199,16 +217,92 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="n">conn</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">run_as</span> <span class="o">=</span> <span class="n">run_as</span>
 
+        <span class="k">if</span> <span class="n">mapred_queue_priority</span><span class="p">:</span>
+            <span class="n">mapred_queue_priority</span> <span class="o">=</span> <span class="n">mapred_queue_priority</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
+            <span class="k">if</span> <span class="n">mapred_queue_priority</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">HIVE_QUEUE_PRIORITIES</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+                    <span class="s2">&quot;Invalid Mapred Queue Priority.  Valid values are: &quot;</span>
+                    <span class="s2">&quot;{}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="s1">&#39;, &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">HIVE_QUEUE_PRIORITIES</span><span class="p">)))</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue</span> <span class="o">=</span> <span class="n">mapred_queue</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue_priority</span> <span class="o">=</span> <span class="n">mapred_queue_priority</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mapred_job_name</span> <span class="o">=</span> <span class="n">mapred_job_name</span>
+
+    <span class="k">def</span> <span class="nf">_prepare_cli_cmd</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        This function creates the command list from available information</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span>
+        <span class="n">hive_bin</span> <span class="o">=</span> <span class="s1">&#39;hive&#39;</span>
+        <span class="n">cmd_extra</span> <span class="o">=</span> <span class="p">[]</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_beeline</span><span class="p">:</span>
+            <span class="n">hive_bin</span> <span class="o">=</span> <span class="s1">&#39;beeline&#39;</span>
+            <span class="n">jdbc_url</span> <span class="o">=</span> <span class="s2">&quot;jdbc:hive2://{conn.host}:{conn.port}/{conn.schema}&quot;</span>
+            <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;security&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s1">&#39;kerberos&#39;</span><span class="p">:</span>
+                <span class="n">template</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+                    <span class="s1">&#39;principal&#39;</span><span class="p">,</span> <span class="s2">&quot;hive/_HOST@EXAMPLE.COM&quot;</span><span class="p">)</span>
+                <span class="k">if</span> <span class="s2">&quot;_HOST&quot;</span> <span class="ow">in</span> <span class="n">template</span><span class="p">:</span>
+                    <span class="n">template</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">replace_hostname_pattern</span><span class="p">(</span>
+                        <span class="n">utils</span><span class="o">.</span><span class="n">get_components</span><span class="p">(</span><span class="n">template</span><span class="p">))</span>
+
+                <span class="n">proxy_user</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>  <span class="c1"># noqa</span>
+                <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;proxy_user&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;login&quot;</span> <span class="ow">and</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
+                    <span class="n">proxy_user</span> <span class="o">=</span> <span class="s2">&quot;hive.server2.proxy.user={0}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">)</span>
+                <span class="k">elif</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;proxy_user&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;owner&quot;</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_as</span><span class="p">:</span>
+                    <span class="n">proxy_user</span> <span class="o">=</span> <span class="s2">&quot;hive.server2.proxy.user={0}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">run_as</span><span class="p">)</span>
+
+                <span class="n">jdbc_url</span> <span class="o">+=</span> <span class="s2">&quot;;principal={template};{proxy_user}&quot;</span>
+            <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">auth</span><span class="p">:</span>
+                <span class="n">jdbc_url</span> <span class="o">+=</span> <span class="s2">&quot;;auth=&quot;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">auth</span>
+
+            <span class="n">jdbc_url</span> <span class="o">=</span> <span class="n">jdbc_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
+
+            <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">&#39;-u&#39;</span><span class="p">,</span> <span class="n">jdbc_url</span><span class="p">]</span>
+            <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
+                <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">&#39;-n&#39;</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">]</span>
+            <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">:</span>
+                <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">&#39;-p&#39;</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">]</span>
+
+        <span class="n">hive_params_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_params</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
+
+        <span class="k">return</span> <span class="p">[</span><span class="n">hive_bin</span><span class="p">]</span> <span class="o">+</span> <span class="n">cmd_extra</span> <span class="o">+</span> <span class="n">hive_params_list</span>
+
+    <span class="k">def</span> <span class="nf">_prepare_hiveconf</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">d</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        This function prepares a list of hiveconf params</span>
+<span class="sd">        from a dictionary of key value pairs.</span>
+
+<span class="sd">        :param d:</span>
+<span class="sd">        :type d: dict</span>
+
+<span class="sd">        &gt;&gt;&gt; hh = HiveCliHook()</span>
+<span class="sd">        &gt;&gt;&gt; hive_conf = {&quot;hive.exec.dynamic.partition&quot;: &quot;true&quot;,</span>
+<span class="sd">        ... &quot;hive.exec.dynamic.partition.mode&quot;: &quot;nonstrict&quot;}</span>
+<span class="sd">        &gt;&gt;&gt; hh._prepare_hiveconf(hive_conf)</span>
+<span class="sd">        [&quot;-hiveconf&quot;, &quot;hive.exec.dynamic.partition=true&quot;,\</span>
+<span class="sd"> &quot;-hiveconf&quot;, &quot;hive.exec.dynamic.partition.mode=nonstrict&quot;]</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">d</span><span class="p">:</span>
+            <span class="k">return</span> <span class="p">[]</span>
+        <span class="k">return</span> <span class="n">as_flattened_list</span><span class="p">(</span>
+            <span class="n">itertools</span><span class="o">.</span><span class="n">izip</span><span class="p">(</span>
+                <span class="p">[</span><span class="s2">&quot;-hiveconf&quot;</span><span class="p">]</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">d</span><span class="p">),</span>
+                <span class="p">[</span><span class="s2">&quot;{}={}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">d</span><span class="o">.</span><span class="n">items</span><span class="p">()]</span>
+                <span class="p">)</span>
+            <span class="p">)</span>
+
 <div class="viewcode-block" id="HiveCliHook.run_cli"><a class="viewcode-back" href="../code.html#airflow.hooks.HiveCliHook.run_cli">[docs]</a>    <span class="k">def</span> <span class="nf">run_cli</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">hive_conf</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Run an hql statement using the hive cli. If hive_conf is specified it should be a</span>
-<span class="sd">        dict and the entries will be set as key/value pairs in HiveConf</span>
+<span class="sd">        Run an hql statement using the hive cli. If hive_conf is specified</span>
+<span class="sd">        it should be a dict and the entries will be set as key/value pairs</span>
+<span class="sd">        in HiveConf</span>
 
 
-<span class="sd">        :param hive_conf: if specified these key value pairs will be passed to hive as</span>
-<span class="sd">            ``-hiveconf &quot;key&quot;=&quot;value&quot;``. Note that they will be passed after the</span>
-<span class="sd">            ``hive_cli_params`` and thus will override whatever values are specified in</span>
-<span class="sd">            the database.</span>
+<span class="sd">        :param hive_conf: if specified these key value pairs will be passed</span>
+<span class="sd">            to hive as ``-hiveconf &quot;key&quot;=&quot;value&quot;``. Note that they will be</span>
+<span class="sd">            passed after the ``hive_cli_params`` and thus will override</span>
+<span class="sd">            whatever values are specified in the database.</span>
 <span class="sd">        :type hive_conf: dict</span>
 
 <span class="sd">        &gt;&gt;&gt; hh = HiveCliHook()</span>
@@ -225,47 +319,29 @@
             <span class="k">with</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="nb">dir</span><span class="o">=</span><span class="n">tmp_dir</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
                 <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">hql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;UTF-8&#39;</span><span class="p">))</span>
                 <span class="n">f</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
-                <span class="n">fname</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span>
-                <span class="n">hive_bin</span> <span class="o">=</span> <span class="s1">&#39;hive&#39;</span>
-                <span class="n">cmd_extra</span> <span class="o">=</span> <span class="p">[]</span>
-
-                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_beeline</span><span class="p">:</span>
-                    <span class="n">hive_bin</span> <span class="o">=</span> <span class="s1">&#39;beeline&#39;</span>
-                    <span class="n">jdbc_url</span> <span class="o">=</span> <span class="s2">&quot;jdbc:hive2://{conn.host}:{conn.port}/{conn.schema}&quot;</span>
-                    <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;security&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s1">&#39;kerberos&#39;</span><span class="p">:</span>
-                        <span class="n">template</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
-                            <span class="s1">&#39;principal&#39;</span><span class="p">,</span> <span class="s2">&quot;hive/_HOST@EXAMPLE.COM&quot;</span><span class="p">)</span>
-                        <span class="k">if</span> <span class="s2">&quot;_HOST&quot;</span> <span class="ow">in</span> <span class="n">template</span><span class="p">:</span>
-                            <span class="n">template</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">replace_hostname_pattern</span><span class="p">(</span>
-                                <span class="n">utils</span><span class="o">.</span><span class="n">get_components</span><span class="p">(</span><span class="n">template</span><span class="p">))</span>
-
-                        <span class="n">proxy_user</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>  <span class="c1"># noqa</span>
-                        <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;proxy_user&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;login&quot;</span> <span class="ow">and</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
-                            <span class="n">proxy_user</span> <span class="o">=</span> <span class="s2">&quot;hive.server2.proxy.user={0}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">)</span>
-                        <span class="k">elif</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;proxy_user&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;owner&quot;</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_as</span><span class="p">:</span>
-                            <span class="n">proxy_user</span> <span class="o">=</span> <span class="s2">&quot;hive.server2.proxy.user={0}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">run_as</span><span class="p">)</span>
-
-                        <span class="n">jdbc_url</span> <span class="o">+=</span> <span class="s2">&quot;;principal={template};{proxy_user}&quot;</span>
-                    <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">auth</span><span class="p">:</span>
-                        <span class="n">jdbc_url</span> <span class="o">+=</span> <span class="s2">&quot;;auth=&quot;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">auth</span>
-
-                    <span class="n">jdbc_url</span> <span class="o">=</span> <span class="n">jdbc_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
-
-                    <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">&#39;-u&#39;</span><span class="p">,</span> <span class="n">jdbc_url</span><span class="p">]</span>
-                    <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
-                        <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">&#39;-n&#39;</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">]</span>
-                    <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">:</span>
-                        <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">&#39;-p&#39;</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">]</span>
-
-                <span class="n">hive_conf</span> <span class="o">=</span> <span class="n">hive_conf</span> <span class="ow">or</span> <span class="p">{}</span>
-                <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">hive_conf</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
-                    <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">&#39;-hiveconf&#39;</span><span class="p">,</span> <span class="s1">&#39;{0}={1}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">)]</span>
-
-                <span class="n">hive_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="n">hive_bin</span><span class="p">,</span> <span class="s1">&#39;-f&#39;</span><span class="p">,</span> <span class="n">fname</span><span class="p">]</span> <span class="o">+</span> <span class="n">cmd_extra</span>
-
-                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_params</span><span class="p">:</span>
-                    <span class="n">hive_params_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_params</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
-                    <span class="n">hive_cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">hive_params_list</span><span class="p">)</span>
+                <span class="n">hive_cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_cli_cmd</span><span class="p">()</span>
+                <span class="n">hive_conf_params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_hiveconf</span><span class="p">(</span><span class="n">hive_conf</span><span class="p">)</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue</span><span class="p">:</span>
+                    <span class="n">hive_conf_params</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span>
+                        <span class="p">[</span><span class="s1">&#39;-hiveconf&#39;</span><span class="p">,</span>
+                         <span class="s1">&#39;mapreduce.job.queuename={}&#39;</span>
+                         <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue</span><span class="p">)])</span>
+
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue_priority</span><span class="p">:</span>
+                    <span class="n">hive_conf_params</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span>
+                        <span class="p">[</span><span class="s1">&#39;-hiveconf&#39;</span><span class="p">,</span>
+                         <span class="s1">&#39;mapreduce.job.priority={}&#39;</span>
+                         <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue_priority</span><span class="p">)])</span>
+
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mapred_job_name</span><span class="p">:</span>
+                    <span class="n">hive_conf_params</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span>
+                        <span class="p">[</span><span class="s1">&#39;-hiveconf&#39;</span><span class="p">,</span>
+                         <span class="s1">&#39;mapred.job.name={}&#39;</span>
+                         <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mapred_job_name</span><span class="p">)])</span>
+
+                <span class="n">hive_cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">hive_conf_params</span><span class="p">)</span>
+                <span class="n">hive_cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">([</span><span class="s1">&#39;-f&#39;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">])</span>
+
                 <span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
                     <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot; &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">hive_cmd</span><span class="p">))</span>
                 <span class="n">sp</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span>
@@ -399,6 +475,8 @@
         <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">&#39;sp&#39;</span><span class="p">):</span>
             <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">poll</span><span class="p">()</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
                 <span class="k">print</span><span class="p">(</span><span class="s2">&quot;Killing the Hive job&quot;</span><span class="p">)</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">terminate</span><span class="p">()</span>
+                <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">60</span><span class="p">)</span>
                 <span class="bp">self</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">kill</span><span class="p">()</span></div>
 
 
@@ -700,11 +778,12 @@
                 <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">hql</span><span class="p">)</span>
                 <span class="n">schema</span> <span class="o">=</span> <span class="n">cur</span><span class="o">.</span><span class="n">description</span>
                 <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">csv_filepath</span><span class="p">,</span> <span class="s1">&#39;wb&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
-                    <span class="n">writer</span> <span class="o">=</span> <span class="n">csv</span><span class="o">.</span><span class="n">writer</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">,</span>
-                        <span class="n">lineterminator</span><span class="o">=</span><span class="n">lineterminator</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
+                    <span class="n">writer</span> <span class="o">=</span> <span class="n">csv</span><span class="o">.</span><span class="n">writer</span><span class="p">(</span><span class="n">f</span><span class="p">,</span>
+                                        <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">,</span>
+                                        <span class="n">lineterminator</span><span class="o">=</span><span class="n">lineterminator</span><span class="p">,</span>
+                                        <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
                     <span class="k">if</span> <span class="n">output_header</span><span class="p">:</span>
-                        <span class="n">writer</span><span class="o">.</span><span class="n">writerow</span><span class="p">([</span><span class="n">c</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
-                            <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">cur</span><span class="o">.</span><span class="n">description</span><span class="p">])</span>
+                        <span class="n">writer</span><span class="o">.</span><span class="n">writerow</span><span class="p">([</span><span class="n">c</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">cur</span><span class="o">.</span><span class="n">description</span><span class="p">])</span>
                     <span class="n">i</span> <span class="o">=</span> <span class="mi">0</span>
                     <span class="k">while</span> <span class="bp">True</span><span class="p">:</span>
                         <span class="n">rows</span> <span class="o">=</span> <span class="p">[</span><span class="n">row</span> <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">cur</span><span class="o">.</span><span class="n">fetchmany</span><span class="p">(</span><span class="n">fetch_size</span><span class="p">)</span> <span class="k">if</span> <span class="n">row</span><span class="p">]</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/hive_operator.html
----------------------------------------------------------------------
diff --git a/_modules/hive_operator.html b/_modules/hive_operator.html
index de5e44b..4d8c62c 100644
--- a/_modules/hive_operator.html
+++ b/_modules/hive_operator.html
@@ -177,6 +177,14 @@
 <span class="sd">    :param script_begin_tag: If defined, the operator will get rid of the</span>
 <span class="sd">        part of the script before the first occurrence of `script_begin_tag`</span>
 <span class="sd">    :type script_begin_tag: str</span>
+<span class="sd">    :param mapred_queue: queue used by the Hadoop CapacityScheduler</span>
+<span class="sd">    :type  mapred_queue: string</span>
+<span class="sd">    :param mapred_queue_priority: priority within CapacityScheduler queue.</span>
+<span class="sd">        Possible settings include: VERY_HIGH, HIGH, NORMAL, LOW, VERY_LOW</span>
+<span class="sd">    :type  mapred_queue_priority: string</span>
+<span class="sd">    :param mapred_job_name: This name will appear in the jobtracker.</span>
+<span class="sd">        This can make monitoring easier.</span>
+<span class="sd">    :type  mapred_job_name: string</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;hql&#39;</span><span class="p">,</span> <span class="s1">&#39;schema&#39;</span><span class="p">)</span>
@@ -191,6 +199,9 @@
             <span class="n">hiveconf_jinja_translate</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
             <span class="n">script_begin_tag</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">run_as_owner</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+            <span class="n">mapred_queue</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">mapred_queue_priority</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">mapred_job_name</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
 
         <span class="nb">super</span><span class="p">(</span><span class="n">HiveOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -203,8 +214,17 @@
         <span class="k">if</span> <span class="n">run_as_owner</span><span class="p">:</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">run_as</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">owner</span>
 
+        <span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue</span> <span class="o">=</span> <span class="n">mapred_queue</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue_priority</span> <span class="o">=</span> <span class="n">mapred_queue_priority</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mapred_job_name</span> <span class="o">=</span> <span class="n">mapred_job_name</span>
+
     <span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-        <span class="k">return</span> <span class="n">HiveCliHook</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_conn_id</span><span class="p">,</span> <span class="n">run_as</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">run_as</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">HiveCliHook</span><span class="p">(</span>
+                        <span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_conn_id</span><span class="p">,</span>
+                        <span class="n">run_as</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">run_as</span><span class="p">,</span>
+                        <span class="n">mapred_queue</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue</span><span class="p">,</span>
+                        <span class="n">mapred_queue_priority</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue_priority</span><span class="p">,</span>
+                        <span class="n">mapred_job_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mapred_job_name</span><span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">prepare_template</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">hiveconf_jinja_translate</span><span class="p">:</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/hive_to_mysql.html
----------------------------------------------------------------------
diff --git a/_modules/hive_to_mysql.html b/_modules/hive_to_mysql.html
index 867023b..fd8b342 100644
--- a/_modules/hive_to_mysql.html
+++ b/_modules/hive_to_mysql.html
@@ -160,6 +160,7 @@
 
 <span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span>
 
+
 <div class="viewcode-block" id="HiveToMySqlTransfer"><a class="viewcode-back" href="../code.html#airflow.operators.HiveToMySqlTransfer">[docs]</a><span class="k">class</span> <span class="nc">HiveToMySqlTransfer</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Moves data from Hive to MySQL, note that for now the data is loaded</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/mysql_hook.html
----------------------------------------------------------------------
diff --git a/_modules/mysql_hook.html b/_modules/mysql_hook.html
index 941737f..a07aeb2 100644
--- a/_modules/mysql_hook.html
+++ b/_modules/mysql_hook.html
@@ -156,6 +156,7 @@
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>
 
+
 <div class="viewcode-block" id="MySqlHook"><a class="viewcode-back" href="../code.html#airflow.hooks.MySqlHook">[docs]</a><span class="k">class</span> <span class="nc">MySqlHook</span><span class="p">(</span><span class="n">DbApiHook</span><span class="p">):</span>
     <span class="sd">&#39;&#39;&#39;</span>
 <span class="sd">    Interact with MySQL.</span>



Mime
View raw message