airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [airflow-site] 04/30: Updating the disclaimer
Date Wed, 27 Nov 2019 19:11:48 GMT
This is an automated email from the ASF dual-hosted git repository.

ash pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/airflow-site.git

commit 457b698d9f92af370061d322267d85886e6cbfe4
Author: Maxime Beauchemin <maxime.beauchemin@apache.org>
AuthorDate: Fri Jul 1 14:54:14 2016 -0700

    Updating the disclaimer
---
 .../airflow/contrib/executors/mesos_executor.html  |  16 +-
 .../contrib/operators/bigquery_operator.html       |  16 +-
 .../airflow/contrib/operators/bigquery_to_gcs.html |  16 +-
 .../contrib/operators/gcs_download_operator.html   |  37 ++++-
 _modules/airflow/executors/celery_executor.html    |  16 +-
 _modules/airflow/executors/local_executor.html     |  16 +-
 .../airflow/executors/sequential_executor.html     |  16 +-
 _modules/airflow/macros.html                       |  40 ++++-
 _modules/airflow/macros/hive.html                  |  20 ++-
 _modules/airflow/models.html                       | 111 ++++++++++---
 _modules/airflow/operators/docker_operator.html    |  16 +-
 _modules/airflow/operators/sensors.html            | 137 ++++++++++++++--
 _modules/bash_operator.html                        |  16 +-
 _modules/dagrun_operator.html                      |  16 +-
 _modules/dbapi_hook.html                           |  24 ++-
 _modules/druid_hook.html                           |  16 +-
 _modules/dummy_operator.html                       |  16 +-
 _modules/email_operator.html                       |  16 +-
 _modules/ftp_hook.html                             |  22 ++-
 _modules/generic_transfer.html                     |  16 +-
 _modules/hive_hooks.html                           |  82 ++++++++--
 _modules/hive_operator.html                        |  22 ++-
 _modules/hive_to_druid.html                        |  48 ++++--
 _modules/hive_to_mysql.html                        |  21 ++-
 _modules/hive_to_samba_operator.html               |  19 ++-
 _modules/http_hook.html                            |   2 +-
 _modules/http_operator.html                        |  18 ++-
 _modules/mssql_hook.html                           |  25 ++-
 _modules/mssql_operator.html                       |  20 ++-
 _modules/mssql_to_hive.html                        |  19 ++-
 _modules/mysql_hook.html                           |  16 +-
 _modules/mysql_operator.html                       |  20 ++-
 _modules/mysql_to_hive.html                        |  19 ++-
 _modules/postgres_hook.html                        |  16 +-
 _modules/postgres_operator.html                    |  20 ++-
 _modules/presto_check_operator.html                |  18 ++-
 _modules/presto_hook.html                          |  25 ++-
 _modules/python_operator.html                      |  16 +-
 _modules/qubole_operator.html                      |  20 ++-
 _modules/s3_to_hive_operator.html                  |  19 ++-
 _modules/sensors.html                              | 137 ++++++++++++++--
 _modules/slack_operator.html                       |  16 +-
 _modules/sqlite_hook.html                          |  16 +-
 _modules/ssh_execute_operator.html                 |  16 +-
 _modules/vertica_operator.html                     |  20 ++-
 _modules/vertica_to_hive.html                      |  20 ++-
 _modules/webhdfs_hook.html                         |  16 +-
 _sources/code.txt                                  |   9 ++
 _sources/concepts.txt                              |   4 +-
 _sources/faq.txt                                   |  15 +-
 _sources/index.txt                                 |   7 +-
 _sources/tutorial.txt                              |   6 +-
 code.html                                          | 180 ++++++++++++++++++---
 concepts.html                                      |   4 +-
 faq.html                                           |  13 +-
 genindex.html                                      |  23 ++-
 index.html                                         |   8 +-
 objects.inv                                        | Bin 2236 -> 2246 bytes
 searchindex.js                                     |   2 +-
 tutorial.html                                      |   6 +-
 60 files changed, 1363 insertions(+), 219 deletions(-)

diff --git a/_modules/airflow/contrib/executors/mesos_executor.html b/_modules/airflow/contrib/executors/mesos_executor.html
index 65557d9..e2d8f61 100644
--- a/_modules/airflow/contrib/executors/mesos_executor.html
+++ b/_modules/airflow/contrib/executors/mesos_executor.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.contrib.executors.mesos_executor</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
 <span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
 <span class="kn">import</span> <span class="nn">logging</span>
diff --git a/_modules/airflow/contrib/operators/bigquery_operator.html b/_modules/airflow/contrib/operators/bigquery_operator.html
index 830fd57..6038f2c 100644
--- a/_modules/airflow/contrib/operators/bigquery_operator.html
+++ b/_modules/airflow/contrib/operators/bigquery_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.contrib.operators.bigquery_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
 
 <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
diff --git a/_modules/airflow/contrib/operators/bigquery_to_gcs.html b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
index 465128c..a3b8d18 100644
--- a/_modules/airflow/contrib/operators/bigquery_to_gcs.html
+++ b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.contrib.operators.bigquery_to_gcs</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
 
 <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
diff --git a/_modules/airflow/contrib/operators/gcs_download_operator.html b/_modules/airflow/contrib/operators/gcs_download_operator.html
index 4af6020..dbf56fb 100644
--- a/_modules/airflow/contrib/operators/gcs_download_operator.html
+++ b/_modules/airflow/contrib/operators/gcs_download_operator.html
@@ -137,7 +137,22 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.contrib.operators.gcs_download_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+<span class="kn">import</span> <span class="nn">sys</span>
 
 <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="kn">import</span> <span class="n">GoogleCloudStorageHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
@@ -147,7 +162,7 @@
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Downloads a file from Google Cloud Storage.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;bucket&#39;</span><span class="p">,</span><span class="s1">&#39;object&#39;</span><span class="p">,</span><span class="s1">&#39;filename&#39;</span><span class="p">,)</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;bucket&#39;</span><span class="p">,</span><span class="s1">&#39;object&#39;</span><span class="p">,</span><span class="s1">&#39;filename&#39;</span><span class="p">,</span><span class="s1">&#39;store_to_xcom_key&#39;</span><span class="p">,)</span>
     <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.sql&#39;</span><span class="p">,)</span>
     <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#f0eee4&#39;</span>
 
@@ -156,7 +171,8 @@
         <span class="bp">self</span><span class="p">,</span>
         <span class="n">bucket</span><span class="p">,</span>
         <span class="nb">object</span><span class="p">,</span>
-        <span class="n">filename</span><span class="p">,</span>
+        <span class="n">filename</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+        <span class="n">store_to_xcom_key</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
         <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_storage_default&#39;</span><span class="p">,</span>
         <span class="n">delegate_to</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
         <span class="o">*</span><span class="n">args</span><span class="p">,</span>
@@ -171,7 +187,13 @@
 <span class="sd">        :type object: string</span>
 <span class="sd">        :param filename: The file path on the local file system (where the</span>
 <span class="sd">            operator is being executed) that the file should be downloaded to.</span>
+<span class="sd">            If false, the downloaded data will not be stored on the local file</span>
+<span class="sd">            system.</span>
 <span class="sd">        :type filename: string</span>
+<span class="sd">        :param store_to_xcom_key: If this param is set, the operator will push</span>
+<span class="sd">            the contents of the downloaded file to XCom with the key set in this</span>
+<span class="sd">            parameter. If false, the downloaded data will not be pushed to XCom.</span>
+<span class="sd">        :type store_to_xcom_key: string</span>
 <span class="sd">        :param google_cloud_storage_conn_id: The connection ID to use when</span>
 <span class="sd">            connecting to Google cloud storage.</span>
 <span class="sd">        :type google_cloud_storage_conn_id: string</span>
@@ -183,6 +205,7 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">object</span> <span class="o">=</span> <span class="nb">object</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">filename</span> <span class="o">=</span> <span class="n">filename</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">store_to_xcom_key</span> <span class="o">=</span> <span class="n">store_to_xcom_key</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
 
@@ -190,7 +213,13 @@
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Executing download: </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class= [...]
         <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span><span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
                                       <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
-        <span class="k">print</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">))</span></div>
+        <span class="n">file_bytes</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">store_to_xcom_key</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">getsizeof</span><span class="p">(</span><span class="n">file_bytes</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">48000</span><span class="p">:</span>
+                <span class="n">context</span><span class="p">[</span><span class="s1">&#39;ti&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">xcom_push</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">store_to_xcom_key</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">file_bytes</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s1">&#39;The size of the downloaded file is too large to push to XCom!&#39;</span><span class="p">)</span>
+        <span class="k">print</span><span class="p">(</span><span class="n">file_bytes</span><span class="p">)</span></div>
 </pre></div>
 
            </div>
diff --git a/_modules/airflow/executors/celery_executor.html b/_modules/airflow/executors/celery_executor.html
index a231e28..9601c12 100644
--- a/_modules/airflow/executors/celery_executor.html
+++ b/_modules/airflow/executors/celery_executor.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.executors.celery_executor</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">object</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">object</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">subprocess</span>
 <span class="kn">import</span> <span class="nn">time</span>
diff --git a/_modules/airflow/executors/local_executor.html b/_modules/airflow/executors/local_executor.html
index 79c456d..1841cb4 100644
--- a/_modules/airflow/executors/local_executor.html
+++ b/_modules/airflow/executors/local_executor.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.executors.local_executor</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">multiprocessing</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">multiprocessing</span>
 <span class="kn">import</span> <span class="nn">subprocess</span>
 <span class="kn">import</span> <span class="nn">time</span>
 
diff --git a/_modules/airflow/executors/sequential_executor.html b/_modules/airflow/executors/sequential_executor.html
index a393b0f..ce84843 100644
--- a/_modules/airflow/executors/sequential_executor.html
+++ b/_modules/airflow/executors/sequential_executor.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.executors.sequential_executor</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
 <span class="kn">import</span> <span class="nn">subprocess</span>
 
 <span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="kn">import</span> <span class="n">BaseExecutor</span>
diff --git a/_modules/airflow/macros.html b/_modules/airflow/macros.html
index 6b4e17f..78efd22 100644
--- a/_modules/airflow/macros.html
+++ b/_modules/airflow/macros.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.macros</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
 <span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">random</span>
 <span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
 <span class="kn">import</span> <span class="nn">dateutil</span>
@@ -187,11 +201,29 @@
     <span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">ds</span><span class="p">,</span> <span class="n">input_format</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="n">output_format</span><span class="p">)</span></div>
 
 
-<div class="viewcode-block" id="integrate_plugins"><a class="viewcode-back" href="../../code.html#airflow.macros.integrate_plugins">[docs]</a><span class="k">def</span> <span class="nf">integrate_plugins</span><span class="p">():</span>
+<span class="k">def</span> <span class="nf">_integrate_plugins</span><span class="p">():</span>
     <span class="sd">&quot;&quot;&quot;Integrate plugins to the context&quot;&quot;&quot;</span>
+    <span class="kn">import</span> <span class="nn">sys</span>
     <span class="kn">from</span> <span class="nn">airflow.plugins_manager</span> <span class="kn">import</span> <span class="n">macros</span> <span class="k">as</span> <span class="n">_macros</span>
-    <span class="k">for</span> <span class="n">_macro</span> <span class="ow">in</span> <span class="n">_macros</span><span class="p">:</span>
-        <span class="nb">globals</span><span class="p">()[</span><span class="n">_macro</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro</span></div>
+    <span class="k">for</span> <span class="n">_macro_module</span> <span class="ow">in</span> <span class="n">_macros</span><span class="p">:</span>
+        <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="n">_macro_module</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro_module</span>
+        <span class="nb">globals</span><span class="p">()[</span><span class="n">_macro_module</span><span class="o">.</span><span class="n">_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro_module</span>
+
+
+        <span class="c1">##########################################################</span>
+        <span class="c1"># TODO FIXME Remove in Airflow 2.0</span>
+
+        <span class="kn">import</span> <span class="nn">os</span> <span class="kn">as</span> <span class="nn">_os</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">_os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AIRFLOW_USE_NEW_IMPORTS&#39;</span><span class="p">,</span> <span class="bp">False</span><span class="p">):</span>
+            <span class="kn">from</span> <span class="nn">zope.deprecation</span> <span class="kn">import</span> <span class="n">deprecated</span> <span class="k">as</span> <span class="n">_deprecated</span>
+            <span class="k">for</span> <span class="n">_macro</span> <span class="ow">in</span> <span class="n">_macro_module</span><span class="o">.</span><span class="n">_objects</span><span class="p">:</span>
+                <span class="nb">globals</span><span class="p">()[</span><span class="n">_macro</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">_deprecated</span><span class="p">(</span>
+                    <span class="n">_macro</span><span class="p">,</span>
+                    <span class="s2">&quot;Importing plugin macro &#39;{i}&#39; directly from &quot;</span>
+                    <span class="s2">&quot;&#39;airflow.macros&#39; has been deprecated. Please &quot;</span>
+                    <span class="s2">&quot;import from &#39;airflow.macros.[plugin_module]&#39; &quot;</span>
+                    <span class="s2">&quot;instead. Support for direct imports will be dropped &quot;</span>
+                    <span class="s2">&quot;entirely in Airflow 2.0.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="o">=</span><span class="n">_macro</span><span class="p">))</span>
 </pre></div>
 
            </div>
diff --git a/_modules/airflow/macros/hive.html b/_modules/airflow/macros/hive.html
index 43cd2a0..92000d3 100644
--- a/_modules/airflow/macros/hive.html
+++ b/_modules/airflow/macros/hive.html
@@ -139,7 +139,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.macros.hive</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">datetime</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">datetime</span>
 
 
 <div class="viewcode-block" id="max_partition"><a class="viewcode-back" href="../../../code.html#airflow.macros.hive.max_partition">[docs]</a><span class="k">def</span> <span class="nf">max_partition</span><span class="p">(</span>
@@ -166,7 +180,7 @@
 <span class="sd">    &gt;&gt;&gt; max_partition(&#39;airflow.static_babynames_partitioned&#39;)</span>
 <span class="sd">    &#39;2015-01-01&#39;</span>
 <span class="sd">    &#39;&#39;&#39;</span>
-    <span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span>
+    <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span>
     <span class="k">if</span> <span class="s1">&#39;.&#39;</span> <span class="ow">in</span> <span class="n">table</span><span class="p">:</span>
         <span class="n">schema</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
     <span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span><span class="n">metastore_conn_id</span><span class="o">=</span><span class="n">metastore_conn_id</span><span class="p">)</span>
@@ -219,7 +233,7 @@
 <span class="sd">    &gt;&gt;&gt; closest_ds_partition(tbl, &#39;2015-01-02&#39;)</span>
 <span class="sd">    &#39;2015-01-01&#39;</span>
 <span class="sd">    &#39;&#39;&#39;</span>
-    <span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span>
+    <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span>
     <span class="k">if</span> <span class="s1">&#39;.&#39;</span> <span class="ow">in</span> <span class="n">table</span><span class="p">:</span>
         <span class="n">schema</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
     <span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span><span class="n">metastore_conn_id</span><span class="o">=</span><span class="n">metastore_conn_id</span><span class="p">)</span>
diff --git a/_modules/airflow/models.html b/_modules/airflow/models.html
index aba90cd..d4ed8a6 100644
--- a/_modules/airflow/models.html
+++ b/_modules/airflow/models.html
@@ -735,33 +735,43 @@
                        <span class="n">descriptor</span><span class="o">=</span><span class="nb">property</span><span class="p">(</span><span class="n">cls</span><span class="o">.</span><span class="n">get_extra</span><span class="p">,</span> <span class="n">cls</span><span class="o">.</span><span class="n">set_extra</span><span class="p">))</span>
 
     <span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-        <span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">hooks</span>
-        <span class="kn">from</span> <span class="nn">airflow.contrib</span> <span class="kn">import</span> <span class="n">hooks</span> <span class="k">as</span> <span class="n">contrib_hooks</span>
         <span class="k">try</span><span class="p">:</span>
             <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;mysql&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span>
+                <span class="k">return</span> <span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;google_cloud_platform&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">contrib_hooks</span><span class="o">.</span><span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
+                <span class="k">return</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;postgres&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">PostgresHook</span><span class="p">(</span><span class="n">postgres_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.postgres_hook</span> <span class="kn">import</span> <span class="n">PostgresHook</span>
+                <span class="k">return</span> <span class="n">PostgresHook</span><span class="p">(</span><span class="n">postgres_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;hive_cli&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HiveCliHook</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
+                <span class="k">return</span> <span class="n">HiveCliHook</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;presto&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">PrestoHook</span><span class="p">(</span><span class="n">presto_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.presto_hook</span> <span class="kn">import</span> <span class="n">PrestoHook</span>
+                <span class="k">return</span> <span class="n">PrestoHook</span><span class="p">(</span><span class="n">presto_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;hiveserver2&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HiveServer2Hook</span><span class="p">(</span><span class="n">hiveserver2_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span>
+                <span class="k">return</span> <span class="n">HiveServer2Hook</span><span class="p">(</span><span class="n">hiveserver2_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;sqlite&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">SqliteHook</span><span class="p">(</span><span class="n">sqlite_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.sqlite_hook</span> <span class="kn">import</span> <span class="n">SqliteHook</span>
+                <span class="k">return</span> <span class="n">SqliteHook</span><span class="p">(</span><span class="n">sqlite_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;jdbc&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">JdbcHook</span><span class="p">(</span><span class="n">jdbc_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.jdbc_hook</span> <span class="kn">import</span> <span class="n">JdbcHook</span>
+                <span class="k">return</span> <span class="n">JdbcHook</span><span class="p">(</span><span class="n">jdbc_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;mssql&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">MsSqlHook</span><span class="p">(</span><span class="n">mssql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.mssql_hook</span> <span class="kn">import</span> <span class="n">MsSqlHook</span>
+                <span class="k">return</span> <span class="n">MsSqlHook</span><span class="p">(</span><span class="n">mssql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;oracle&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">OracleHook</span><span class="p">(</span><span class="n">oracle_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.oracle_hook</span> <span class="kn">import</span> <span class="n">OracleHook</span>
+                <span class="k">return</span> <span class="n">OracleHook</span><span class="p">(</span><span class="n">oracle_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;vertica&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">contrib_hooks</span><span class="o">.</span><span class="n">VerticaHook</span><span class="p">(</span><span class="n">vertica_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.vertica_hook</span> <span class="kn">import</span> <span class="n">VerticaHook</span>
+                <span class="k">return</span> <span class="n">VerticaHook</span><span class="p">(</span><span class="n">vertica_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;cloudant&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">contrib_hooks</span><span class="o">.</span><span class="n">CloudantHook</span><span class="p">(</span><span class="n">cloudant_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.cloudant_hook</span> <span class="kn">import</span> <span class="n">CloudantHook</span>
+                <span class="k">return</span> <span class="n">CloudantHook</span><span class="p">(</span><span class="n">cloudant_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
         <span class="k">except</span><span class="p">:</span>
             <span class="k">return</span> <span class="bp">None</span>
 
@@ -844,6 +854,7 @@
 
     <span class="n">__table_args__</span> <span class="o">=</span> <span class="p">(</span>
         <span class="n">Index</span><span class="p">(</span><span class="s1">&#39;ti_dag_state&#39;</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">,</span> <span class="n">state</span><span class="p">),</span>
+        <span class="n">Index</span><span class="p">(</span><span class="s1">&#39;ti_state&#39;</span><span class="p">,</span> <span class="n">state</span><span class="p">),</span>
         <span class="n">Index</span><span class="p">(</span><span class="s1">&#39;ti_state_lkp&#39;</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">execution_date</span><span class="p">,</span> <span class="n">state</span><span class="p">),</span>
         <span class="n">Index</span><span class="p">(</span><span class="s1">&#39;ti_pool&#39;</span><span class="p">,</span> <span class="n">pool</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">priority_weight</span><span class="p">),</span>
     <span class="p">)</span>
@@ -1308,17 +1319,32 @@
 <div class="viewcode-block" id="TaskInstance.run"><a class="viewcode-back" href="../../code.html#airflow.models.TaskInstance.run">[docs]</a>    <span class="k">def</span> <span class="nf">run</span><span class="p">(</span>
             <span class="bp">self</span><span class="p">,</span>
             <span class="n">verbose</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
-            <span class="n">ignore_dependencies</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>  <span class="c1"># Doesn&#39;t check for deps, just runs</span>
-            <span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>   <span class="c1"># Ignore depends_on_past but respect</span>
-                                            <span class="c1"># other deps</span>
-            <span class="n">force</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>  <span class="c1"># Disregards previous successes</span>
-            <span class="n">mark_success</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>  <span class="c1"># Don&#39;t run the task, act as if it succeeded</span>
-            <span class="n">test_mode</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>  <span class="c1"># Doesn&#39;t record success or failure in the DB</span>
+            <span class="n">ignore_dependencies</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+            <span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+            <span class="n">force</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+            <span class="n">mark_success</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+            <span class="n">test_mode</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
             <span class="n">job_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">pool</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">session</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Runs the task instance.</span>
+
+<span class="sd">        :param verbose: whether to turn on more verbose loggin</span>
+<span class="sd">        :type verbose: boolean</span>
+<span class="sd">        :param ignore_dependencies: Doesn&#39;t check for deps, just runs</span>
+<span class="sd">        :type ignore_dependencies: boolean</span>
+<span class="sd">        :param ignore_depends_on_past: Ignore depends_on_past but respect</span>
+<span class="sd">            other dependencies</span>
+<span class="sd">        :type ignore_depends_on_past: boolean</span>
+<span class="sd">        :param force: Forces a run regarless of previous success</span>
+<span class="sd">        :type force: boolean</span>
+<span class="sd">        :param mark_success: Don&#39;t run the task, mark its state as success</span>
+<span class="sd">        :type mark_success: boolean</span>
+<span class="sd">        :param test_mode: Doesn&#39;t record success or failure in the DB</span>
+<span class="sd">        :type test_mode: boolean</span>
+<span class="sd">        :param pool: specifies the pool to use to run the task instance</span>
+<span class="sd">        :type pool: str</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">task</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">pool</span> <span class="o">=</span> <span class="n">pool</span> <span class="ow">or</span> <span class="n">task</span><span class="o">.</span><span class="n">pool</span>
@@ -1560,6 +1586,24 @@
         <span class="k">if</span> <span class="n">task</span><span class="o">.</span><span class="n">params</span><span class="p">:</span>
             <span class="n">params</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">params</span><span class="p">)</span>
 
+        <span class="k">class</span> <span class="nc">VariableAccessor</span><span class="p">:</span>
+            <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">            Wrapper around Variable. This way you can get variables in templates by using</span>
+<span class="sd">            {var.variable_name}.</span>
+<span class="sd">            &quot;&quot;&quot;</span>
+            <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+                <span class="k">pass</span>
+
+            <span class="k">def</span> <span class="nf">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span>
+                <span class="k">return</span> <span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
+
+        <span class="k">class</span> <span class="nc">VariableJsonAccessor</span><span class="p">:</span>
+            <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+                <span class="k">pass</span>
+
+            <span class="k">def</span> <span class="nf">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span>
+                <span class="k">return</span> <span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="n">deserialize_json</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
+
         <span class="k">return</span> <span class="p">{</span>
             <span class="s1">&#39;dag&#39;</span><span class="p">:</span> <span class="n">task</span><span class="o">.</span><span class="n">dag</span><span class="p">,</span>
             <span class="s1">&#39;ds&#39;</span><span class="p">:</span> <span class="n">ds</span><span class="p">,</span>
@@ -1585,6 +1629,10 @@
             <span class="s1">&#39;task_instance_key_str&#39;</span><span class="p">:</span> <span class="n">ti_key_str</span><span class="p">,</span>
             <span class="s1">&#39;conf&#39;</span><span class="p">:</span> <span class="n">configuration</span><span class="p">,</span>
             <span class="s1">&#39;test_mode&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_mode</span><span class="p">,</span>
+            <span class="s1">&#39;var&#39;</span><span class="p">:</span> <span class="p">{</span>
+                <span class="s1">&#39;value&#39;</span><span class="p">:</span> <span class="n">VariableAccessor</span><span class="p">(),</span>
+                <span class="s1">&#39;json&#39;</span><span class="p">:</span> <span class="n">VariableJsonAccessor</span><span class="p">()</span>
+            <span class="p">}</span>
         <span class="p">}</span>
 
     <span class="k">def</span> <span class="nf">render_templates</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
@@ -2706,6 +2754,21 @@
         <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span><span class="p">,</span> <span class="n">timedelta</span><span class="p">):</span>
             <span class="k">return</span> <span class="n">dttm</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span>
 
+<div class="viewcode-block" id="DAG.normalize_schedule"><a class="viewcode-back" href="../../code.html#airflow.models.DAG.normalize_schedule">[docs]</a>    <span class="k">def</span> <span class="nf">normalize_schedule</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dttm</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Returns dttm + interval unless dttm is first interval then it returns dttm</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">following</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">following_schedule</span><span class="p">(</span><span class="n">dttm</span><span class="p">)</span>
+
+        <span class="c1"># in case of @once</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">following</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">dttm</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">previous_schedule</span><span class="p">(</span><span class="n">following</span><span class="p">)</span> <span class="o">!=</span> <span class="n">dttm</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">following</span>
+
+        <span class="k">return</span> <span class="n">dttm</span></div>
+
     <span class="nd">@property</span>
     <span class="k">def</span> <span class="nf">tasks</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="o">.</span><span class="n">values</span><span class="p">())</span>
@@ -3485,10 +3548,12 @@
 
     <span class="nd">@staticmethod</span>
     <span class="nd">@provide_session</span>
-    <span class="k">def</span> <span class="nf">find</span><span class="p">(</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">run_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">execution_date</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+    <span class="k">def</span> <span class="nf">find</span><span class="p">(</span><span class="n">dag_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">run_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">execution_date</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
              <span class="n">state</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">external_trigger</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Returns a set of dag runs for the given search criteria.</span>
+<span class="sd">        :param dag_id: the dag_id to find dag runs for</span>
+<span class="sd">        :type dag_id: integer, list</span>
 <span class="sd">        :param run_id: defines the the run id for this dag run</span>
 <span class="sd">        :type run_id: string</span>
 <span class="sd">        :param execution_date: the execution date</span>
@@ -3502,7 +3567,9 @@
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">DR</span> <span class="o">=</span> <span class="n">DagRun</span>
 
-        <span class="n">qry</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DR</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">)</span>
+        <span class="n">qry</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DR</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">dag_id</span><span class="p">:</span>
+            <span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">)</span>
         <span class="k">if</span> <span class="n">run_id</span><span class="p">:</span>
             <span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">run_id</span> <span class="o">==</span> <span class="n">run_id</span><span class="p">)</span>
         <span class="k">if</span> <span class="n">execution_date</span><span class="p">:</span>
diff --git a/_modules/airflow/operators/docker_operator.html b/_modules/airflow/operators/docker_operator.html
index 46fcf7b..06aa65e 100644
--- a/_modules/airflow/operators/docker_operator.html
+++ b/_modules/airflow/operators/docker_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.operators.docker_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">json</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">json</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
diff --git a/_modules/airflow/operators/sensors.html b/_modules/airflow/operators/sensors.html
index cbadb0a..65a0587 100644
--- a/_modules/airflow/operators/sensors.html
+++ b/_modules/airflow/operators/sensors.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.operators.sensors</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
 <span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
 <span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
@@ -146,10 +160,11 @@
 <span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlparse</span>
 <span class="kn">from</span> <span class="nn">time</span> <span class="kn">import</span> <span class="n">sleep</span>
 
+<span class="kn">import</span> <span class="nn">airflow</span>
 <span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">hooks</span><span class="p">,</span> <span class="n">settings</span>
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span><span class="p">,</span> <span class="n">AirflowSensorTimeout</span><span class="p">,</span> <span class="n">AirflowSkipException</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span><span class="p">,</span> <span class="n">TaskInstance</span><span class="p">,</span> <span class="n">Connection</span> <span class="k">as</span> <span class="n">DB</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="kn">import</span> <span class="n">State</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
@@ -193,12 +208,12 @@
     <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="n">started_at</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span>
         <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">poke</span><span class="p">(</span><span class="n">context</span><span class="p">):</span>
-            <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">poke_interval</span><span class="p">)</span>
-            <span class="k">if</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">started_at</span><span class="p">)</span><span class="o">.</span><span class="n">seconds</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">:</span>
+            <span class="k">if</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">started_at</span><span class="p">)</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">()</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">:</span>
                 <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">soft_fail</span><span class="p">:</span>
                     <span class="k">raise</span> <span class="n">AirflowSkipException</span><span class="p">(</span><span class="s1">&#39;Snap. Time is OUT.&#39;</span><span class="p">)</span>
                 <span class="k">else</span><span class="p">:</span>
                     <span class="k">raise</span> <span class="n">AirflowSensorTimeout</span><span class="p">(</span><span class="s1">&#39;Snap. Time is OUT.&#39;</span><span class="p">)</span>
+            <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">poke_interval</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Success criteria met. Exiting.&quot;</span><span class="p">)</span></div>
 
 
@@ -303,8 +318,14 @@
 <span class="sd">    :type allowed_states: list</span>
 <span class="sd">    :param execution_delta: time difference with the previous execution to</span>
 <span class="sd">        look at, the default is the same execution_date as the current task.</span>
-<span class="sd">        For yesterday, use [positive!] datetime.timedelta(days=1)</span>
+<span class="sd">        For yesterday, use [positive!] datetime.timedelta(days=1). Either</span>
+<span class="sd">        execution_delta or execution_date_fn can be passed to</span>
+<span class="sd">        ExternalTaskSensor, but not both.</span>
 <span class="sd">    :type execution_delta: datetime.timedelta</span>
+<span class="sd">    :param execution_date_fn: function that receives the current execution date</span>
+<span class="sd">        and returns the desired execution date to query. Either execution_delta</span>
+<span class="sd">        or execution_date_fn can be passed to ExternalTaskSensor, but not both.</span>
+<span class="sd">    :type execution_date_fn: callable</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="nd">@apply_defaults</span>
@@ -314,16 +335,25 @@
             <span class="n">external_task_id</span><span class="p">,</span>
             <span class="n">allowed_states</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">execution_delta</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">execution_date_fn</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
         <span class="nb">super</span><span class="p">(</span><span class="n">ExternalTaskSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">allowed_states</span> <span class="o">=</span> <span class="n">allowed_states</span> <span class="ow">or</span> <span class="p">[</span><span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">execution_delta</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span> <span class="ow">and</span> <span class="n">execution_date_fn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s1">&#39;Only one of `execution_date` or `execution_date_fn` may&#39;</span>
+                <span class="s1">&#39;be provided to ExternalTaskSensor; not both.&#39;</span><span class="p">)</span>
+
         <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span> <span class="o">=</span> <span class="n">execution_delta</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span> <span class="o">=</span> <span class="n">execution_date_fn</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">external_dag_id</span> <span class="o">=</span> <span class="n">external_dag_id</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">external_task_id</span> <span class="o">=</span> <span class="n">external_task_id</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span><span class="p">:</span>
             <span class="n">dttm</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">&#39;execution_date&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span><span class="p">:</span>
+            <span class="n">dttm</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span><span class="p">(</span><span class="n">context</span><span class="p">[</span><span class="s1">&#39;execution_date&#39;</span><span class="p">])</span>
         <span class="k">else</span><span class="p">:</span>
             <span class="n">dttm</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">&#39;execution_date&#39;</span><span class="p">]</span>
 
@@ -346,15 +376,91 @@
         <span class="k">return</span> <span class="n">count</span>
 
 
+<span class="k">class</span> <span class="nc">NamedHivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Waits for a set of partitions to show up in Hive.</span>
+
+<span class="sd">    :param partition_names: List of fully qualified names of the</span>
+<span class="sd">        partitions to wait for. A fully qualified name is of the</span>
+<span class="sd">        form schema.table/pk1=pv1/pk2=pv2, for example,</span>
+<span class="sd">        default.users/ds=2016-01-01. This is passed as is to the metastore</span>
+<span class="sd">        Thrift client &quot;get_partitions_by_name&quot; method. Note that</span>
+<span class="sd">        you cannot use logical operators as in HivePartitionSensor.</span>
+<span class="sd">    :type partition_names: list of strings</span>
+<span class="sd">    :param metastore_conn_id: reference to the metastore thrift service</span>
+<span class="sd">        connection id</span>
+<span class="sd">    :type metastore_conn_id: str</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;partition_names&#39;</span><span class="p">,</span> <span class="p">)</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">partition_names</span><span class="p">,</span>
+            <span class="n">metastore_conn_id</span><span class="o">=</span><span class="s1">&#39;metastore_default&#39;</span><span class="p">,</span>
+            <span class="n">poke_interval</span><span class="o">=</span><span class="mi">60</span><span class="o">*</span><span class="mi">3</span><span class="p">,</span>
+            <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">NamedHivePartitionSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span>
+            <span class="n">poke_interval</span><span class="o">=</span><span class="n">poke_interval</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">partition_names</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
+            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;partition_names must be an array of strings&#39;</span><span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">partition_name</span> <span class="ow">in</span> <span class="n">partition_names</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">parse_partition_name</span><span class="p">(</span><span class="n">partition_name</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span> <span class="o">=</span> <span class="n">metastore_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span> <span class="o">=</span> <span class="n">partition_names</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">=</span> <span class="mi">0</span>
+
+    <span class="k">def</span> <span class="nf">parse_partition_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">partition</span><span class="p">):</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">schema</span><span class="p">,</span> <span class="n">table_partition</span> <span class="o">=</span> <span class="n">partition</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
+            <span class="n">table</span><span class="p">,</span> <span class="n">partition</span> <span class="o">=</span> <span class="n">table_partition</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span>
+        <span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Could not parse &#39;</span> <span class="o">+</span> <span class="n">partition</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">&#39;hook&#39;</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">hive_hooks</span><span class="o">.</span><span class="n">HiveMetastoreHook</span><span class="p">(</span>
+                <span class="n">metastore_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span><span class="p">)</span>
+
+        <span class="k">def</span> <span class="nf">poke_partition</span><span class="p">(</span><span class="n">partition</span><span class="p">):</span>
+
+            <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_partition_name</span><span class="p">(</span><span class="n">partition</span><span class="p">)</span>
+
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                <span class="s1">&#39;Poking for {schema}.{table}/{partition}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
+            <span class="p">)</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">check_for_named_partition</span><span class="p">(</span>
+                <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span><span class="p">)</span>
+
+        <span class="k">while</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span><span class="p">):</span>
+            <span class="k">if</span> <span class="n">poke_partition</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span><span class="p">]):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">+=</span> <span class="mi">1</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">return</span> <span class="bp">False</span>
+
+        <span class="k">return</span> <span class="bp">True</span>
+
+
 <span class="k">class</span> <span class="nc">HivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Waits for a partition to show up in Hive</span>
+<span class="sd">    Waits for a partition to show up in Hive.</span>
+
+<span class="sd">    Note: Because @partition supports general logical operators, it</span>
+<span class="sd">    can be inefficient. Consider using NamedHivePartitionSensor instead if</span>
+<span class="sd">    you don&#39;t need the full flexibility of HivePartitionSensor.</span>
 
 <span class="sd">    :param table: The name of the table to wait for, supports the dot</span>
 <span class="sd">        notation (my_database.my_table)</span>
 <span class="sd">    :type table: string</span>
 <span class="sd">    :param partition: The partition clause to wait for. This is passed as</span>
-<span class="sd">        is to the Metastore Thrift client &quot;get_partitions_by_filter&quot; method,</span>
+<span class="sd">        is to the metastore Thrift client &quot;get_partitions_by_filter&quot; method,</span>
 <span class="sd">        and apparently supports SQL like notation as in `ds=&#39;2015-01-01&#39;</span>
 <span class="sd">        AND type=&#39;value&#39;` and &gt; &lt; sings as in &quot;ds&gt;=2015-01-01&quot;</span>
 <span class="sd">    :type partition: string</span>
@@ -388,7 +494,7 @@
             <span class="s1">&#39;Poking for table {self.schema}.{self.table}, &#39;</span>
             <span class="s1">&#39;partition {self.partition}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
         <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">&#39;hook&#39;</span><span class="p">):</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HiveMetastoreHook</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">hive_hooks</span><span class="o">.</span><span class="n">HiveMetastoreHook</span><span class="p">(</span>
                 <span class="n">metastore_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span><span class="p">)</span>
         <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">check_for_partition</span><span class="p">(</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">partition</span><span class="p">)</span>
@@ -411,7 +517,8 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span> <span class="o">=</span> <span class="n">hdfs_conn_id</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">sb</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="kn">import</span> <span class="nn">airflow.hooks.hdfs_hook</span>
+        <span class="n">sb</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">hdfs_hook</span><span class="o">.</span><span class="n">HDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;snakebite&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">WARNING</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
             <span class="s1">&#39;Poking for file {self.filepath} &#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
@@ -439,7 +546,7 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span> <span class="o">=</span> <span class="n">webhdfs_conn_id</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">c</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">WebHDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span><span class="p">)</span>
+        <span class="n">c</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">webhdfs_hook</span><span class="o">.</span><span class="n">WebHDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
             <span class="s1">&#39;Poking for file {self.filepath} &#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
         <span class="k">return</span> <span class="n">c</span><span class="o">.</span><span class="n">check_for_path</span><span class="p">(</span><span class="n">hdfs_path</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">)</span>
@@ -495,7 +602,8 @@
         <span class="n">session</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
+        <span class="kn">import</span> <span class="nn">airflow.hooks.S3_hook</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">S3_hook</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
         <span class="n">full_url</span> <span class="o">=</span> <span class="s2">&quot;s3://&quot;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span> <span class="o">+</span> <span class="s2">&quot;/&quot;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_key</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Poking for key : {full_url}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">wildcard_match</span><span class="p">:</span>
@@ -547,7 +655,8 @@
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Poking for prefix : {self.prefix}</span><span class="se">\n</span><span class="s1">&#39;</span>
                      <span class="s1">&#39;in bucket s3://{self.bucket_name}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
-        <span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
+        <span class="kn">import</span> <span class="nn">airflow.hooks.S3_hook</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">S3_hook</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
         <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">check_for_prefix</span><span class="p">(</span>
             <span class="n">prefix</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prefix</span><span class="p">,</span>
             <span class="n">delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span><span class="p">,</span>
@@ -639,7 +748,7 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">extra_options</span> <span class="o">=</span> <span class="n">extra_options</span> <span class="ow">or</span> <span class="p">{}</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span> <span class="o">=</span> <span class="n">response_check</span>
 
-        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HttpHook</span><span class="p">(</span><span class="n">method</span><span class="o">=</span><span class="s1">&#39;GET&#39;</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="o">=</span><span class="n">http_conn_id</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">http_hook</span><span class="o">.</span><span class="n">HttpHook</span><span class="p">(</span><span class="n">method</span><span class="o">=</span><span class="s1">&#39;GET&#39;</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="o">=</span><span class="n">http_conn_id</span><span c [...]
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Poking: &#39;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">endpoint</span><span class="p">)</span>
@@ -652,7 +761,7 @@
                 <span class="c1"># run content check on response</span>
                 <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
         <span class="k">except</span> <span class="n">AirflowException</span> <span class="k">as</span> <span class="n">ae</span><span class="p">:</span>
-            <span class="k">if</span> <span class="n">ae</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;404&quot;</span><span class="p">):</span>
+            <span class="k">if</span> <span class="nb">str</span><span class="p">(</span><span class="n">ae</span><span class="p">)</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;404&quot;</span><span class="p">):</span>
                 <span class="k">return</span> <span class="bp">False</span>
 
             <span class="k">raise</span> <span class="n">ae</span>
diff --git a/_modules/bash_operator.html b/_modules/bash_operator.html
index bda570d..d38a3f8 100644
--- a/_modules/bash_operator.html
+++ b/_modules/bash_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for bash_operator</h1><div class="highlight"><pre>
-<span></span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">bytes</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">from</span> <span class="nn">subprocess</span> <span class="kn">import</span> <span class="n">Popen</span><span class="p">,</span> <span class="n">STDOUT</span><span class="p">,</span> <span class="n">PIPE</span>
diff --git a/_modules/dagrun_operator.html b/_modules/dagrun_operator.html
index be98472..c6725bb 100644
--- a/_modules/dagrun_operator.html
+++ b/_modules/dagrun_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for dagrun_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span><span class="p">,</span> <span class="n">DagRun</span>
diff --git a/_modules/dbapi_hook.html b/_modules/dbapi_hook.html
index 4a6a3a0..e608a23 100644
--- a/_modules/dbapi_hook.html
+++ b/_modules/dbapi_hook.html
@@ -137,12 +137,26 @@
            <div itemprop="articleBody">
             
   <h1>Source code for dbapi_hook</h1><div class="highlight"><pre>
-<span></span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
 <span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="nb">basestring</span>
 <span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span>
 <span class="kn">import</span> <span class="nn">numpy</span>
 <span class="kn">import</span> <span class="nn">logging</span>
+<span class="kn">import</span> <span class="nn">sys</span>
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
@@ -192,6 +206,8 @@
 <span class="sd">        :param parameters: The parameters to render the SQL query with.</span>
 <span class="sd">        :type parameters: mapping or iterable</span>
 <span class="sd">        &#39;&#39;&#39;</span>
+        <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&lt;</span> <span class="mi">3</span><span class="p">:</span>
+            <span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
         <span class="kn">import</span> <span class="nn">pandas.io.sql</span> <span class="kn">as</span> <span class="nn">psql</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
         <span class="n">df</span> <span class="o">=</span> <span class="n">psql</span><span class="o">.</span><span class="n">read_sql</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="n">con</span><span class="o">=</span><span class="n">conn</span><span class="p">,</span> <span class="n">params</span><span class="o">=</span><span class="n">parameters</span><span class="p">)</span>
@@ -208,6 +224,8 @@
 <span class="sd">        :param parameters: The parameters to render the SQL query with.</span>
 <span class="sd">        :type parameters: mapping or iterable</span>
 <span class="sd">        &#39;&#39;&#39;</span>
+        <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&lt;</span> <span class="mi">3</span><span class="p">:</span>
+            <span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
         <span class="n">cur</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_cursor</span><span class="p">()</span>
         <span class="k">if</span> <span class="n">parameters</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
@@ -229,6 +247,8 @@
 <span class="sd">        :param parameters: The parameters to render the SQL query with.</span>
 <span class="sd">        :type parameters: mapping or iterable</span>
 <span class="sd">        &#39;&#39;&#39;</span>
+        <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&lt;</span> <span class="mi">3</span><span class="p">:</span>
+            <span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
         <span class="n">cur</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
         <span class="k">if</span> <span class="n">parameters</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
@@ -264,6 +284,8 @@
 
         <span class="n">cur</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
         <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">sql</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&lt;</span> <span class="mi">3</span><span class="p">:</span>
+                <span class="n">s</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
             <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
             <span class="k">if</span> <span class="n">parameters</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
                 <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="n">parameters</span><span class="p">)</span>
diff --git a/_modules/druid_hook.html b/_modules/druid_hook.html
index 1508432..f5b2a4d 100644
--- a/_modules/druid_hook.html
+++ b/_modules/druid_hook.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for druid_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">json</span>
 <span class="kn">import</span> <span class="nn">time</span>
diff --git a/_modules/dummy_operator.html b/_modules/dummy_operator.html
index 4d8943a..a531fd1 100644
--- a/_modules/dummy_operator.html
+++ b/_modules/dummy_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for dummy_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
 
diff --git a/_modules/email_operator.html b/_modules/email_operator.html
index 59f4127..6bb4434 100644
--- a/_modules/email_operator.html
+++ b/_modules/email_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for email_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.email</span> <span class="kn">import</span> <span class="n">send_email</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
diff --git a/_modules/ftp_hook.html b/_modules/ftp_hook.html
index 1cc038a..19a7498 100644
--- a/_modules/ftp_hook.html
+++ b/_modules/ftp_hook.html
@@ -161,8 +161,8 @@
 
 
 <span class="k">def</span> <span class="nf">mlsd</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="n">path</span><span class="o">=</span><span class="s2">&quot;&quot;</span><span class="p">,</span> <span class="n">facts</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
-    <span class="sd">&#39;&#39;&#39;</span>
-<span class="sd">    BACKPORT FROM PYTHON3 FTPLIB</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    BACKPORT FROM PYTHON3 FTPLIB.</span>
 
 <span class="sd">    List a directory in a standardized format by using MLSD</span>
 <span class="sd">    command (RFC-3659). If path is omitted the current directory</span>
@@ -174,7 +174,7 @@
 <span class="sd">    First element is the file name, the second one is a dictionary</span>
 <span class="sd">    including a variable number of &quot;facts&quot; depending on the server</span>
 <span class="sd">    and whether &quot;facts&quot; argument has been provided.</span>
-<span class="sd">    &#39;&#39;&#39;</span>
+<span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">facts</span> <span class="o">=</span> <span class="n">facts</span> <span class="ow">or</span> <span class="p">[]</span>
     <span class="k">if</span> <span class="n">facts</span><span class="p">:</span>
         <span class="n">conn</span><span class="o">.</span><span class="n">sendcmd</span><span class="p">(</span><span class="s2">&quot;OPTS MLST &quot;</span> <span class="o">+</span> <span class="s2">&quot;;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">facts</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot;;&quot;</span><span class="p">)</span>
@@ -194,7 +194,6 @@
 
 
 <div class="viewcode-block" id="FTPHook"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.FTPHook">[docs]</a><span class="k">class</span> <span class="nc">FTPHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
-
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Interact with FTP.</span>
 
@@ -206,6 +205,13 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">ftp_conn_id</span> <span class="o">=</span> <span class="n">ftp_conn_id</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">None</span>
 
+    <span class="k">def</span> <span class="nf">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span>
+
+    <span class="k">def</span> <span class="nf">__exit__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">exc_type</span><span class="p">,</span> <span class="n">exc_val</span><span class="p">,</span> <span class="n">exc_tb</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">close_conn</span><span class="p">()</span>
+
 <div class="viewcode-block" id="FTPHook.get_conn"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.FTPHook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Returns a FTP connection object</span>
@@ -219,7 +225,7 @@
 <div class="viewcode-block" id="FTPHook.close_conn"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.FTPHook.close_conn">[docs]</a>    <span class="k">def</span> <span class="nf">close_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Closes the connection. An error will occur if the</span>
-<span class="sd">        connection wasnt ever opened.</span>
+<span class="sd">        connection wasn&#39;t ever opened.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span>
         <span class="n">conn</span><span class="o">.</span><span class="n">quit</span><span class="p">()</span></div>
@@ -301,7 +307,7 @@
         <span class="n">conn</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">remote_path</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Retrieving file from FTP: {}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">remote_full_path</span><span class="p">))</span>
         <span class="n">conn</span><span class="o">.</span><span class="n">retrbinary</span><span class="p">(</span><span class="s1">&#39;RETR </span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">remote_file_name</span><span class="p">,</span> <span class="n">output_handle</span><span class="o">.</span><span class="n">write</span><span class="p">)</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Finished etrieving file from FTP: {}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Finished retrieving file from FTP: {}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
             <span class="n">remote_full_path</span><span class="p">))</span>
 
         <span class="k">if</span> <span class="n">is_path</span><span class="p">:</span>
@@ -338,7 +344,7 @@
 
 <div class="viewcode-block" id="FTPHook.delete_file"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.FTPHook.delete_file">[docs]</a>    <span class="k">def</span> <span class="nf">delete_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Removes a file on the FTP Server</span>
+<span class="sd">        Removes a file on the FTP Server.</span>
 
 <span class="sd">        :param path: full path to the remote file</span>
 <span class="sd">        :type path: str</span>
@@ -356,7 +362,7 @@
 
     <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Returns a FTPS connection object</span>
+<span class="sd">        Returns a FTPS connection object.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
             <span class="n">params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ftp_conn_id</span><span class="p">)</span>
diff --git a/_modules/generic_transfer.html b/_modules/generic_transfer.html
index 4e2cf14..ed7f740 100644
--- a/_modules/generic_transfer.html
+++ b/_modules/generic_transfer.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for generic_transfer</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
 
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
diff --git a/_modules/hive_hooks.html b/_modules/hive_hooks.html
index 7f6cc2d..e25ec97 100644
--- a/_modules/hive_hooks.html
+++ b/_modules/hive_hooks.html
@@ -155,11 +155,13 @@
 <span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">zip</span>
 <span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="nb">basestring</span>
+
 <span class="kn">import</span> <span class="nn">unicodecsv</span> <span class="kn">as</span> <span class="nn">csv</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">re</span>
 <span class="kn">import</span> <span class="nn">subprocess</span>
 <span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span>
+<span class="kn">import</span> <span class="nn">hive_metastore</span>
 
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
 <span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
@@ -180,6 +182,7 @@
 <span class="sd">    Note that you can also set default hive CLI parameters using the</span>
 <span class="sd">    ``hive_cli_params`` to be used in your connection as in</span>
 <span class="sd">    ``{&quot;hive_cli_params&quot;: &quot;-hiveconf mapred.job.tracker=some.jobtracker:444&quot;}``</span>
+<span class="sd">    Parameters passed here can be overridden by run_cli&#39;s hive_conf param</span>
 
 <span class="sd">    The extra connection parameter ``auth`` gets passed as in the ``jdbc``</span>
 <span class="sd">    connection string as is.</span>
@@ -196,9 +199,17 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="n">conn</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">run_as</span> <span class="o">=</span> <span class="n">run_as</span>
 
-<div class="viewcode-block" id="HiveCliHook.run_cli"><a class="viewcode-back" href="../code.html#airflow.hooks.HiveCliHook.run_cli">[docs]</a>    <span class="k">def</span> <span class="nf">run_cli</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span> [...]
+<div class="viewcode-block" id="HiveCliHook.run_cli"><a class="viewcode-back" href="../code.html#airflow.hooks.HiveCliHook.run_cli">[docs]</a>    <span class="k">def</span> <span class="nf">run_cli</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span> [...]
         <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Run an hql statement using the hive cli</span>
+<span class="sd">        Run an hql statement using the hive cli. If hive_conf is specified it should be a</span>
+<span class="sd">        dict and the entries will be set as key/value pairs in HiveConf</span>
+
+
+<span class="sd">        :param hive_conf: if specified these key value pairs will be passed to hive as</span>
+<span class="sd">            ``-hiveconf &quot;key&quot;=&quot;value&quot;``. Note that they will be passed after the</span>
+<span class="sd">            ``hive_cli_params`` and thus will override whatever values are specified in</span>
+<span class="sd">            the database.</span>
+<span class="sd">        :type hive_conf: dict</span>
 
 <span class="sd">        &gt;&gt;&gt; hh = HiveCliHook()</span>
 <span class="sd">        &gt;&gt;&gt; result = hh.run_cli(&quot;USE airflow;&quot;)</span>
@@ -246,6 +257,10 @@
                     <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">:</span>
                         <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">&#39;-p&#39;</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">]</span>
 
+                <span class="n">hive_conf</span> <span class="o">=</span> <span class="n">hive_conf</span> <span class="ow">or</span> <span class="p">{}</span>
+                <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">hive_conf</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+                    <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">&#39;-hiveconf&#39;</span><span class="p">,</span> <span class="s1">&#39;{0}={1}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">)]</span>
+
                 <span class="n">hive_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="n">hive_bin</span><span class="p">,</span> <span class="s1">&#39;-f&#39;</span><span class="p">,</span> <span class="n">fname</span><span class="p">]</span> <span class="o">+</span> <span class="n">cmd_extra</span>
 
                 <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_params</span><span class="p">:</span>
@@ -446,7 +461,17 @@
         <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metastore</span>
 
 <div class="viewcode-block" id="HiveMetastoreHook.check_for_partition"><a class="viewcode-back" href="../code.html#airflow.hooks.HiveMetastoreHook.check_for_partition">[docs]</a>    <span class="k">def</span> <span class="nf">check_for_partition</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span><span class="p">):</span>
-        <span class="sd">&quot;&quot;&quot;Checks whether a partition exists</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Checks whether a partition exists</span>
+
+<span class="sd">        :param schema: Name of hive schema (database) @table belongs to</span>
+<span class="sd">        :type schema: string</span>
+<span class="sd">        :param table: Name of hive table @partition belongs to</span>
+<span class="sd">        :type schema: string</span>
+<span class="sd">        :partition: Expression that matches the partitions to check for</span>
+<span class="sd">            (eg `a = &#39;b&#39; AND c = &#39;d&#39;`)</span>
+<span class="sd">        :type schema: string</span>
+<span class="sd">        :rtype: boolean</span>
 
 <span class="sd">        &gt;&gt;&gt; hh = HiveMetastoreHook()</span>
 <span class="sd">        &gt;&gt;&gt; t = &#39;static_babynames_partitioned&#39;</span>
@@ -462,6 +487,35 @@
         <span class="k">else</span><span class="p">:</span>
             <span class="k">return</span> <span class="bp">False</span></div>
 
+<div class="viewcode-block" id="HiveMetastoreHook.check_for_named_partition"><a class="viewcode-back" href="../code.html#airflow.hooks.HiveMetastoreHook.check_for_named_partition">[docs]</a>    <span class="k">def</span> <span class="nf">check_for_named_partition</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition_name</span>< [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Checks whether a partition with a given name exists</span>
+
+<span class="sd">        :param schema: Name of hive schema (database) @table belongs to</span>
+<span class="sd">        :type schema: string</span>
+<span class="sd">        :param table: Name of hive table @partition belongs to</span>
+<span class="sd">        :type schema: string</span>
+<span class="sd">        :partition: Name of the partitions to check for (eg `a=b/c=d`)</span>
+<span class="sd">        :type schema: string</span>
+<span class="sd">        :rtype: boolean</span>
+
+<span class="sd">        &gt;&gt;&gt; hh = HiveMetastoreHook()</span>
+<span class="sd">        &gt;&gt;&gt; t = &#39;static_babynames_partitioned&#39;</span>
+<span class="sd">        &gt;&gt;&gt; hh.check_for_named_partition(&#39;airflow&#39;, t, &quot;ds=2015-01-01&quot;)</span>
+<span class="sd">        True</span>
+<span class="sd">        &gt;&gt;&gt; hh.check_for_named_partition(&#39;airflow&#39;, t, &quot;ds=xxx&quot;)</span>
+<span class="sd">        False</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">metastore</span><span class="o">.</span><span class="n">_oprot</span><span class="o">.</span><span class="n">trans</span><span class="o">.</span><span class="n">open</span><span class="p">()</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">metastore</span><span class="o">.</span><span class="n">get_partition_by_name</span><span class="p">(</span>
+                <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition_name</span><span class="p">)</span>
+            <span class="k">return</span> <span class="bp">True</span>
+        <span class="k">except</span> <span class="n">hive_metastore</span><span class="o">.</span><span class="n">ttypes</span><span class="o">.</span><span class="n">NoSuchObjectException</span><span class="p">:</span>
+            <span class="k">return</span> <span class="bp">False</span>
+        <span class="k">finally</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">metastore</span><span class="o">.</span><span class="n">_oprot</span><span class="o">.</span><span class="n">trans</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div>
+
 <div class="viewcode-block" id="HiveMetastoreHook.get_table"><a class="viewcode-back" href="../code.html#airflow.hooks.HiveMetastoreHook.get_table">[docs]</a>    <span class="k">def</span> <span class="nf">get_table</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table_name</span><span class="p">,</span> <span class="n">db</span><span class="o">=</span><span class="s1">&#39;default&#39;</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;Get a metastore table object</span>
 
@@ -604,6 +658,7 @@
             <span class="n">database</span><span class="o">=</span><span class="n">db</span><span class="o">.</span><span class="n">schema</span> <span class="ow">or</span> <span class="s1">&#39;default&#39;</span><span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">get_results</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="s1">&#39;default&#39;</span><span class="p">,</span> <span class="n">arraysize</span><span class="o">=</span><span class="mi">1000</span><span class="p">):</span>
+        <span class="kn">from</span> <span class="nn">impala.error</span> <span class="kn">import</span> <span class="n">ProgrammingError</span>
         <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
             <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hql</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
                 <span class="n">hql</span> <span class="o">=</span> <span class="p">[</span><span class="n">hql</span><span class="p">]</span>
@@ -611,15 +666,22 @@
                 <span class="s1">&#39;data&#39;</span><span class="p">:</span> <span class="p">[],</span>
                 <span class="s1">&#39;header&#39;</span><span class="p">:</span> <span class="p">[],</span>
             <span class="p">}</span>
+            <span class="n">cur</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
             <span class="k">for</span> <span class="n">statement</span> <span class="ow">in</span> <span class="n">hql</span><span class="p">:</span>
-                <span class="k">with</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span> <span class="k">as</span> <span class="n">cur</span><span class="p">:</span>
-                    <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">statement</span><span class="p">)</span>
+                <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">statement</span><span class="p">)</span>
+                <span class="n">records</span> <span class="o">=</span> <span class="p">[]</span>
+                <span class="k">try</span><span class="p">:</span>
+                    <span class="c1"># impala Lib raises when no results are returned</span>
+                    <span class="c1"># we&#39;re silencing here as some statements in the list</span>
+                    <span class="c1"># may be `SET` or DDL</span>
                     <span class="n">records</span> <span class="o">=</span> <span class="n">cur</span><span class="o">.</span><span class="n">fetchall</span><span class="p">()</span>
-                    <span class="k">if</span> <span class="n">records</span><span class="p">:</span>
-                        <span class="n">results</span> <span class="o">=</span> <span class="p">{</span>
-                            <span class="s1">&#39;data&#39;</span><span class="p">:</span> <span class="n">records</span><span class="p">,</span>
-                            <span class="s1">&#39;header&#39;</span><span class="p">:</span> <span class="n">cur</span><span class="o">.</span><span class="n">description</span><span class="p">,</span>
-                        <span class="p">}</span>
+                <span class="k">except</span> <span class="n">ProgrammingError</span><span class="p">:</span>
+                    <span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;get_results returned no records&quot;</span><span class="p">)</span>
+                <span class="k">if</span> <span class="n">records</span><span class="p">:</span>
+                    <span class="n">results</span> <span class="o">=</span> <span class="p">{</span>
+                        <span class="s1">&#39;data&#39;</span><span class="p">:</span> <span class="n">records</span><span class="p">,</span>
+                        <span class="s1">&#39;header&#39;</span><span class="p">:</span> <span class="n">cur</span><span class="o">.</span><span class="n">description</span><span class="p">,</span>
+                    <span class="p">}</span>
             <span class="k">return</span> <span class="n">results</span>
 
     <span class="k">def</span> <span class="nf">to_csv</span><span class="p">(</span>
diff --git a/_modules/hive_operator.html b/_modules/hive_operator.html
index d32e3ad..de5e44b 100644
--- a/_modules/hive_operator.html
+++ b/_modules/hive_operator.html
@@ -137,12 +137,27 @@
            <div itemprop="articleBody">
             
   <h1>Source code for hive_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">re</span>
 
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.operator_helpers</span> <span class="kn">import</span> <span class="n">context_to_airflow_vars</span>
 
 
 <div class="viewcode-block" id="HiveOperator"><a class="viewcode-back" href="../code.html#airflow.operators.HiveOperator">[docs]</a><span class="k">class</span> <span class="nc">HiveOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -201,7 +216,8 @@
     <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Executing: &#39;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">hql</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hql</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hql</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span>
+                          <span class="n">hive_conf</span><span class="o">=</span><span class="n">context_to_airflow_vars</span><span class="p">(</span><span class="n">context</span><span class="p">))</span>
 
     <span class="k">def</span> <span class="nf">dry_run</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span>
diff --git a/_modules/hive_to_druid.html b/_modules/hive_to_druid.html
index cbd1586..d85bfb3 100644
--- a/_modules/hive_to_druid.html
+++ b/_modules/hive_to_druid.html
@@ -137,9 +137,24 @@
            <div itemprop="articleBody">
             
   <h1>Source code for hive_to_druid</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">DruidHook</span><span class="p">,</span> <span class="n">HiveMetastoreHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">HiveMetastoreHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.druid_hook</span> <span class="kn">import</span> <span class="n">DruidHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
@@ -240,19 +255,20 @@
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Inserting rows into Druid&quot;</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;HDFS path: &quot;</span> <span class="o">+</span> <span class="n">static_path</span><span class="p">)</span>
 
-        <span class="n">druid</span><span class="o">.</span><span class="n">load_from_hdfs</span><span class="p">(</span>
-            <span class="n">datasource</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">druid_datasource</span><span class="p">,</span>
-            <span class="n">intervals</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">intervals</span><span class="p">,</span>
-            <span class="n">static_path</span><span class="o">=</span><span class="n">static_path</span><span class="p">,</span> <span class="n">ts_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">ts_dim</span><span class="p">,</span>
-            <span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">target_partition_size</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">target_partition_size</span><span class="p">,</span>
-            <span class="n">metric_spec</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metric_spec</span><span class="p">,</span> <span class="n">hadoop_dependency_coordinates</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hadoop_dependency_coordinates</span><span class="p">)</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Load seems to have succeeded!&quot;</span><span class="p">)</span>
-
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
-            <span class="s2">&quot;Cleaning up by dropping the temp &quot;</span>
-            <span class="s2">&quot;Hive table {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">))</span>
-        <span class="n">hql</span> <span class="o">=</span> <span class="s2">&quot;DROP TABLE IF EXISTS {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">)</span>
-        <span class="n">hive</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="p">)</span></div>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">druid</span><span class="o">.</span><span class="n">load_from_hdfs</span><span class="p">(</span>
+                <span class="n">datasource</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">druid_datasource</span><span class="p">,</span>
+                <span class="n">intervals</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">intervals</span><span class="p">,</span>
+                <span class="n">static_path</span><span class="o">=</span><span class="n">static_path</span><span class="p">,</span> <span class="n">ts_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">ts_dim</span><span class="p">,</span>
+                <span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">target_partition_size</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">target_partition_size</span><span class="p">,</span>
+                <span class="n">metric_spec</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metric_spec</span><span class="p">,</span> <span class="n">hadoop_dependency_coordinates</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hadoop_dependency_coordinates</span><span class="p">)</span>
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Load seems to have succeeded!&quot;</span><span class="p">)</span>
+        <span class="k">finally</span><span class="p">:</span>
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                <span class="s2">&quot;Cleaning up by dropping the temp &quot;</span>
+                <span class="s2">&quot;Hive table {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">))</span>
+            <span class="n">hql</span> <span class="o">=</span> <span class="s2">&quot;DROP TABLE IF EXISTS {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">)</span>
+            <span class="n">hive</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="p">)</span></div>
 </pre></div>
 
            </div>
diff --git a/_modules/hive_to_mysql.html b/_modules/hive_to_mysql.html
index 6a5bedd..867023b 100644
--- a/_modules/hive_to_mysql.html
+++ b/_modules/hive_to_mysql.html
@@ -137,9 +137,24 @@
            <div itemprop="articleBody">
             
   <h1>Source code for hive_to_mysql</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span><span class="p">,</span> <span class="n">MySqlHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
diff --git a/_modules/hive_to_samba_operator.html b/_modules/hive_to_samba_operator.html
index 3008244..a69680d 100644
--- a/_modules/hive_to_samba_operator.html
+++ b/_modules/hive_to_samba_operator.html
@@ -137,10 +137,25 @@
            <div itemprop="articleBody">
             
   <h1>Source code for hive_to_samba_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">tempfile</span>
 
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span><span class="p">,</span> <span class="n">SambaHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.samba_hook</span> <span class="kn">import</span> <span class="n">SambaHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
diff --git a/_modules/http_hook.html b/_modules/http_hook.html
index 98ed5ad..b62b0d0 100644
--- a/_modules/http_hook.html
+++ b/_modules/http_hook.html
@@ -150,7 +150,7 @@
 <span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
 <span class="c1"># See the License for the specific language governing permissions and</span>
 <span class="c1"># limitations under the License.</span>
-<span class="c1">#</span>
+
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 
diff --git a/_modules/http_operator.html b/_modules/http_operator.html
index 942ae0e..00496c9 100644
--- a/_modules/http_operator.html
+++ b/_modules/http_operator.html
@@ -137,10 +137,24 @@
            <div itemprop="articleBody">
             
   <h1>Source code for http_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
 
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HttpHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.http_hook</span> <span class="kn">import</span> <span class="n">HttpHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
diff --git a/_modules/mssql_hook.html b/_modules/mssql_hook.html
index b3b5503..9aff59d 100644
--- a/_modules/mssql_hook.html
+++ b/_modules/mssql_hook.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for mssql_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">pymssql</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">pymssql</span>
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>
 
@@ -157,10 +171,11 @@
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mssql_conn_id</span><span class="p">)</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="n">pymssql</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span>
-            <span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
-            <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
-            <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span>
-            <span class="n">conn</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span>
+            <span class="n">server</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+            <span class="n">user</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
+            <span class="n">password</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span>
+            <span class="n">database</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span>
+            <span class="n">port</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span>
         <span class="k">return</span> <span class="n">conn</span></div>
 
     <span class="k">def</span> <span class="nf">set_autocommit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">):</span>
diff --git a/_modules/mssql_operator.html b/_modules/mssql_operator.html
index dcc2872..e933004 100644
--- a/_modules/mssql_operator.html
+++ b/_modules/mssql_operator.html
@@ -137,9 +137,23 @@
            <div itemprop="articleBody">
             
   <h1>Source code for mssql_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">MsSqlHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.mssql_hook</span> <span class="kn">import</span> <span class="n">MsSqlHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
diff --git a/_modules/mssql_to_hive.html b/_modules/mssql_to_hive.html
index 451dc7f..590dfb8 100644
--- a/_modules/mssql_to_hive.html
+++ b/_modules/mssql_to_hive.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for mssql_to_hive</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
 <span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
 <span class="kn">import</span> <span class="nn">unicodecsv</span> <span class="kn">as</span> <span class="nn">csv</span>
 <span class="kn">import</span> <span class="nn">logging</span>
@@ -145,7 +159,8 @@
 <span class="kn">import</span> <span class="nn">pymssql</span>
 
 
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">MsSqlHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.mssql_hook</span> <span class="kn">import</span> <span class="n">MsSqlHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
diff --git a/_modules/mysql_hook.html b/_modules/mysql_hook.html
index 23d3726..941737f 100644
--- a/_modules/mysql_hook.html
+++ b/_modules/mysql_hook.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for mysql_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">MySQLdb</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">MySQLdb</span>
 <span class="kn">import</span> <span class="nn">MySQLdb.cursors</span>
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>
diff --git a/_modules/mysql_operator.html b/_modules/mysql_operator.html
index 331f2b4..a1fc11a 100644
--- a/_modules/mysql_operator.html
+++ b/_modules/mysql_operator.html
@@ -137,9 +137,23 @@
            <div itemprop="articleBody">
             
   <h1>Source code for mysql_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">MySqlHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
diff --git a/_modules/mysql_to_hive.html b/_modules/mysql_to_hive.html
index fa6b20f..cc8496b 100644
--- a/_modules/mysql_to_hive.html
+++ b/_modules/mysql_to_hive.html
@@ -137,14 +137,29 @@
            <div itemprop="articleBody">
             
   <h1>Source code for mysql_to_hive</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
 <span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
 <span class="kn">import</span> <span class="nn">unicodecsv</span> <span class="kn">as</span> <span class="nn">csv</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span>
 <span class="kn">import</span> <span class="nn">MySQLdb</span>
 
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">MySqlHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
diff --git a/_modules/postgres_hook.html b/_modules/postgres_hook.html
index d8896ac..3ecfa69 100644
--- a/_modules/postgres_hook.html
+++ b/_modules/postgres_hook.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for postgres_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">psycopg2</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">psycopg2</span>
 <span class="kn">import</span> <span class="nn">psycopg2.extensions</span>
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>
diff --git a/_modules/postgres_operator.html b/_modules/postgres_operator.html
index 4eb0e00..5ade37d 100644
--- a/_modules/postgres_operator.html
+++ b/_modules/postgres_operator.html
@@ -137,9 +137,23 @@
            <div itemprop="articleBody">
             
   <h1>Source code for postgres_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">PostgresHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.postgres_hook</span> <span class="kn">import</span> <span class="n">PostgresHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
diff --git a/_modules/presto_check_operator.html b/_modules/presto_check_operator.html
index 52dc0e6..e24b271 100644
--- a/_modules/presto_check_operator.html
+++ b/_modules/presto_check_operator.html
@@ -137,8 +137,22 @@
            <div itemprop="articleBody">
             
   <h1>Source code for presto_check_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">PrestoHook</span>
-<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="kn">import</span> <span class="n">CheckOperator</span><span class="p">,</span> <span class="n">ValueCheckOperator</span><span class="p">,</span> <span class="n">IntervalCheckOperator</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.presto_hook</span> <span class="kn">import</span> <span class="n">PrestoHook</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.check_operator</span> <span class="kn">import</span> <span class="n">CheckOperator</span><span class="p">,</span> <span class="n">ValueCheckOperator</span><span class="p">,</span> <span class="n">IntervalCheckOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
 
diff --git a/_modules/presto_hook.html b/_modules/presto_hook.html
index 73e4cc0..49b93a9 100644
--- a/_modules/presto_hook.html
+++ b/_modules/presto_hook.html
@@ -137,13 +137,28 @@
            <div itemprop="articleBody">
             
   <h1>Source code for presto_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span class="kn">import</span> <span class="nn">logging</span>
+
 <span class="kn">from</span> <span class="nn">pyhive</span> <span class="kn">import</span> <span class="n">presto</span>
 <span class="kn">from</span> <span class="nn">pyhive.exc</span> <span class="kn">import</span> <span class="n">DatabaseError</span>
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>
 
-<span class="kn">import</span> <span class="nn">logging</span>
 <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;pyhive&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">)</span>
 
 
@@ -204,8 +219,7 @@
             <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">PrestoHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">get_first</span><span class="p">(</span>
                 <span class="bp">self</span><span class="o">.</span><span class="n">_strip_sql</span><span class="p">(</span><span class="n">hql</span><span class="p">),</span> <span class="n">parameters</span><span class="p">)</span>
         <span class="k">except</span> <span class="n">DatabaseError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
-            <span class="n">obj</span> <span class="o">=</span> <span class="nb">eval</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
-            <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="n">obj</span><span class="p">[</span><span class="s1">&#39;message&#39;</span><span class="p">])</span></div>
+            <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="n">e</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">&#39;message&#39;</span><span class="p">])</span></div>
 
 <div class="viewcode-block" id="PrestoHook.get_pandas_df"><a class="viewcode-back" href="../code.html#airflow.hooks.PrestoHook.get_pandas_df">[docs]</a>    <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
@@ -217,8 +231,7 @@
             <span class="n">cursor</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_strip_sql</span><span class="p">(</span><span class="n">hql</span><span class="p">),</span> <span class="n">parameters</span><span class="p">)</span>
             <span class="n">data</span> <span class="o">=</span> <span class="n">cursor</span><span class="o">.</span><span class="n">fetchall</span><span class="p">()</span>
         <span class="k">except</span> <span class="n">DatabaseError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
-            <span class="n">obj</span> <span class="o">=</span> <span class="nb">eval</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
-            <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="n">obj</span><span class="p">[</span><span class="s1">&#39;message&#39;</span><span class="p">])</span>
+            <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="n">e</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">&#39;message&#39;</span><span class="p">])</span>
         <span class="n">column_descriptions</span> <span class="o">=</span> <span class="n">cursor</span><span class="o">.</span><span class="n">description</span>
         <span class="k">if</span> <span class="n">data</span><span class="p">:</span>
             <span class="n">df</span> <span class="o">=</span> <span class="n">pandas</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
diff --git a/_modules/python_operator.html b/_modules/python_operator.html
index 0f0de31..059ebb2 100644
--- a/_modules/python_operator.html
+++ b/_modules/python_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for python_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
 <span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 
diff --git a/_modules/qubole_operator.html b/_modules/qubole_operator.html
index 07ea626..fb33047 100644
--- a/_modules/qubole_operator.html
+++ b/_modules/qubole_operator.html
@@ -137,9 +137,23 @@
            <div itemprop="articleBody">
             
   <h1>Source code for qubole_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks</span> <span class="kn">import</span> <span class="n">QuboleHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.qubole_hook</span> <span class="kn">import</span> <span class="n">QuboleHook</span>
 
 
 <div class="viewcode-block" id="QuboleOperator"><a class="viewcode-back" href="../code.html#airflow.contrib.operators.QuboleOperator">[docs]</a><span class="k">class</span> <span class="nc">QuboleOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -220,7 +234,7 @@
 
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;query&#39;</span><span class="p">,</span> <span class="s1">&#39;script_location&#39;</span><span class="p">,</span> <span class="s1">&#39;sub_command&#39;</span><span class="p">,</span> <span class="s1">&#39;script&#39;</span><span class="p">,</span> <span class="s1">&#39;files&#39;</span><span class="p">,</span> <span class="s1">&#39;archives&#39;</span><span class="p">,</sp [...]
                        <span class="s1">&#39;sql&#39;</span><span class="p">,</span> <span class="s1">&#39;where_clause&#39;</span><span class="p">,</span> <span class="s1">&#39;extract_query&#39;</span><span class="p">,</span> <span class="s1">&#39;boundary_query&#39;</span><span class="p">,</span> <span class="s1">&#39;macros&#39;</span><span class="p">,</span> <span class="s1">&#39;tags&#39;</span><span class="p">,</span> <span class="s1">&#39;name&#39;</span><span class="p">,</span>  [...]
-    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.txt&#39;</span><span class="p">)</span>
+    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.txt&#39;</span><span class="p">,)</span>
     <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#3064A1&#39;</span>
     <span class="n">ui_fgcolor</span> <span class="o">=</span> <span class="s1">&#39;#fff&#39;</span>
 
diff --git a/_modules/s3_to_hive_operator.html b/_modules/s3_to_hive_operator.html
index dc23eb5..ad7715e 100644
--- a/_modules/s3_to_hive_operator.html
+++ b/_modules/s3_to_hive_operator.html
@@ -137,13 +137,28 @@
            <div itemprop="articleBody">
             
   <h1>Source code for s3_to_hive_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">next</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">next</span>
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">zip</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span>
 
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">S3Hook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.S3_hook</span> <span class="kn">import</span> <span class="n">S3Hook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
diff --git a/_modules/sensors.html b/_modules/sensors.html
index 19336b2..fab6313 100644
--- a/_modules/sensors.html
+++ b/_modules/sensors.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for sensors</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
 <span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
 <span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
@@ -146,10 +160,11 @@
 <span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlparse</span>
 <span class="kn">from</span> <span class="nn">time</span> <span class="kn">import</span> <span class="n">sleep</span>
 
+<span class="kn">import</span> <span class="nn">airflow</span>
 <span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">hooks</span><span class="p">,</span> <span class="n">settings</span>
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span><span class="p">,</span> <span class="n">AirflowSensorTimeout</span><span class="p">,</span> <span class="n">AirflowSkipException</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span><span class="p">,</span> <span class="n">TaskInstance</span><span class="p">,</span> <span class="n">Connection</span> <span class="k">as</span> <span class="n">DB</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="kn">import</span> <span class="n">State</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
@@ -193,12 +208,12 @@
     <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="n">started_at</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span>
         <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">poke</span><span class="p">(</span><span class="n">context</span><span class="p">):</span>
-            <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">poke_interval</span><span class="p">)</span>
-            <span class="k">if</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">started_at</span><span class="p">)</span><span class="o">.</span><span class="n">seconds</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">:</span>
+            <span class="k">if</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">started_at</span><span class="p">)</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">()</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">:</span>
                 <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">soft_fail</span><span class="p">:</span>
                     <span class="k">raise</span> <span class="n">AirflowSkipException</span><span class="p">(</span><span class="s1">&#39;Snap. Time is OUT.&#39;</span><span class="p">)</span>
                 <span class="k">else</span><span class="p">:</span>
                     <span class="k">raise</span> <span class="n">AirflowSensorTimeout</span><span class="p">(</span><span class="s1">&#39;Snap. Time is OUT.&#39;</span><span class="p">)</span>
+            <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">poke_interval</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Success criteria met. Exiting.&quot;</span><span class="p">)</span>
 
 
@@ -303,8 +318,14 @@
 <span class="sd">    :type allowed_states: list</span>
 <span class="sd">    :param execution_delta: time difference with the previous execution to</span>
 <span class="sd">        look at, the default is the same execution_date as the current task.</span>
-<span class="sd">        For yesterday, use [positive!] datetime.timedelta(days=1)</span>
+<span class="sd">        For yesterday, use [positive!] datetime.timedelta(days=1). Either</span>
+<span class="sd">        execution_delta or execution_date_fn can be passed to</span>
+<span class="sd">        ExternalTaskSensor, but not both.</span>
 <span class="sd">    :type execution_delta: datetime.timedelta</span>
+<span class="sd">    :param execution_date_fn: function that receives the current execution date</span>
+<span class="sd">        and returns the desired execution date to query. Either execution_delta</span>
+<span class="sd">        or execution_date_fn can be passed to ExternalTaskSensor, but not both.</span>
+<span class="sd">    :type execution_date_fn: callable</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="nd">@apply_defaults</span>
@@ -314,16 +335,25 @@
             <span class="n">external_task_id</span><span class="p">,</span>
             <span class="n">allowed_states</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">execution_delta</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">execution_date_fn</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
         <span class="nb">super</span><span class="p">(</span><span class="n">ExternalTaskSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">allowed_states</span> <span class="o">=</span> <span class="n">allowed_states</span> <span class="ow">or</span> <span class="p">[</span><span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">execution_delta</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span> <span class="ow">and</span> <span class="n">execution_date_fn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s1">&#39;Only one of `execution_date` or `execution_date_fn` may&#39;</span>
+                <span class="s1">&#39;be provided to ExternalTaskSensor; not both.&#39;</span><span class="p">)</span>
+
         <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span> <span class="o">=</span> <span class="n">execution_delta</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span> <span class="o">=</span> <span class="n">execution_date_fn</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">external_dag_id</span> <span class="o">=</span> <span class="n">external_dag_id</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">external_task_id</span> <span class="o">=</span> <span class="n">external_task_id</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span><span class="p">:</span>
             <span class="n">dttm</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">&#39;execution_date&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span><span class="p">:</span>
+            <span class="n">dttm</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span><span class="p">(</span><span class="n">context</span><span class="p">[</span><span class="s1">&#39;execution_date&#39;</span><span class="p">])</span>
         <span class="k">else</span><span class="p">:</span>
             <span class="n">dttm</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">&#39;execution_date&#39;</span><span class="p">]</span>
 
@@ -346,15 +376,91 @@
         <span class="k">return</span> <span class="n">count</span></div>
 
 
+<span class="k">class</span> <span class="nc">NamedHivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Waits for a set of partitions to show up in Hive.</span>
+
+<span class="sd">    :param partition_names: List of fully qualified names of the</span>
+<span class="sd">        partitions to wait for. A fully qualified name is of the</span>
+<span class="sd">        form schema.table/pk1=pv1/pk2=pv2, for example,</span>
+<span class="sd">        default.users/ds=2016-01-01. This is passed as is to the metastore</span>
+<span class="sd">        Thrift client &quot;get_partitions_by_name&quot; method. Note that</span>
+<span class="sd">        you cannot use logical operators as in HivePartitionSensor.</span>
+<span class="sd">    :type partition_names: list of strings</span>
+<span class="sd">    :param metastore_conn_id: reference to the metastore thrift service</span>
+<span class="sd">        connection id</span>
+<span class="sd">    :type metastore_conn_id: str</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;partition_names&#39;</span><span class="p">,</span> <span class="p">)</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">partition_names</span><span class="p">,</span>
+            <span class="n">metastore_conn_id</span><span class="o">=</span><span class="s1">&#39;metastore_default&#39;</span><span class="p">,</span>
+            <span class="n">poke_interval</span><span class="o">=</span><span class="mi">60</span><span class="o">*</span><span class="mi">3</span><span class="p">,</span>
+            <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">NamedHivePartitionSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span>
+            <span class="n">poke_interval</span><span class="o">=</span><span class="n">poke_interval</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">partition_names</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
+            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;partition_names must be an array of strings&#39;</span><span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">partition_name</span> <span class="ow">in</span> <span class="n">partition_names</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">parse_partition_name</span><span class="p">(</span><span class="n">partition_name</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span> <span class="o">=</span> <span class="n">metastore_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span> <span class="o">=</span> <span class="n">partition_names</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">=</span> <span class="mi">0</span>
+
+    <span class="k">def</span> <span class="nf">parse_partition_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">partition</span><span class="p">):</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">schema</span><span class="p">,</span> <span class="n">table_partition</span> <span class="o">=</span> <span class="n">partition</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
+            <span class="n">table</span><span class="p">,</span> <span class="n">partition</span> <span class="o">=</span> <span class="n">table_partition</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span>
+        <span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Could not parse &#39;</span> <span class="o">+</span> <span class="n">partition</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">&#39;hook&#39;</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">hive_hooks</span><span class="o">.</span><span class="n">HiveMetastoreHook</span><span class="p">(</span>
+                <span class="n">metastore_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span><span class="p">)</span>
+
+        <span class="k">def</span> <span class="nf">poke_partition</span><span class="p">(</span><span class="n">partition</span><span class="p">):</span>
+
+            <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_partition_name</span><span class="p">(</span><span class="n">partition</span><span class="p">)</span>
+
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                <span class="s1">&#39;Poking for {schema}.{table}/{partition}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
+            <span class="p">)</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">check_for_named_partition</span><span class="p">(</span>
+                <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span><span class="p">)</span>
+
+        <span class="k">while</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span><span class="p">):</span>
+            <span class="k">if</span> <span class="n">poke_partition</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span><span class="p">]):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">+=</span> <span class="mi">1</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">return</span> <span class="bp">False</span>
+
+        <span class="k">return</span> <span class="bp">True</span>
+
+
 <div class="viewcode-block" id="HivePartitionSensor"><a class="viewcode-back" href="../code.html#airflow.operators.HivePartitionSensor">[docs]</a><span class="k">class</span> <span class="nc">HivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Waits for a partition to show up in Hive</span>
+<span class="sd">    Waits for a partition to show up in Hive.</span>
+
+<span class="sd">    Note: Because @partition supports general logical operators, it</span>
+<span class="sd">    can be inefficient. Consider using NamedHivePartitionSensor instead if</span>
+<span class="sd">    you don&#39;t need the full flexibility of HivePartitionSensor.</span>
 
 <span class="sd">    :param table: The name of the table to wait for, supports the dot</span>
 <span class="sd">        notation (my_database.my_table)</span>
 <span class="sd">    :type table: string</span>
 <span class="sd">    :param partition: The partition clause to wait for. This is passed as</span>
-<span class="sd">        is to the Metastore Thrift client &quot;get_partitions_by_filter&quot; method,</span>
+<span class="sd">        is to the metastore Thrift client &quot;get_partitions_by_filter&quot; method,</span>
 <span class="sd">        and apparently supports SQL like notation as in `ds=&#39;2015-01-01&#39;</span>
 <span class="sd">        AND type=&#39;value&#39;` and &gt; &lt; sings as in &quot;ds&gt;=2015-01-01&quot;</span>
 <span class="sd">    :type partition: string</span>
@@ -388,7 +494,7 @@
             <span class="s1">&#39;Poking for table {self.schema}.{self.table}, &#39;</span>
             <span class="s1">&#39;partition {self.partition}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
         <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">&#39;hook&#39;</span><span class="p">):</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HiveMetastoreHook</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">hive_hooks</span><span class="o">.</span><span class="n">HiveMetastoreHook</span><span class="p">(</span>
                 <span class="n">metastore_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span><span class="p">)</span>
         <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">check_for_partition</span><span class="p">(</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">partition</span><span class="p">)</span></div>
@@ -411,7 +517,8 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span> <span class="o">=</span> <span class="n">hdfs_conn_id</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">sb</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="kn">import</span> <span class="nn">airflow.hooks.hdfs_hook</span>
+        <span class="n">sb</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">hdfs_hook</span><span class="o">.</span><span class="n">HDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;snakebite&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">WARNING</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
             <span class="s1">&#39;Poking for file {self.filepath} &#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
@@ -439,7 +546,7 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span> <span class="o">=</span> <span class="n">webhdfs_conn_id</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">c</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">WebHDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span><span class="p">)</span>
+        <span class="n">c</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">webhdfs_hook</span><span class="o">.</span><span class="n">WebHDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
             <span class="s1">&#39;Poking for file {self.filepath} &#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
         <span class="k">return</span> <span class="n">c</span><span class="o">.</span><span class="n">check_for_path</span><span class="p">(</span><span class="n">hdfs_path</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">)</span></div>
@@ -495,7 +602,8 @@
         <span class="n">session</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
+        <span class="kn">import</span> <span class="nn">airflow.hooks.S3_hook</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">S3_hook</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
         <span class="n">full_url</span> <span class="o">=</span> <span class="s2">&quot;s3://&quot;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span> <span class="o">+</span> <span class="s2">&quot;/&quot;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_key</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Poking for key : {full_url}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">wildcard_match</span><span class="p">:</span>
@@ -547,7 +655,8 @@
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Poking for prefix : {self.prefix}</span><span class="se">\n</span><span class="s1">&#39;</span>
                      <span class="s1">&#39;in bucket s3://{self.bucket_name}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
-        <span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
+        <span class="kn">import</span> <span class="nn">airflow.hooks.S3_hook</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">S3_hook</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
         <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">check_for_prefix</span><span class="p">(</span>
             <span class="n">prefix</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prefix</span><span class="p">,</span>
             <span class="n">delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span><span class="p">,</span>
@@ -639,7 +748,7 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">extra_options</span> <span class="o">=</span> <span class="n">extra_options</span> <span class="ow">or</span> <span class="p">{}</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span> <span class="o">=</span> <span class="n">response_check</span>
 
-        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HttpHook</span><span class="p">(</span><span class="n">method</span><span class="o">=</span><span class="s1">&#39;GET&#39;</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="o">=</span><span class="n">http_conn_id</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">http_hook</span><span class="o">.</span><span class="n">HttpHook</span><span class="p">(</span><span class="n">method</span><span class="o">=</span><span class="s1">&#39;GET&#39;</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="o">=</span><span class="n">http_conn_id</span><span c [...]
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Poking: &#39;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">endpoint</span><span class="p">)</span>
@@ -652,7 +761,7 @@
                 <span class="c1"># run content check on response</span>
                 <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
         <span class="k">except</span> <span class="n">AirflowException</span> <span class="k">as</span> <span class="n">ae</span><span class="p">:</span>
-            <span class="k">if</span> <span class="n">ae</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;404&quot;</span><span class="p">):</span>
+            <span class="k">if</span> <span class="nb">str</span><span class="p">(</span><span class="n">ae</span><span class="p">)</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;404&quot;</span><span class="p">):</span>
                 <span class="k">return</span> <span class="bp">False</span>
 
             <span class="k">raise</span> <span class="n">ae</span>
diff --git a/_modules/slack_operator.html b/_modules/slack_operator.html
index f958c5b..152902b 100644
--- a/_modules/slack_operator.html
+++ b/_modules/slack_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for slack_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">slackclient</span> <span class="kn">import</span> <span class="n">SlackClient</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">slackclient</span> <span class="kn">import</span> <span class="n">SlackClient</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
diff --git a/_modules/sqlite_hook.html b/_modules/sqlite_hook.html
index 6ed5de8..70abb8c 100644
--- a/_modules/sqlite_hook.html
+++ b/_modules/sqlite_hook.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for sqlite_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">sqlite3</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">sqlite3</span>
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>
 
diff --git a/_modules/ssh_execute_operator.html b/_modules/ssh_execute_operator.html
index 4346f31..c4c952d 100644
--- a/_modules/ssh_execute_operator.html
+++ b/_modules/ssh_execute_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for ssh_execute_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">bytes</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">bytes</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">subprocess</span>
 <span class="kn">from</span> <span class="nn">subprocess</span> <span class="kn">import</span> <span class="n">STDOUT</span>
diff --git a/_modules/vertica_operator.html b/_modules/vertica_operator.html
index 2ae2681..46f8a92 100644
--- a/_modules/vertica_operator.html
+++ b/_modules/vertica_operator.html
@@ -137,9 +137,23 @@
            <div itemprop="articleBody">
             
   <h1>Source code for vertica_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks</span> <span class="kn">import</span> <span class="n">VerticaHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.vertica_hook</span> <span class="kn">import</span> <span class="n">VerticaHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
diff --git a/_modules/vertica_to_hive.html b/_modules/vertica_to_hive.html
index 57c83f1..a700374 100644
--- a/_modules/vertica_to_hive.html
+++ b/_modules/vertica_to_hive.html
@@ -137,14 +137,28 @@
            <div itemprop="articleBody">
             
   <h1>Source code for vertica_to_hive</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
 <span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
 <span class="kn">import</span> <span class="nn">unicodecsv</span> <span class="kn">as</span> <span class="nn">csv</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span>
 
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks</span> <span class="kn">import</span> <span class="n">VerticaHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.vertica_hook</span> <span class="kn">import</span> <span class="n">VerticaHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
diff --git a/_modules/webhdfs_hook.html b/_modules/webhdfs_hook.html
index d54cb05..3a44028 100644
--- a/_modules/webhdfs_hook.html
+++ b/_modules/webhdfs_hook.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for webhdfs_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
 <span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">configuration</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 
diff --git a/_sources/code.txt b/_sources/code.txt
index 2876876..ba1cddb 100644
--- a/_sources/code.txt
+++ b/_sources/code.txt
@@ -133,6 +133,10 @@ Variable                            Description
 ``{{ latest_date }}``               same as ``{{ ds }}``
 ``{{ ti }}``                        same as ``{{ task_instance }}``
 ``{{ params }}``                    a reference to the user-defined params dictionary
+``{{ var.value.my_var }}``          global defined variables represented as a dictionary
+``{{ var.json.my_var.path }}``      global defined variables represented as a dictionary
+                                    with deserialized JSON object, append the path to the
+                                    key within the JSON object
 ``{{ task_instance_key_str }}``     a unique, human-readable key to the task instance
                                     formatted ``{dag_id}_{task_id}_{ds}``
 ``conf``                            the full configuration object located at
@@ -151,6 +155,11 @@ dot notation. Here are some examples of what is possible:
 Refer to the models documentation for more information on the objects'
 attributes and methods.
 
+The ``var`` template variable allows you to access variables defined in Airflow's
+UI. You can access them as either plain-text or JSON. If you use JSON, you are
+also able to walk nested structures, such as dictionaries like:
+``{{ var.json.my_dict_var.key1 }}``
+
 Macros
 ''''''
 Macros are a way to expose objects to your templates and live under the
diff --git a/_sources/concepts.txt b/_sources/concepts.txt
index 811130d..8cfc8ab 100644
--- a/_sources/concepts.txt
+++ b/_sources/concepts.txt
@@ -485,7 +485,7 @@ the main UI. For example:
 
   #dags/subdag.py
   from airflow.models import DAG
-  from airflow.operators import DummyOperator
+  from airflow.operators.dummy_operator import DummyOperator
 
 
   # Dag is returned by a factory method
@@ -510,7 +510,7 @@ This SubDAG can then be referenced in your main DAG file:
   # main_dag.py
   from datetime import datetime, timedelta
   from airflow.models import DAG
-  from airflow.operators import SubDagOperator
+  from airflow.operators.subdag_operator import SubDagOperator
   from dags.subdag import sub_dag
 
 
diff --git a/_sources/faq.txt b/_sources/faq.txt
index 21623fc..e61c1bf 100644
--- a/_sources/faq.txt
+++ b/_sources/faq.txt
@@ -78,12 +78,17 @@ We recommend against using dynamic values as ``start_date``, especially
 once the period closes, and in theory an ``@hourly`` DAG would never get to
 an hour after now as ``now()`` moves along.
 
-We also recommend using rounded ``start_date`` in relation to your
-``schedule_interval``. This means an ``@hourly`` would be at ``00:00``
+
+Previously we also recommended using rounded ``start_date`` in relation to your
+``schedule_interval``. This meant an ``@hourly`` would be at ``00:00``
 minutes:seconds, a ``@daily`` job at midnight, a ``@monthly`` job on the
-first of the month. You can use any sensor or a ``TimeDeltaSensor`` to delay
-the execution of tasks within that period. While ``schedule_interval``
-does allow specifying a ``datetime.timedelta``
+first of the month. This is no longer required. Airflow will not auto align
+the ``start_date`` and the ``schedule_interval``, by using the ``start_date``
+as the moment to start looking.
+
+You can use any sensor or a ``TimeDeltaSensor`` to delay
+the execution of tasks within the schedule interval.
+While ``schedule_interval`` does allow specifying a ``datetime.timedelta``
 object, we recommend using the macros or cron expressions instead, as
 it enforces this idea of rounded schedules.
 
diff --git a/_sources/index.txt b/_sources/index.txt
index c98d619..133d5ab 100644
--- a/_sources/index.txt
+++ b/_sources/index.txt
@@ -7,10 +7,10 @@
 Apache Airflow (incubating) Documentation
 =========================================
 
-.. note ::
+.. important::
 
-    Apache Airflow is an effort undergoing incubation at The Apache
-    Software Foundation (ASF), sponsored by Chris Riccomini.
+    **Disclaimer**: Apache Airflow is an effort undergoing incubation at The
+    Apache Software Foundation (ASF), sponsored by the Apache Incubator.
     Incubation is required of all newly accepted projects until a further
     review indicates that the infrastructure, communications, and
     decision making process have stabilized in a manner consistent with
@@ -19,6 +19,7 @@ Apache Airflow (incubating) Documentation
     the code, it does indicate that the project has yet to be fully
     endorsed by the ASF.
 
+
 Airflow is a platform to programmatically author, schedule and monitor
 workflows.
 
diff --git a/_sources/tutorial.txt b/_sources/tutorial.txt
index e9d382b..a93479c 100644
--- a/_sources/tutorial.txt
+++ b/_sources/tutorial.txt
@@ -18,7 +18,7 @@ complicated, a line by line explanation follows below.
     https://github.com/airbnb/airflow/blob/master/airflow/example_dags/tutorial.py
     """
     from airflow import DAG
-    from airflow.operators import BashOperator
+    from airflow.operators.bash_operator import BashOperator
     from datetime import datetime, timedelta
 
 
@@ -100,7 +100,7 @@ Airflow DAG object. Let's start by importing the libraries we will need.
     from airflow import DAG
 
     # Operators; we need this to operate!
-    from airflow.operators import BashOperator
+    from airflow.operators.bash_operator import BashOperator
 
 Default Arguments
 -----------------
@@ -270,7 +270,7 @@ something like this:
     http://airflow.readthedocs.org/en/latest/tutorial.html
     """
     from airflow import DAG
-    from airflow.operators import BashOperator
+    from airflow.operators.bash_operator import BashOperator
     from datetime import datetime, timedelta
 
 
diff --git a/code.html b/code.html
index 06a0aab..592ec27 100644
--- a/code.html
+++ b/code.html
@@ -322,6 +322,15 @@ between each tries</li>
 </div>
 <div class="section" id="module-airflow.operators">
 <span id="operator-api"></span><h3>Operator API<a class="headerlink" href="#module-airflow.operators" title="Permalink to this headline">¶</a></h3>
+<p>Importer that dynamically loads a class and module from its parent. This
+allows Airflow to support <cite>from airflow.operators.bash_operator import
+BashOperator</cite> even though BashOperator is actually in
+airflow.operators.bash_operator.</p>
+<p>The importer also takes over for the parent_module by wrapping it. This is
+required to support attribute-based usage:</p>
+<blockquote>
+<div>from airflow import operators
+operators.BashOperator(...)</div></blockquote>
 <dl class="class">
 <dt id="airflow.operators.BashOperator">
 <em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">BashOperator</code><span class="sig-paren">(</span><em>bash_command</em>, <em>xcom_push=False</em>, <em>env=None</em>, <em>output_encoding='utf-8'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/bash_operator.html#BashOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.oper [...]
@@ -431,7 +440,7 @@ is allowed</li>
 
 <dl class="class">
 <dt id="airflow.operators.ExternalTaskSensor">
-<em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">ExternalTaskSensor</code><span class="sig-paren">(</span><em>external_dag_id</em>, <em>external_task_id</em>, <em>allowed_states=None</em>, <em>execution_delta=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/sensors.html#ExternalTaskSensor"><span class="viewcode-link">[source]</span></a><a class="headerlink" h [...]
+<em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">ExternalTaskSensor</code><span class="sig-paren">(</span><em>external_dag_id</em>, <em>external_task_id</em>, <em>allowed_states=None</em>, <em>execution_delta=None</em>, <em>execution_date_fn=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/sensors.html#ExternalTaskSensor"><span class="viewcode-link">[source]< [...]
 <dd><p>Bases: <a class="reference internal" href="#airflow.operators.sensors.BaseSensorOperator" title="airflow.operators.sensors.BaseSensorOperator"><code class="xref py py-class docutils literal"><span class="pre">sensors.BaseSensorOperator</span></code></a></p>
 <p>Waits for a task to complete in a different DAG</p>
 <table class="docutils field-list" frame="void" rules="none">
@@ -446,7 +455,12 @@ wait for</li>
 <li><strong>allowed_states</strong> (<em>list</em>) &#8211; list of allowed states, default is <code class="docutils literal"><span class="pre">['success']</span></code></li>
 <li><strong>execution_delta</strong> (<em>datetime.timedelta</em>) &#8211; time difference with the previous execution to
 look at, the default is the same execution_date as the current task.
-For yesterday, use [positive!] datetime.timedelta(days=1)</li>
+For yesterday, use [positive!] datetime.timedelta(days=1). Either
+execution_delta or execution_date_fn can be passed to
+ExternalTaskSensor, but not both.</li>
+<li><strong>execution_date_fn</strong> (<em>callable</em>) &#8211; function that receives the current execution date
+and returns the desired execution date to query. Either execution_delta
+or execution_date_fn can be passed to ExternalTaskSensor, but not both.</li>
 </ul>
 </td>
 </tr>
@@ -539,7 +553,10 @@ part of the script before the first occurrence of <cite>script_begin_tag</cite><
 <dt id="airflow.operators.HivePartitionSensor">
 <em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">HivePartitionSensor</code><span class="sig-paren">(</span><em>table</em>, <em>partition=&quot;ds='{{ ds }}'&quot;</em>, <em>metastore_conn_id='metastore_default'</em>, <em>schema='default'</em>, <em>poke_interval=180</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/sensors.html#HivePartitionSensor"><span class="view [...]
 <dd><p>Bases: <a class="reference internal" href="#airflow.operators.sensors.BaseSensorOperator" title="airflow.operators.sensors.BaseSensorOperator"><code class="xref py py-class docutils literal"><span class="pre">sensors.BaseSensorOperator</span></code></a></p>
-<p>Waits for a partition to show up in Hive</p>
+<p>Waits for a partition to show up in Hive.</p>
+<p>Note: Because &#64;partition supports general logical operators, it
+can be inefficient. Consider using NamedHivePartitionSensor instead if
+you don&#8217;t need the full flexibility of HivePartitionSensor.</p>
 <table class="docutils field-list" frame="void" rules="none">
 <col class="field-name" />
 <col class="field-body" />
@@ -548,7 +565,7 @@ part of the script before the first occurrence of <cite>script_begin_tag</cite><
 <li><strong>table</strong> (<em>string</em>) &#8211; The name of the table to wait for, supports the dot
 notation (my_database.my_table)</li>
 <li><strong>partition</strong> (<em>string</em>) &#8211; The partition clause to wait for. This is passed as
-is to the Metastore Thrift client &#8220;get_partitions_by_filter&#8221; method,
+is to the metastore Thrift client &#8220;get_partitions_by_filter&#8221; method,
 and apparently supports SQL like notation as in <cite>ds=&#8216;2015-01-01&#8217;
 AND type=&#8217;value&#8217;</cite> and &gt; &lt; sings as in &#8220;ds&gt;=2015-01-01&#8221;</li>
 <li><strong>metastore_conn_id</strong> (<em>str</em>) &#8211; reference to the metastore thrift service
@@ -841,7 +858,7 @@ Template reference are recognized by str ending in '.sql'</em>) &#8211; the sql
 <dl class="class">
 <dt id="airflow.operators.PrestoCheckOperator">
 <em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">PrestoCheckOperator</code><span class="sig-paren">(</span><em>sql</em>, <em>presto_conn_id='presto_default'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/presto_check_operator.html#PrestoCheckOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.PrestoCheckOperato [...]
-<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">check_operator.CheckOperator</span></code></p>
+<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.operators.check_operator.CheckOperator</span></code></p>
 <p>Performs checks against Presto. The <code class="docutils literal"><span class="pre">PrestoCheckOperator</span></code> expects
 a sql query that will return a single row. Each value on that
 first row is evaluated using python <code class="docutils literal"><span class="pre">bool</span></code> casting. If any of the
@@ -880,7 +897,7 @@ without stopping the progress of the DAG.</p>
 <dl class="class">
 <dt id="airflow.operators.PrestoIntervalCheckOperator">
 <em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">PrestoIntervalCheckOperator</code><span class="sig-paren">(</span><em>table</em>, <em>metrics_thresholds</em>, <em>date_filter_column='ds'</em>, <em>days_back=-7</em>, <em>presto_conn_id='presto_default'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/presto_check_operator.html#PrestoIntervalCheckOperator"><span cl [...]
-<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">check_operator.IntervalCheckOperator</span></code></p>
+<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.operators.check_operator.IntervalCheckOperator</span></code></p>
 <p>Checks that the values of metrics given as SQL expressions are within
 a certain tolerance of the ones from days_back before.</p>
 <table class="docutils field-list" frame="void" rules="none">
@@ -903,7 +920,7 @@ against. Defaults to 7 days</li>
 <dl class="class">
 <dt id="airflow.operators.PrestoValueCheckOperator">
 <em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">PrestoValueCheckOperator</code><span class="sig-paren">(</span><em>sql</em>, <em>pass_value</em>, <em>tolerance=None</em>, <em>presto_conn_id='presto_default'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/presto_check_operator.html#PrestoValueCheckOperator"><span class="viewcode-link">[source]</span></a><a class= [...]
-<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">check_operator.ValueCheckOperator</span></code></p>
+<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.operators.check_operator.ValueCheckOperator</span></code></p>
 <p>Performs a simple value check using sql code.</p>
 <table class="docutils field-list" frame="void" rules="none">
 <col class="field-name" />
@@ -1188,6 +1205,15 @@ The default is False.</li>
 </div>
 <div class="section" id="module-airflow.contrib.operators">
 <span id="community-contributed-operators"></span><h3>Community-contributed Operators<a class="headerlink" href="#module-airflow.contrib.operators" title="Permalink to this headline">¶</a></h3>
+<p>Importer that dynamically loads a class and module from its parent. This
+allows Airflow to support <cite>from airflow.operators.bash_operator import
+BashOperator</cite> even though BashOperator is actually in
+airflow.operators.bash_operator.</p>
+<p>The importer also takes over for the parent_module by wrapping it. This is
+required to support attribute-based usage:</p>
+<blockquote>
+<div>from airflow import operators
+operators.BashOperator(...)</div></blockquote>
 <dl class="class">
 <dt id="airflow.contrib.operators.SSHExecuteOperator">
 <em class="property">class </em><code class="descclassname">airflow.contrib.operators.</code><code class="descname">SSHExecuteOperator</code><span class="sig-paren">(</span><em>ssh_hook</em>, <em>bash_command</em>, <em>xcom_push=False</em>, <em>env=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/ssh_execute_operator.html#SSHExecuteOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="# [...]
@@ -1286,7 +1312,7 @@ specific database</li>
 
 <dl class="class">
 <dt id="airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator">
-<em class="property">class </em><code class="descclassname">airflow.contrib.operators.gcs_download_operator.</code><code class="descname">GoogleCloudStorageDownloadOperator</code><span class="sig-paren">(</span><em>bucket</em>, <em>object</em>, <em>filename</em>, <em>google_cloud_storage_conn_id='google_cloud_storage_default'</em>, <em>delegate_to=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/opera [...]
+<em class="property">class </em><code class="descclassname">airflow.contrib.operators.gcs_download_operator.</code><code class="descname">GoogleCloudStorageDownloadOperator</code><span class="sig-paren">(</span><em>bucket</em>, <em>object</em>, <em>filename=False</em>, <em>store_to_xcom_key=False</em>, <em>google_cloud_storage_conn_id='google_cloud_storage_default'</em>, <em>delegate_to=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference intern [...]
 <dd><p>Downloads a file from Google Cloud Storage.</p>
 </dd></dl>
 
@@ -1560,8 +1586,8 @@ More info: <a class="reference external" href="https://www.hipchat.com/docs/apiv
 in all templates</p>
 <table border="1" class="docutils">
 <colgroup>
-<col width="40%" />
-<col width="60%" />
+<col width="38%" />
+<col width="62%" />
 </colgroup>
 <thead valign="bottom">
 <tr class="row-odd"><th class="head">Variable</th>
@@ -1620,6 +1646,14 @@ in all templates</p>
 <tr class="row-even"><td><code class="docutils literal"><span class="pre">{{</span> <span class="pre">params</span> <span class="pre">}}</span></code></td>
 <td>a reference to the user-defined params dictionary</td>
 </tr>
+<tr class="row-odd"><td><code class="docutils literal"><span class="pre">{{</span> <span class="pre">var.value.my_var</span> <span class="pre">}}</span></code></td>
+<td>global defined variables represented as a dictionary</td>
+</tr>
+<tr class="row-even"><td><code class="docutils literal"><span class="pre">{{</span> <span class="pre">var.json.my_var.path</span> <span class="pre">}}</span></code></td>
+<td>global defined variables represented as a dictionary
+with deserialized JSON object, append the path to the
+key within the JSON object</td>
+</tr>
 <tr class="row-odd"><td><code class="docutils literal"><span class="pre">{{</span> <span class="pre">task_instance_key_str</span> <span class="pre">}}</span></code></td>
 <td>a unique, human-readable key to the task instance
 formatted <code class="docutils literal"><span class="pre">{dag_id}_{task_id}_{ds}</span></code></td>
@@ -1647,6 +1681,10 @@ dot notation. Here are some examples of what is possible:
 <code class="docutils literal"><span class="pre">{{</span> <span class="pre">task.owner</span> <span class="pre">}}</span></code>, <code class="docutils literal"><span class="pre">{{</span> <span class="pre">task.task_id</span> <span class="pre">}}</span></code>, <code class="docutils literal"><span class="pre">{{</span> <span class="pre">ti.hostname</span> <span class="pre">}}</span></code>, ...
 Refer to the models documentation for more information on the objects&#8217;
 attributes and methods.</p>
+<p>The <code class="docutils literal"><span class="pre">var</span></code> template variable allows you to access variables defined in Airflow&#8217;s
+UI. You can access them as either plain-text or JSON. If you use JSON, you are
+also able to walk nested structures, such as dictionaries like:
+<code class="docutils literal"><span class="pre">{{</span> <span class="pre">var.json.my_dict_var.key1</span> <span class="pre">}}</span></code></p>
 </div>
 <div class="section" id="id2">
 <h3>Macros<a class="headerlink" href="#id2" title="Permalink to this headline">¶</a></h3>
@@ -1736,12 +1774,6 @@ as specified in the output format</p>
 </dd></dl>
 
 <dl class="function">
-<dt id="airflow.macros.integrate_plugins">
-<code class="descclassname">airflow.macros.</code><code class="descname">integrate_plugins</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/macros.html#integrate_plugins"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.macros.integrate_plugins" title="Permalink to this definition">¶</a></dt>
-<dd><p>Integrate plugins to the context</p>
-</dd></dl>
-
-<dl class="function">
 <dt id="airflow.macros.random">
 <code class="descclassname">airflow.macros.</code><code class="descname">random</code><span class="sig-paren">(</span><span class="sig-paren">)</span> &rarr; x in the interval [0, 1).<a class="headerlink" href="#airflow.macros.random" title="Permalink to this definition">¶</a></dt>
 <dd></dd></dl>
@@ -1972,6 +2004,12 @@ template_searchpath and user_defined_macros</p>
 </dd></dl>
 
 <dl class="method">
+<dt id="airflow.models.DAG.normalize_schedule">
+<code class="descname">normalize_schedule</code><span class="sig-paren">(</span><em>dttm</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#DAG.normalize_schedule"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.DAG.normalize_schedule" title="Permalink to this definition">¶</a></dt>
+<dd><p>Returns dttm + interval unless dttm is first interval then it returns dttm</p>
+</dd></dl>
+
+<dl class="method">
 <dt id="airflow.models.DAG.run">
 <code class="descname">run</code><span class="sig-paren">(</span><em>start_date=None</em>, <em>end_date=None</em>, <em>mark_success=False</em>, <em>include_adhoc=False</em>, <em>local=False</em>, <em>executor=None</em>, <em>donot_pickle=False</em>, <em>ignore_dependencies=False</em>, <em>ignore_first_depends_on_past=False</em>, <em>pool=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#DAG.run"><span class="viewcode-link">[source]</ [...]
 <dd><p>Runs the DAG.</p>
@@ -2465,6 +2503,24 @@ session is committed.</td>
 <dt id="airflow.models.TaskInstance.run">
 <code class="descname">run</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#TaskInstance.run"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.TaskInstance.run" title="Permalink to this definition">¶</a></dt>
 <dd><p>Runs the task instance.</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
+<li><strong>verbose</strong> (<em>boolean</em>) &#8211; whether to turn on more verbose loggin</li>
+<li><strong>ignore_dependencies</strong> (<em>boolean</em>) &#8211; Doesn&#8217;t check for deps, just runs</li>
+<li><strong>ignore_depends_on_past</strong> (<em>boolean</em>) &#8211; Ignore depends_on_past but respect
+other dependencies</li>
+<li><strong>force</strong> (<em>boolean</em>) &#8211; Forces a run regarless of previous success</li>
+<li><strong>mark_success</strong> (<em>boolean</em>) &#8211; Don&#8217;t run the task, mark its state as success</li>
+<li><strong>test_mode</strong> (<em>boolean</em>) &#8211; Doesn&#8217;t record success or failure in the DB</li>
+<li><strong>pool</strong> (<em>str</em>) &#8211; specifies the pool to use to run the task instance</li>
+</ul>
+</td>
+</tr>
+</tbody>
+</table>
 </dd></dl>
 
 <dl class="method">
@@ -2631,6 +2687,15 @@ passwords when using operators or hooks.</p>
 </div>
 <div class="section" id="module-airflow.hooks">
 <span id="hooks"></span><h2>Hooks<a class="headerlink" href="#module-airflow.hooks" title="Permalink to this headline">¶</a></h2>
+<p>Importer that dynamically loads a class and module from its parent. This
+allows Airflow to support <cite>from airflow.operators.bash_operator import
+BashOperator</cite> even though BashOperator is actually in
+airflow.operators.bash_operator.</p>
+<p>The importer also takes over for the parent_module by wrapping it. This is
+required to support attribute-based usage:</p>
+<blockquote>
+<div>from airflow import operators
+operators.BashOperator(...)</div></blockquote>
 <dl class="class">
 <dt id="airflow.hooks.DbApiHook">
 <em class="property">class </em><code class="descclassname">airflow.hooks.</code><code class="descname">DbApiHook</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/dbapi_hook.html#DbApiHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.DbApiHook" title="Permalink to this definition">¶</a></dt>
@@ -2799,7 +2864,8 @@ traditional CLI. To enable <code class="docutils literal"><span class="pre">beel
 extra field of your connection as in <code class="docutils literal"><span class="pre">{</span> <span class="pre">&quot;use_beeline&quot;:</span> <span class="pre">true</span> <span class="pre">}</span></code></p>
 <p>Note that you can also set default hive CLI parameters using the
 <code class="docutils literal"><span class="pre">hive_cli_params</span></code> to be used in your connection as in
-<code class="docutils literal"><span class="pre">{&quot;hive_cli_params&quot;:</span> <span class="pre">&quot;-hiveconf</span> <span class="pre">mapred.job.tracker=some.jobtracker:444&quot;}</span></code></p>
+<code class="docutils literal"><span class="pre">{&quot;hive_cli_params&quot;:</span> <span class="pre">&quot;-hiveconf</span> <span class="pre">mapred.job.tracker=some.jobtracker:444&quot;}</span></code>
+Parameters passed here can be overridden by run_cli&#8217;s hive_conf param</p>
 <p>The extra connection parameter <code class="docutils literal"><span class="pre">auth</span></code> gets passed as in the <code class="docutils literal"><span class="pre">jdbc</span></code>
 connection string as is.</p>
 <dl class="method">
@@ -2834,8 +2900,20 @@ and values</li>
 
 <dl class="method">
 <dt id="airflow.hooks.HiveCliHook.run_cli">
-<code class="descname">run_cli</code><span class="sig-paren">(</span><em>hql</em>, <em>schema=None</em>, <em>verbose=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/hive_hooks.html#HiveCliHook.run_cli"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.HiveCliHook.run_cli" title="Permalink to this definition">¶</a></dt>
-<dd><p>Run an hql statement using the hive cli</p>
+<code class="descname">run_cli</code><span class="sig-paren">(</span><em>hql</em>, <em>schema=None</em>, <em>verbose=True</em>, <em>hive_conf=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/hive_hooks.html#HiveCliHook.run_cli"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.HiveCliHook.run_cli" title="Permalink to this definition">¶</a></dt>
+<dd><p>Run an hql statement using the hive cli. If hive_conf is specified it should be a
+dict and the entries will be set as key/value pairs in HiveConf</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>hive_conf</strong> (<em>dict</em>) &#8211; if specified these key value pairs will be passed to hive as
+<code class="docutils literal"><span class="pre">-hiveconf</span> <span class="pre">&quot;key&quot;=&quot;value&quot;</span></code>. Note that they will be passed after the
+<code class="docutils literal"><span class="pre">hive_cli_params</span></code> and thus will override whatever values are specified in
+the database.</td>
+</tr>
+</tbody>
+</table>
 <div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">hh</span> <span class="o">=</span> <span class="n">HiveCliHook</span><span class="p">()</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">result</span> <span class="o">=</span> <span class="n">hh</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="s2">&quot;USE airflow;&quot;</span><span class="p">)</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="p">(</span><span class="s2">&quot;OK&quot;</span> <span class="ow">in</span> <span class="n">result</span><span class="p">)</span>
@@ -2858,9 +2936,60 @@ and values</li>
 <dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.hooks.base_hook.BaseHook</span></code></p>
 <p>Wrapper to interact with the Hive Metastore</p>
 <dl class="method">
+<dt id="airflow.hooks.HiveMetastoreHook.check_for_named_partition">
+<code class="descname">check_for_named_partition</code><span class="sig-paren">(</span><em>schema</em>, <em>table</em>, <em>partition_name</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/hive_hooks.html#HiveMetastoreHook.check_for_named_partition"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.HiveMetastoreHook.check_for_named_partition" title="Permalink to this definition">¶</a></dt>
+<dd><p>Checks whether a partition with a given name exists</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
+<li><strong>schema</strong> (<em>string</em>) &#8211; Name of hive schema (database) &#64;table belongs to</li>
+<li><strong>table</strong> &#8211; Name of hive table &#64;partition belongs to</li>
+</ul>
+</td>
+</tr>
+<tr class="field-even field"><th class="field-name">Partition:</th><td class="field-body"><p class="first">Name of the partitions to check for (eg <cite>a=b/c=d</cite>)</p>
+</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">boolean</p>
+</td>
+</tr>
+</tbody>
+</table>
+<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">()</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">t</span> <span class="o">=</span> <span class="s1">&#39;static_babynames_partitioned&#39;</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">hh</span><span class="o">.</span><span class="n">check_for_named_partition</span><span class="p">(</span><span class="s1">&#39;airflow&#39;</span><span class="p">,</span> <span class="n">t</span><span class="p">,</span> <span class="s2">&quot;ds=2015-01-01&quot;</span><span class="p">)</span>
+<span class="go">True</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">hh</span><span class="o">.</span><span class="n">check_for_named_partition</span><span class="p">(</span><span class="s1">&#39;airflow&#39;</span><span class="p">,</span> <span class="n">t</span><span class="p">,</span> <span class="s2">&quot;ds=xxx&quot;</span><span class="p">)</span>
+<span class="go">False</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="method">
 <dt id="airflow.hooks.HiveMetastoreHook.check_for_partition">
 <code class="descname">check_for_partition</code><span class="sig-paren">(</span><em>schema</em>, <em>table</em>, <em>partition</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/hive_hooks.html#HiveMetastoreHook.check_for_partition"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.HiveMetastoreHook.check_for_partition" title="Permalink to this definition">¶</a></dt>
 <dd><p>Checks whether a partition exists</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
+<li><strong>schema</strong> (<em>string</em>) &#8211; Name of hive schema (database) &#64;table belongs to</li>
+<li><strong>table</strong> &#8211; Name of hive table &#64;partition belongs to</li>
+</ul>
+</td>
+</tr>
+<tr class="field-even field"><th class="field-name">Partition:</th><td class="field-body"><p class="first">Expression that matches the partitions to check for
+(eg <cite>a = &#8216;b&#8217; AND c = &#8216;d&#8217;</cite>)</p>
+</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">boolean</p>
+</td>
+</tr>
+</tbody>
+</table>
 <div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">()</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">t</span> <span class="o">=</span> <span class="s1">&#39;static_babynames_partitioned&#39;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">hh</span><span class="o">.</span><span class="n">check_for_partition</span><span class="p">(</span><span class="s1">&#39;airflow&#39;</span><span class="p">,</span> <span class="n">t</span><span class="p">,</span> <span class="s2">&quot;ds=&#39;2015-01-01&#39;&quot;</span><span class="p">)</span>
@@ -3377,6 +3506,15 @@ directory, files will be uploaded inside.</li>
 
 <div class="section" id="module-airflow.contrib.hooks">
 <span id="community-contributed-hooks"></span><h3>Community contributed hooks<a class="headerlink" href="#module-airflow.contrib.hooks" title="Permalink to this headline">¶</a></h3>
+<p>Importer that dynamically loads a class and module from its parent. This
+allows Airflow to support <cite>from airflow.operators.bash_operator import
+BashOperator</cite> even though BashOperator is actually in
+airflow.operators.bash_operator.</p>
+<p>The importer also takes over for the parent_module by wrapping it. This is
+required to support attribute-based usage:</p>
+<blockquote>
+<div>from airflow import operators
+operators.BashOperator(...)</div></blockquote>
 <dl class="class">
 <dt id="airflow.contrib.hooks.BigQueryHook">
 <em class="property">class </em><code class="descclassname">airflow.contrib.hooks.</code><code class="descname">BigQueryHook</code><span class="sig-paren">(</span><em>bigquery_conn_id='bigquery_default'</em>, <em>delegate_to=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/bigquery_hook.html#BigQueryHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.BigQueryHook" title="Permalink to this definition" [...]
@@ -3500,7 +3638,7 @@ downstream.</p>
 <dt id="airflow.contrib.hooks.FTPHook.close_conn">
 <code class="descname">close_conn</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/ftp_hook.html#FTPHook.close_conn"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.FTPHook.close_conn" title="Permalink to this definition">¶</a></dt>
 <dd><p>Closes the connection. An error will occur if the
-connection wasnt ever opened.</p>
+connection wasn&#8217;t ever opened.</p>
 </dd></dl>
 
 <dl class="method">
@@ -3534,7 +3672,7 @@ connection wasnt ever opened.</p>
 <dl class="method">
 <dt id="airflow.contrib.hooks.FTPHook.delete_file">
 <code class="descname">delete_file</code><span class="sig-paren">(</span><em>path</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/ftp_hook.html#FTPHook.delete_file"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.FTPHook.delete_file" title="Permalink to this definition">¶</a></dt>
-<dd><p>Removes a file on the FTP Server</p>
+<dd><p>Removes a file on the FTP Server.</p>
 <table class="docutils field-list" frame="void" rules="none">
 <col class="field-name" />
 <col class="field-body" />
diff --git a/concepts.html b/concepts.html
index b41766f..15b2b4d 100644
--- a/concepts.html
+++ b/concepts.html
@@ -573,7 +573,7 @@ object. This will prevent the SubDAG from being treated like a separate DAG in
 the main UI. For example:</p>
 <div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="c1">#dags/subdag.py</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">DAG</span>
-<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="k">import</span> <span class="n">DummyOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.dummy_operator</span> <span class="k">import</span> <span class="n">DummyOperator</span>
 
 
 <span class="c1"># Dag is returned by a factory method</span>
@@ -596,7 +596,7 @@ the main UI. For example:</p>
 <div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="c1"># main_dag.py</span>
 <span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">DAG</span>
-<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="k">import</span> <span class="n">SubDagOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.subdag_operator</span> <span class="k">import</span> <span class="n">SubDagOperator</span>
 <span class="kn">from</span> <span class="nn">dags.subdag</span> <span class="k">import</span> <span class="n">sub_dag</span>
 
 
diff --git a/faq.html b/faq.html
index 050da7b..3d8cb5d 100644
--- a/faq.html
+++ b/faq.html
@@ -202,12 +202,15 @@ inactive DagRuns to get the new task to get onboarded properly.</p>
 <code class="docutils literal"><span class="pre">datetime.now()</span></code> as it can be quite confusing. The task is triggered
 once the period closes, and in theory an <code class="docutils literal"><span class="pre">&#64;hourly</span></code> DAG would never get to
 an hour after now as <code class="docutils literal"><span class="pre">now()</span></code> moves along.</p>
-<p>We also recommend using rounded <code class="docutils literal"><span class="pre">start_date</span></code> in relation to your
-<code class="docutils literal"><span class="pre">schedule_interval</span></code>. This means an <code class="docutils literal"><span class="pre">&#64;hourly</span></code> would be at <code class="docutils literal"><span class="pre">00:00</span></code>
+<p>Previously we also recommended using rounded <code class="docutils literal"><span class="pre">start_date</span></code> in relation to your
+<code class="docutils literal"><span class="pre">schedule_interval</span></code>. This meant an <code class="docutils literal"><span class="pre">&#64;hourly</span></code> would be at <code class="docutils literal"><span class="pre">00:00</span></code>
 minutes:seconds, a <code class="docutils literal"><span class="pre">&#64;daily</span></code> job at midnight, a <code class="docutils literal"><span class="pre">&#64;monthly</span></code> job on the
-first of the month. You can use any sensor or a <code class="docutils literal"><span class="pre">TimeDeltaSensor</span></code> to delay
-the execution of tasks within that period. While <code class="docutils literal"><span class="pre">schedule_interval</span></code>
-does allow specifying a <code class="docutils literal"><span class="pre">datetime.timedelta</span></code>
+first of the month. This is no longer required. Airflow will not auto align
+the <code class="docutils literal"><span class="pre">start_date</span></code> and the <code class="docutils literal"><span class="pre">schedule_interval</span></code>, by using the <code class="docutils literal"><span class="pre">start_date</span></code>
+as the moment to start looking.</p>
+<p>You can use any sensor or a <code class="docutils literal"><span class="pre">TimeDeltaSensor</span></code> to delay
+the execution of tasks within the schedule interval.
+While <code class="docutils literal"><span class="pre">schedule_interval</span></code> does allow specifying a <code class="docutils literal"><span class="pre">datetime.timedelta</span></code>
 object, we recommend using the macros or cron expressions instead, as
 it enforces this idea of rounded schedules.</p>
 <p>When using <code class="docutils literal"><span class="pre">depends_on_past=True</span></code> it&#8217;s important to pay special attention
diff --git a/genindex.html b/genindex.html
index ecba6b9..20b510f 100644
--- a/genindex.html
+++ b/genindex.html
@@ -150,6 +150,7 @@
  | <a href="#K"><strong>K</strong></a>
  | <a href="#L"><strong>L</strong></a>
  | <a href="#M"><strong>M</strong></a>
+ | <a href="#N"><strong>N</strong></a>
  | <a href="#O"><strong>O</strong></a>
  | <a href="#P"><strong>P</strong></a>
  | <a href="#Q"><strong>Q</strong></a>
@@ -288,6 +289,10 @@
   </dt>
 
       
+  <dt><a href="code.html#airflow.hooks.HiveMetastoreHook.check_for_named_partition">check_for_named_partition() (airflow.hooks.HiveMetastoreHook method)</a>
+  </dt>
+
+      
   <dt><a href="code.html#airflow.hooks.HiveMetastoreHook.check_for_partition">check_for_partition() (airflow.hooks.HiveMetastoreHook method)</a>
   </dt>
 
@@ -774,19 +779,15 @@
 
       </dl></dd>
       
-  <dt><a href="code.html#airflow.macros.integrate_plugins">integrate_plugins() (in module airflow.macros)</a>
-  </dt>
-
-      
   <dt><a href="code.html#airflow.models.DAG.is_paused">is_paused (airflow.models.DAG attribute)</a>
   </dt>
 
-  </dl></td>
-  <td style="width: 33%" valign="top"><dl>
       
   <dt><a href="code.html#airflow.models.TaskInstance.is_premature">is_premature() (airflow.models.TaskInstance method)</a>
   </dt>
 
+  </dl></td>
+  <td style="width: 33%" valign="top"><dl>
       
   <dt><a href="code.html#airflow.models.TaskInstance.is_queueable">is_queueable() (airflow.models.TaskInstance method)</a>
   </dt>
@@ -914,6 +915,16 @@
   </dl></td>
 </tr></table>
 
+<h2 id="N">N</h2>
+<table style="width: 100%" class="indextable genindextable"><tr>
+  <td style="width: 33%" valign="top"><dl>
+      
+  <dt><a href="code.html#airflow.models.DAG.normalize_schedule">normalize_schedule() (airflow.models.DAG method)</a>
+  </dt>
+
+  </dl></td>
+</tr></table>
+
 <h2 id="O">O</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%" valign="top"><dl>
diff --git a/index.html b/index.html
index 383bab0..5ac4744 100644
--- a/index.html
+++ b/index.html
@@ -140,10 +140,10 @@
 <a class="reference internal image-reference" href="_images/incubator.jpg"><img alt="_images/incubator.jpg" src="_images/incubator.jpg" style="width: 150px;" /></a>
 <div class="section" id="apache-airflow-incubating-documentation">
 <h1>Apache Airflow (incubating) Documentation<a class="headerlink" href="#apache-airflow-incubating-documentation" title="Permalink to this headline">¶</a></h1>
-<div class="admonition note">
-<p class="first admonition-title">Note</p>
-<p class="last">Apache Airflow is an effort undergoing incubation at The Apache
-Software Foundation (ASF), sponsored by Chris Riccomini.
+<div class="admonition important">
+<p class="first admonition-title">Important</p>
+<p class="last"><strong>Disclaimer</strong>: Apache Airflow is an effort undergoing incubation at The
+Apache Software Foundation (ASF), sponsored by the Apache Incubator.
 Incubation is required of all newly accepted projects until a further
 review indicates that the infrastructure, communications, and
 decision making process have stabilized in a manner consistent with
diff --git a/objects.inv b/objects.inv
index 36f1543..56e566e 100644
Binary files a/objects.inv and b/objects.inv differ
diff --git a/searchindex.js b/searchindex.js
index 901905a..97adc3c 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({envversion:47,filenames:["cli","code","concepts","configuration","faq","index","installation","license","plugins","profiling","project","scheduler","security","start","tutorial","ui"],objects:{"airflow.contrib":{hooks:[1,1,0,"-"],operators:[1,1,0,"-"]},"airflow.contrib.executors.mesos_executor":{MesosExecutor:[1,0,1,""]},"airflow.contrib.hooks":{BigQueryHook:[1,0,1,""],CloudantHook:[1,0,1,""],FTPHook:[1,0,1,""],GoogleCloudStorageHook:[1,0,1,""],SSHHook:[1,0,1,""],Vertica [...]
\ No newline at end of file
+Search.setIndex({envversion:47,filenames:["cli","code","concepts","configuration","faq","index","installation","license","plugins","profiling","project","scheduler","security","start","tutorial","ui"],objects:{"airflow.contrib":{hooks:[1,1,0,"-"],operators:[1,1,0,"-"]},"airflow.contrib.executors.mesos_executor":{MesosExecutor:[1,0,1,""]},"airflow.contrib.hooks":{BigQueryHook:[1,0,1,""],CloudantHook:[1,0,1,""],FTPHook:[1,0,1,""],GoogleCloudStorageHook:[1,0,1,""],SSHHook:[1,0,1,""],Vertica [...]
\ No newline at end of file
diff --git a/tutorial.html b/tutorial.html
index 2b379c9..abcbbc0 100644
--- a/tutorial.html
+++ b/tutorial.html
@@ -169,7 +169,7 @@ complicated, a line by line explanation follows below.</p>
 <span class="sd">https://github.com/airbnb/airflow/blob/master/airflow/example_dags/tutorial.py</span>
 <span class="sd">&quot;&quot;&quot;</span>
 <span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">DAG</span>
-<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="k">import</span> <span class="n">BashOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.bash_operator</span> <span class="k">import</span> <span class="n">BashOperator</span>
 <span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
 
 
@@ -245,7 +245,7 @@ Airflow DAG object. Let&#8217;s start by importing the libraries we will need.</
 <span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">DAG</span>
 
 <span class="c1"># Operators; we need this to operate!</span>
-<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="k">import</span> <span class="n">BashOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.bash_operator</span> <span class="k">import</span> <span class="n">BashOperator</span>
 </pre></div>
 </div>
 </div>
@@ -397,7 +397,7 @@ something like this:</p>
 <span class="sd">http://airflow.readthedocs.org/en/latest/tutorial.html</span>
 <span class="sd">&quot;&quot;&quot;</span>
 <span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">DAG</span>
-<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="k">import</span> <span class="n">BashOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.bash_operator</span> <span class="k">import</span> <span class="n">BashOperator</span>
 <span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
 
 


Mime
View raw message