airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maximebeauche...@apache.org
Subject [5/5] incubator-airflow-site git commit: Updating the disclaimer
Date Fri, 01 Jul 2016 21:54:31 GMT
Updating the disclaimer


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/commit/457b698d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/tree/457b698d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/diff/457b698d

Branch: refs/heads/asf-site
Commit: 457b698d9f92af370061d322267d85886e6cbfe4
Parents: 502934c
Author: Maxime Beauchemin <maxime.beauchemin@apache.org>
Authored: Fri Jul 1 14:54:14 2016 -0700
Committer: Maxime Beauchemin <maxime.beauchemin@apache.org>
Committed: Fri Jul 1 14:54:14 2016 -0700

----------------------------------------------------------------------
 .../contrib/executors/mesos_executor.html       |  16 +-
 .../contrib/operators/bigquery_operator.html    |  16 +-
 .../contrib/operators/bigquery_to_gcs.html      |  16 +-
 .../operators/gcs_download_operator.html        |  37 +++-
 _modules/airflow/executors/celery_executor.html |  16 +-
 _modules/airflow/executors/local_executor.html  |  16 +-
 .../airflow/executors/sequential_executor.html  |  16 +-
 _modules/airflow/macros.html                    |  40 ++++-
 _modules/airflow/macros/hive.html               |  20 ++-
 _modules/airflow/models.html                    | 111 +++++++++---
 _modules/airflow/operators/docker_operator.html |  16 +-
 _modules/airflow/operators/sensors.html         | 137 ++++++++++++--
 _modules/bash_operator.html                     |  16 +-
 _modules/dagrun_operator.html                   |  16 +-
 _modules/dbapi_hook.html                        |  24 ++-
 _modules/druid_hook.html                        |  16 +-
 _modules/dummy_operator.html                    |  16 +-
 _modules/email_operator.html                    |  16 +-
 _modules/ftp_hook.html                          |  22 ++-
 _modules/generic_transfer.html                  |  16 +-
 _modules/hive_hooks.html                        |  82 +++++++--
 _modules/hive_operator.html                     |  22 ++-
 _modules/hive_to_druid.html                     |  48 +++--
 _modules/hive_to_mysql.html                     |  21 ++-
 _modules/hive_to_samba_operator.html            |  19 +-
 _modules/http_hook.html                         |   2 +-
 _modules/http_operator.html                     |  18 +-
 _modules/mssql_hook.html                        |  25 ++-
 _modules/mssql_operator.html                    |  20 ++-
 _modules/mssql_to_hive.html                     |  19 +-
 _modules/mysql_hook.html                        |  16 +-
 _modules/mysql_operator.html                    |  20 ++-
 _modules/mysql_to_hive.html                     |  19 +-
 _modules/postgres_hook.html                     |  16 +-
 _modules/postgres_operator.html                 |  20 ++-
 _modules/presto_check_operator.html             |  18 +-
 _modules/presto_hook.html                       |  25 ++-
 _modules/python_operator.html                   |  16 +-
 _modules/qubole_operator.html                   |  20 ++-
 _modules/s3_to_hive_operator.html               |  19 +-
 _modules/sensors.html                           | 137 ++++++++++++--
 _modules/slack_operator.html                    |  16 +-
 _modules/sqlite_hook.html                       |  16 +-
 _modules/ssh_execute_operator.html              |  16 +-
 _modules/vertica_operator.html                  |  20 ++-
 _modules/vertica_to_hive.html                   |  20 ++-
 _modules/webhdfs_hook.html                      |  16 +-
 _sources/code.txt                               |   9 +
 _sources/concepts.txt                           |   4 +-
 _sources/faq.txt                                |  15 +-
 _sources/index.txt                              |   7 +-
 _sources/tutorial.txt                           |   6 +-
 code.html                                       | 180 ++++++++++++++++---
 concepts.html                                   |   4 +-
 faq.html                                        |  13 +-
 genindex.html                                   |  23 ++-
 index.html                                      |   8 +-
 objects.inv                                     | Bin 2236 -> 2246 bytes
 searchindex.js                                  |   2 +-
 tutorial.html                                   |   6 +-
 60 files changed, 1363 insertions(+), 219 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/contrib/executors/mesos_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/executors/mesos_executor.html b/_modules/airflow/contrib/executors/mesos_executor.html
index 65557d9..e2d8f61 100644
--- a/_modules/airflow/contrib/executors/mesos_executor.html
+++ b/_modules/airflow/contrib/executors/mesos_executor.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.contrib.executors.mesos_executor</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
 <span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
 <span class="kn">import</span> <span class="nn">logging</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/contrib/operators/bigquery_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_operator.html b/_modules/airflow/contrib/operators/bigquery_operator.html
index 830fd57..6038f2c 100644
--- a/_modules/airflow/contrib/operators/bigquery_operator.html
+++ b/_modules/airflow/contrib/operators/bigquery_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.contrib.operators.bigquery_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
 
 <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/contrib/operators/bigquery_to_gcs.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_to_gcs.html b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
index 465128c..a3b8d18 100644
--- a/_modules/airflow/contrib/operators/bigquery_to_gcs.html
+++ b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.contrib.operators.bigquery_to_gcs</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
 
 <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/contrib/operators/gcs_download_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/gcs_download_operator.html b/_modules/airflow/contrib/operators/gcs_download_operator.html
index 4af6020..dbf56fb 100644
--- a/_modules/airflow/contrib/operators/gcs_download_operator.html
+++ b/_modules/airflow/contrib/operators/gcs_download_operator.html
@@ -137,7 +137,22 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.contrib.operators.gcs_download_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+<span class="kn">import</span> <span class="nn">sys</span>
 
 <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="kn">import</span> <span class="n">GoogleCloudStorageHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
@@ -147,7 +162,7 @@
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Downloads a file from Google Cloud Storage.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;bucket&#39;</span><span class="p">,</span><span class="s1">&#39;object&#39;</span><span class="p">,</span><span class="s1">&#39;filename&#39;</span><span class="p">,)</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;bucket&#39;</span><span class="p">,</span><span class="s1">&#39;object&#39;</span><span class="p">,</span><span class="s1">&#39;filename&#39;</span><span class="p">,</span><span class="s1">&#39;store_to_xcom_key&#39;</span><span class="p">,)</span>
     <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.sql&#39;</span><span class="p">,)</span>
     <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#f0eee4&#39;</span>
 
@@ -156,7 +171,8 @@
         <span class="bp">self</span><span class="p">,</span>
         <span class="n">bucket</span><span class="p">,</span>
         <span class="nb">object</span><span class="p">,</span>
-        <span class="n">filename</span><span class="p">,</span>
+        <span class="n">filename</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+        <span class="n">store_to_xcom_key</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
         <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_storage_default&#39;</span><span class="p">,</span>
         <span class="n">delegate_to</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
         <span class="o">*</span><span class="n">args</span><span class="p">,</span>
@@ -171,7 +187,13 @@
 <span class="sd">        :type object: string</span>
 <span class="sd">        :param filename: The file path on the local file system (where the</span>
 <span class="sd">            operator is being executed) that the file should be downloaded to.</span>
+<span class="sd">            If false, the downloaded data will not be stored on the local file</span>
+<span class="sd">            system.</span>
 <span class="sd">        :type filename: string</span>
+<span class="sd">        :param store_to_xcom_key: If this param is set, the operator will push</span>
+<span class="sd">            the contents of the downloaded file to XCom with the key set in this</span>
+<span class="sd">            parameter. If false, the downloaded data will not be pushed to XCom.</span>
+<span class="sd">        :type store_to_xcom_key: string</span>
 <span class="sd">        :param google_cloud_storage_conn_id: The connection ID to use when</span>
 <span class="sd">            connecting to Google cloud storage.</span>
 <span class="sd">        :type google_cloud_storage_conn_id: string</span>
@@ -183,6 +205,7 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">object</span> <span class="o">=</span> <span class="nb">object</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">filename</span> <span class="o">=</span> <span class="n">filename</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">store_to_xcom_key</span> <span class="o">=</span> <span class="n">store_to_xcom_key</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
 
@@ -190,7 +213,13 @@
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Executing download: </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
         <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span><span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
                                       <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
-        <span class="k">print</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">))</span></div>
+        <span class="n">file_bytes</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">store_to_xcom_key</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">getsizeof</span><span class="p">(</span><span class="n">file_bytes</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">48000</span><span class="p">:</span>
+                <span class="n">context</span><span class="p">[</span><span class="s1">&#39;ti&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">xcom_push</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">store_to_xcom_key</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">file_bytes</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s1">&#39;The size of the downloaded file is too large to push to XCom!&#39;</span><span class="p">)</span>
+        <span class="k">print</span><span class="p">(</span><span class="n">file_bytes</span><span class="p">)</span></div>
 </pre></div>
 
            </div>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/executors/celery_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/executors/celery_executor.html b/_modules/airflow/executors/celery_executor.html
index a231e28..9601c12 100644
--- a/_modules/airflow/executors/celery_executor.html
+++ b/_modules/airflow/executors/celery_executor.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.executors.celery_executor</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">object</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">object</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">subprocess</span>
 <span class="kn">import</span> <span class="nn">time</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/executors/local_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/executors/local_executor.html b/_modules/airflow/executors/local_executor.html
index 79c456d..1841cb4 100644
--- a/_modules/airflow/executors/local_executor.html
+++ b/_modules/airflow/executors/local_executor.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.executors.local_executor</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">multiprocessing</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">multiprocessing</span>
 <span class="kn">import</span> <span class="nn">subprocess</span>
 <span class="kn">import</span> <span class="nn">time</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/executors/sequential_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/executors/sequential_executor.html b/_modules/airflow/executors/sequential_executor.html
index a393b0f..ce84843 100644
--- a/_modules/airflow/executors/sequential_executor.html
+++ b/_modules/airflow/executors/sequential_executor.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.executors.sequential_executor</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
 <span class="kn">import</span> <span class="nn">subprocess</span>
 
 <span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="kn">import</span> <span class="n">BaseExecutor</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/macros.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/macros.html b/_modules/airflow/macros.html
index 6b4e17f..78efd22 100644
--- a/_modules/airflow/macros.html
+++ b/_modules/airflow/macros.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.macros</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
 <span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">random</span>
 <span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
 <span class="kn">import</span> <span class="nn">dateutil</span>
@@ -187,11 +201,29 @@
     <span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">ds</span><span class="p">,</span> <span class="n">input_format</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="n">output_format</span><span class="p">)</span></div>
 
 
-<div class="viewcode-block" id="integrate_plugins"><a class="viewcode-back" href="../../code.html#airflow.macros.integrate_plugins">[docs]</a><span class="k">def</span> <span class="nf">integrate_plugins</span><span class="p">():</span>
+<span class="k">def</span> <span class="nf">_integrate_plugins</span><span class="p">():</span>
     <span class="sd">&quot;&quot;&quot;Integrate plugins to the context&quot;&quot;&quot;</span>
+    <span class="kn">import</span> <span class="nn">sys</span>
     <span class="kn">from</span> <span class="nn">airflow.plugins_manager</span> <span class="kn">import</span> <span class="n">macros</span> <span class="k">as</span> <span class="n">_macros</span>
-    <span class="k">for</span> <span class="n">_macro</span> <span class="ow">in</span> <span class="n">_macros</span><span class="p">:</span>
-        <span class="nb">globals</span><span class="p">()[</span><span class="n">_macro</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro</span></div>
+    <span class="k">for</span> <span class="n">_macro_module</span> <span class="ow">in</span> <span class="n">_macros</span><span class="p">:</span>
+        <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="n">_macro_module</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro_module</span>
+        <span class="nb">globals</span><span class="p">()[</span><span class="n">_macro_module</span><span class="o">.</span><span class="n">_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro_module</span>
+
+
+        <span class="c1">##########################################################</span>
+        <span class="c1"># TODO FIXME Remove in Airflow 2.0</span>
+
+        <span class="kn">import</span> <span class="nn">os</span> <span class="kn">as</span> <span class="nn">_os</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">_os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AIRFLOW_USE_NEW_IMPORTS&#39;</span><span class="p">,</span> <span class="bp">False</span><span class="p">):</span>
+            <span class="kn">from</span> <span class="nn">zope.deprecation</span> <span class="kn">import</span> <span class="n">deprecated</span> <span class="k">as</span> <span class="n">_deprecated</span>
+            <span class="k">for</span> <span class="n">_macro</span> <span class="ow">in</span> <span class="n">_macro_module</span><span class="o">.</span><span class="n">_objects</span><span class="p">:</span>
+                <span class="nb">globals</span><span class="p">()[</span><span class="n">_macro</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">_deprecated</span><span class="p">(</span>
+                    <span class="n">_macro</span><span class="p">,</span>
+                    <span class="s2">&quot;Importing plugin macro &#39;{i}&#39; directly from &quot;</span>
+                    <span class="s2">&quot;&#39;airflow.macros&#39; has been deprecated. Please &quot;</span>
+                    <span class="s2">&quot;import from &#39;airflow.macros.[plugin_module]&#39; &quot;</span>
+                    <span class="s2">&quot;instead. Support for direct imports will be dropped &quot;</span>
+                    <span class="s2">&quot;entirely in Airflow 2.0.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="o">=</span><span class="n">_macro</span><span class="p">))</span>
 </pre></div>
 
            </div>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/macros/hive.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/macros/hive.html b/_modules/airflow/macros/hive.html
index 43cd2a0..92000d3 100644
--- a/_modules/airflow/macros/hive.html
+++ b/_modules/airflow/macros/hive.html
@@ -139,7 +139,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.macros.hive</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">datetime</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">datetime</span>
 
 
 <div class="viewcode-block" id="max_partition"><a class="viewcode-back" href="../../../code.html#airflow.macros.hive.max_partition">[docs]</a><span class="k">def</span> <span class="nf">max_partition</span><span class="p">(</span>
@@ -166,7 +180,7 @@
 <span class="sd">    &gt;&gt;&gt; max_partition(&#39;airflow.static_babynames_partitioned&#39;)</span>
 <span class="sd">    &#39;2015-01-01&#39;</span>
 <span class="sd">    &#39;&#39;&#39;</span>
-    <span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span>
+    <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span>
     <span class="k">if</span> <span class="s1">&#39;.&#39;</span> <span class="ow">in</span> <span class="n">table</span><span class="p">:</span>
         <span class="n">schema</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
     <span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span><span class="n">metastore_conn_id</span><span class="o">=</span><span class="n">metastore_conn_id</span><span class="p">)</span>
@@ -219,7 +233,7 @@
 <span class="sd">    &gt;&gt;&gt; closest_ds_partition(tbl, &#39;2015-01-02&#39;)</span>
 <span class="sd">    &#39;2015-01-01&#39;</span>
 <span class="sd">    &#39;&#39;&#39;</span>
-    <span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span>
+    <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span>
     <span class="k">if</span> <span class="s1">&#39;.&#39;</span> <span class="ow">in</span> <span class="n">table</span><span class="p">:</span>
         <span class="n">schema</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
     <span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span><span class="n">metastore_conn_id</span><span class="o">=</span><span class="n">metastore_conn_id</span><span class="p">)</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/models.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/models.html b/_modules/airflow/models.html
index aba90cd..d4ed8a6 100644
--- a/_modules/airflow/models.html
+++ b/_modules/airflow/models.html
@@ -735,33 +735,43 @@
                        <span class="n">descriptor</span><span class="o">=</span><span class="nb">property</span><span class="p">(</span><span class="n">cls</span><span class="o">.</span><span class="n">get_extra</span><span class="p">,</span> <span class="n">cls</span><span class="o">.</span><span class="n">set_extra</span><span class="p">))</span>
 
     <span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-        <span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">hooks</span>
-        <span class="kn">from</span> <span class="nn">airflow.contrib</span> <span class="kn">import</span> <span class="n">hooks</span> <span class="k">as</span> <span class="n">contrib_hooks</span>
         <span class="k">try</span><span class="p">:</span>
             <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;mysql&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span>
+                <span class="k">return</span> <span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;google_cloud_platform&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">contrib_hooks</span><span class="o">.</span><span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
+                <span class="k">return</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;postgres&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">PostgresHook</span><span class="p">(</span><span class="n">postgres_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.postgres_hook</span> <span class="kn">import</span> <span class="n">PostgresHook</span>
+                <span class="k">return</span> <span class="n">PostgresHook</span><span class="p">(</span><span class="n">postgres_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;hive_cli&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HiveCliHook</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
+                <span class="k">return</span> <span class="n">HiveCliHook</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;presto&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">PrestoHook</span><span class="p">(</span><span class="n">presto_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.presto_hook</span> <span class="kn">import</span> <span class="n">PrestoHook</span>
+                <span class="k">return</span> <span class="n">PrestoHook</span><span class="p">(</span><span class="n">presto_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;hiveserver2&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HiveServer2Hook</span><span class="p">(</span><span class="n">hiveserver2_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span>
+                <span class="k">return</span> <span class="n">HiveServer2Hook</span><span class="p">(</span><span class="n">hiveserver2_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;sqlite&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">SqliteHook</span><span class="p">(</span><span class="n">sqlite_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.sqlite_hook</span> <span class="kn">import</span> <span class="n">SqliteHook</span>
+                <span class="k">return</span> <span class="n">SqliteHook</span><span class="p">(</span><span class="n">sqlite_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;jdbc&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">JdbcHook</span><span class="p">(</span><span class="n">jdbc_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.jdbc_hook</span> <span class="kn">import</span> <span class="n">JdbcHook</span>
+                <span class="k">return</span> <span class="n">JdbcHook</span><span class="p">(</span><span class="n">jdbc_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;mssql&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">MsSqlHook</span><span class="p">(</span><span class="n">mssql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.mssql_hook</span> <span class="kn">import</span> <span class="n">MsSqlHook</span>
+                <span class="k">return</span> <span class="n">MsSqlHook</span><span class="p">(</span><span class="n">mssql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;oracle&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">OracleHook</span><span class="p">(</span><span class="n">oracle_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.oracle_hook</span> <span class="kn">import</span> <span class="n">OracleHook</span>
+                <span class="k">return</span> <span class="n">OracleHook</span><span class="p">(</span><span class="n">oracle_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;vertica&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">contrib_hooks</span><span class="o">.</span><span class="n">VerticaHook</span><span class="p">(</span><span class="n">vertica_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.vertica_hook</span> <span class="kn">import</span> <span class="n">VerticaHook</span>
+                <span class="k">return</span> <span class="n">VerticaHook</span><span class="p">(</span><span class="n">vertica_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;cloudant&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">contrib_hooks</span><span class="o">.</span><span class="n">CloudantHook</span><span class="p">(</span><span class="n">cloudant_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.cloudant_hook</span> <span class="kn">import</span> <span class="n">CloudantHook</span>
+                <span class="k">return</span> <span class="n">CloudantHook</span><span class="p">(</span><span class="n">cloudant_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
         <span class="k">except</span><span class="p">:</span>
             <span class="k">return</span> <span class="bp">None</span>
 
@@ -844,6 +854,7 @@
 
     <span class="n">__table_args__</span> <span class="o">=</span> <span class="p">(</span>
         <span class="n">Index</span><span class="p">(</span><span class="s1">&#39;ti_dag_state&#39;</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">,</span> <span class="n">state</span><span class="p">),</span>
+        <span class="n">Index</span><span class="p">(</span><span class="s1">&#39;ti_state&#39;</span><span class="p">,</span> <span class="n">state</span><span class="p">),</span>
         <span class="n">Index</span><span class="p">(</span><span class="s1">&#39;ti_state_lkp&#39;</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">execution_date</span><span class="p">,</span> <span class="n">state</span><span class="p">),</span>
         <span class="n">Index</span><span class="p">(</span><span class="s1">&#39;ti_pool&#39;</span><span class="p">,</span> <span class="n">pool</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">priority_weight</span><span class="p">),</span>
     <span class="p">)</span>
@@ -1308,17 +1319,32 @@
 <div class="viewcode-block" id="TaskInstance.run"><a class="viewcode-back" href="../../code.html#airflow.models.TaskInstance.run">[docs]</a>    <span class="k">def</span> <span class="nf">run</span><span class="p">(</span>
             <span class="bp">self</span><span class="p">,</span>
             <span class="n">verbose</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
-            <span class="n">ignore_dependencies</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>  <span class="c1"># Doesn&#39;t check for deps, just runs</span>
-            <span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>   <span class="c1"># Ignore depends_on_past but respect</span>
-                                            <span class="c1"># other deps</span>
-            <span class="n">force</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>  <span class="c1"># Disregards previous successes</span>
-            <span class="n">mark_success</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>  <span class="c1"># Don&#39;t run the task, act as if it succeeded</span>
-            <span class="n">test_mode</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>  <span class="c1"># Doesn&#39;t record success or failure in the DB</span>
+            <span class="n">ignore_dependencies</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+            <span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+            <span class="n">force</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+            <span class="n">mark_success</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+            <span class="n">test_mode</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
             <span class="n">job_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">pool</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">session</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Runs the task instance.</span>
+
+<span class="sd">        :param verbose: whether to turn on more verbose loggin</span>
+<span class="sd">        :type verbose: boolean</span>
+<span class="sd">        :param ignore_dependencies: Doesn&#39;t check for deps, just runs</span>
+<span class="sd">        :type ignore_dependencies: boolean</span>
+<span class="sd">        :param ignore_depends_on_past: Ignore depends_on_past but respect</span>
+<span class="sd">            other dependencies</span>
+<span class="sd">        :type ignore_depends_on_past: boolean</span>
+<span class="sd">        :param force: Forces a run regarless of previous success</span>
+<span class="sd">        :type force: boolean</span>
+<span class="sd">        :param mark_success: Don&#39;t run the task, mark its state as success</span>
+<span class="sd">        :type mark_success: boolean</span>
+<span class="sd">        :param test_mode: Doesn&#39;t record success or failure in the DB</span>
+<span class="sd">        :type test_mode: boolean</span>
+<span class="sd">        :param pool: specifies the pool to use to run the task instance</span>
+<span class="sd">        :type pool: str</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">task</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">pool</span> <span class="o">=</span> <span class="n">pool</span> <span class="ow">or</span> <span class="n">task</span><span class="o">.</span><span class="n">pool</span>
@@ -1560,6 +1586,24 @@
         <span class="k">if</span> <span class="n">task</span><span class="o">.</span><span class="n">params</span><span class="p">:</span>
             <span class="n">params</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">params</span><span class="p">)</span>
 
+        <span class="k">class</span> <span class="nc">VariableAccessor</span><span class="p">:</span>
+            <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">            Wrapper around Variable. This way you can get variables in templates by using</span>
+<span class="sd">            {var.variable_name}.</span>
+<span class="sd">            &quot;&quot;&quot;</span>
+            <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+                <span class="k">pass</span>
+
+            <span class="k">def</span> <span class="nf">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span>
+                <span class="k">return</span> <span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
+
+        <span class="k">class</span> <span class="nc">VariableJsonAccessor</span><span class="p">:</span>
+            <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+                <span class="k">pass</span>
+
+            <span class="k">def</span> <span class="nf">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span>
+                <span class="k">return</span> <span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="n">deserialize_json</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
+
         <span class="k">return</span> <span class="p">{</span>
             <span class="s1">&#39;dag&#39;</span><span class="p">:</span> <span class="n">task</span><span class="o">.</span><span class="n">dag</span><span class="p">,</span>
             <span class="s1">&#39;ds&#39;</span><span class="p">:</span> <span class="n">ds</span><span class="p">,</span>
@@ -1585,6 +1629,10 @@
             <span class="s1">&#39;task_instance_key_str&#39;</span><span class="p">:</span> <span class="n">ti_key_str</span><span class="p">,</span>
             <span class="s1">&#39;conf&#39;</span><span class="p">:</span> <span class="n">configuration</span><span class="p">,</span>
             <span class="s1">&#39;test_mode&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_mode</span><span class="p">,</span>
+            <span class="s1">&#39;var&#39;</span><span class="p">:</span> <span class="p">{</span>
+                <span class="s1">&#39;value&#39;</span><span class="p">:</span> <span class="n">VariableAccessor</span><span class="p">(),</span>
+                <span class="s1">&#39;json&#39;</span><span class="p">:</span> <span class="n">VariableJsonAccessor</span><span class="p">()</span>
+            <span class="p">}</span>
         <span class="p">}</span>
 
     <span class="k">def</span> <span class="nf">render_templates</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
@@ -2706,6 +2754,21 @@
         <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span><span class="p">,</span> <span class="n">timedelta</span><span class="p">):</span>
             <span class="k">return</span> <span class="n">dttm</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span>
 
+<div class="viewcode-block" id="DAG.normalize_schedule"><a class="viewcode-back" href="../../code.html#airflow.models.DAG.normalize_schedule">[docs]</a>    <span class="k">def</span> <span class="nf">normalize_schedule</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dttm</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Returns dttm + interval unless dttm is first interval then it returns dttm</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">following</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">following_schedule</span><span class="p">(</span><span class="n">dttm</span><span class="p">)</span>
+
+        <span class="c1"># in case of @once</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">following</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">dttm</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">previous_schedule</span><span class="p">(</span><span class="n">following</span><span class="p">)</span> <span class="o">!=</span> <span class="n">dttm</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">following</span>
+
+        <span class="k">return</span> <span class="n">dttm</span></div>
+
     <span class="nd">@property</span>
     <span class="k">def</span> <span class="nf">tasks</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="o">.</span><span class="n">values</span><span class="p">())</span>
@@ -3485,10 +3548,12 @@
 
     <span class="nd">@staticmethod</span>
     <span class="nd">@provide_session</span>
-    <span class="k">def</span> <span class="nf">find</span><span class="p">(</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">run_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">execution_date</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+    <span class="k">def</span> <span class="nf">find</span><span class="p">(</span><span class="n">dag_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">run_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">execution_date</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
              <span class="n">state</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">external_trigger</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Returns a set of dag runs for the given search criteria.</span>
+<span class="sd">        :param dag_id: the dag_id to find dag runs for</span>
+<span class="sd">        :type dag_id: integer, list</span>
 <span class="sd">        :param run_id: defines the the run id for this dag run</span>
 <span class="sd">        :type run_id: string</span>
 <span class="sd">        :param execution_date: the execution date</span>
@@ -3502,7 +3567,9 @@
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">DR</span> <span class="o">=</span> <span class="n">DagRun</span>
 
-        <span class="n">qry</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DR</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">)</span>
+        <span class="n">qry</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DR</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">dag_id</span><span class="p">:</span>
+            <span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">)</span>
         <span class="k">if</span> <span class="n">run_id</span><span class="p">:</span>
             <span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">run_id</span> <span class="o">==</span> <span class="n">run_id</span><span class="p">)</span>
         <span class="k">if</span> <span class="n">execution_date</span><span class="p">:</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/operators/docker_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/docker_operator.html b/_modules/airflow/operators/docker_operator.html
index 46fcf7b..06aa65e 100644
--- a/_modules/airflow/operators/docker_operator.html
+++ b/_modules/airflow/operators/docker_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.operators.docker_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">json</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">json</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>


Mime
View raw message