airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maximebeauche...@apache.org
Subject [3/5] incubator-airflow-site git commit: Updating the disclaimer
Date Fri, 01 Jul 2016 21:54:29 GMT
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/hive_to_druid.html
----------------------------------------------------------------------
diff --git a/_modules/hive_to_druid.html b/_modules/hive_to_druid.html
index cbd1586..d85bfb3 100644
--- a/_modules/hive_to_druid.html
+++ b/_modules/hive_to_druid.html
@@ -137,9 +137,24 @@
            <div itemprop="articleBody">
             
   <h1>Source code for hive_to_druid</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">DruidHook</span><span class="p">,</span> <span class="n">HiveMetastoreHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">HiveMetastoreHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.druid_hook</span> <span class="kn">import</span> <span class="n">DruidHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
@@ -240,19 +255,20 @@
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Inserting rows into Druid&quot;</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;HDFS path: &quot;</span> <span class="o">+</span> <span class="n">static_path</span><span class="p">)</span>
 
-        <span class="n">druid</span><span class="o">.</span><span class="n">load_from_hdfs</span><span class="p">(</span>
-            <span class="n">datasource</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">druid_datasource</span><span class="p">,</span>
-            <span class="n">intervals</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">intervals</span><span class="p">,</span>
-            <span class="n">static_path</span><span class="o">=</span><span class="n">static_path</span><span class="p">,</span> <span class="n">ts_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">ts_dim</span><span class="p">,</span>
-            <span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">target_partition_size</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">target_partition_size</span><span class="p">,</span>
-            <span class="n">metric_spec</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metric_spec</span><span class="p">,</span> <span class="n">hadoop_dependency_coordinates</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hadoop_dependency_coordinates</span><span class="p">)</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Load seems to have succeeded!&quot;</span><span class="p">)</span>
-
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
-            <span class="s2">&quot;Cleaning up by dropping the temp &quot;</span>
-            <span class="s2">&quot;Hive table {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">))</span>
-        <span class="n">hql</span> <span class="o">=</span> <span class="s2">&quot;DROP TABLE IF EXISTS {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">)</span>
-        <span class="n">hive</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="p">)</span></div>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">druid</span><span class="o">.</span><span class="n">load_from_hdfs</span><span class="p">(</span>
+                <span class="n">datasource</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">druid_datasource</span><span class="p">,</span>
+                <span class="n">intervals</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">intervals</span><span class="p">,</span>
+                <span class="n">static_path</span><span class="o">=</span><span class="n">static_path</span><span class="p">,</span> <span class="n">ts_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">ts_dim</span><span class="p">,</span>
+                <span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">target_partition_size</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">target_partition_size</span><span class="p">,</span>
+                <span class="n">metric_spec</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metric_spec</span><span class="p">,</span> <span class="n">hadoop_dependency_coordinates</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hadoop_dependency_coordinates</span><span class="p">)</span>
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Load seems to have succeeded!&quot;</span><span class="p">)</span>
+        <span class="k">finally</span><span class="p">:</span>
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                <span class="s2">&quot;Cleaning up by dropping the temp &quot;</span>
+                <span class="s2">&quot;Hive table {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">))</span>
+            <span class="n">hql</span> <span class="o">=</span> <span class="s2">&quot;DROP TABLE IF EXISTS {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">)</span>
+            <span class="n">hive</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="p">)</span></div>
 </pre></div>
 
            </div>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/hive_to_mysql.html
----------------------------------------------------------------------
diff --git a/_modules/hive_to_mysql.html b/_modules/hive_to_mysql.html
index 6a5bedd..867023b 100644
--- a/_modules/hive_to_mysql.html
+++ b/_modules/hive_to_mysql.html
@@ -137,9 +137,24 @@
            <div itemprop="articleBody">
             
   <h1>Source code for hive_to_mysql</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span><span class="p">,</span> <span class="n">MySqlHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/hive_to_samba_operator.html
----------------------------------------------------------------------
diff --git a/_modules/hive_to_samba_operator.html b/_modules/hive_to_samba_operator.html
index 3008244..a69680d 100644
--- a/_modules/hive_to_samba_operator.html
+++ b/_modules/hive_to_samba_operator.html
@@ -137,10 +137,25 @@
            <div itemprop="articleBody">
             
   <h1>Source code for hive_to_samba_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">tempfile</span>
 
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span><span class="p">,</span> <span class="n">SambaHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.samba_hook</span> <span class="kn">import</span> <span class="n">SambaHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/http_hook.html
----------------------------------------------------------------------
diff --git a/_modules/http_hook.html b/_modules/http_hook.html
index 98ed5ad..b62b0d0 100644
--- a/_modules/http_hook.html
+++ b/_modules/http_hook.html
@@ -150,7 +150,7 @@
 <span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
 <span class="c1"># See the License for the specific language governing permissions and</span>
 <span class="c1"># limitations under the License.</span>
-<span class="c1">#</span>
+
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/http_operator.html
----------------------------------------------------------------------
diff --git a/_modules/http_operator.html b/_modules/http_operator.html
index 942ae0e..00496c9 100644
--- a/_modules/http_operator.html
+++ b/_modules/http_operator.html
@@ -137,10 +137,24 @@
            <div itemprop="articleBody">
             
   <h1>Source code for http_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
 
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HttpHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.http_hook</span> <span class="kn">import</span> <span class="n">HttpHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/mssql_hook.html
----------------------------------------------------------------------
diff --git a/_modules/mssql_hook.html b/_modules/mssql_hook.html
index b3b5503..9aff59d 100644
--- a/_modules/mssql_hook.html
+++ b/_modules/mssql_hook.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for mssql_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">pymssql</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">pymssql</span>
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>
 
@@ -157,10 +171,11 @@
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mssql_conn_id</span><span class="p">)</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="n">pymssql</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span>
-            <span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
-            <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
-            <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span>
-            <span class="n">conn</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span>
+            <span class="n">server</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+            <span class="n">user</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
+            <span class="n">password</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span>
+            <span class="n">database</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span>
+            <span class="n">port</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span>
         <span class="k">return</span> <span class="n">conn</span></div>
 
     <span class="k">def</span> <span class="nf">set_autocommit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">):</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/mssql_operator.html
----------------------------------------------------------------------
diff --git a/_modules/mssql_operator.html b/_modules/mssql_operator.html
index dcc2872..e933004 100644
--- a/_modules/mssql_operator.html
+++ b/_modules/mssql_operator.html
@@ -137,9 +137,23 @@
            <div itemprop="articleBody">
             
   <h1>Source code for mssql_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">MsSqlHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.mssql_hook</span> <span class="kn">import</span> <span class="n">MsSqlHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/mssql_to_hive.html
----------------------------------------------------------------------
diff --git a/_modules/mssql_to_hive.html b/_modules/mssql_to_hive.html
index 451dc7f..590dfb8 100644
--- a/_modules/mssql_to_hive.html
+++ b/_modules/mssql_to_hive.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for mssql_to_hive</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
 <span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
 <span class="kn">import</span> <span class="nn">unicodecsv</span> <span class="kn">as</span> <span class="nn">csv</span>
 <span class="kn">import</span> <span class="nn">logging</span>
@@ -145,7 +159,8 @@
 <span class="kn">import</span> <span class="nn">pymssql</span>
 
 
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">MsSqlHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.mssql_hook</span> <span class="kn">import</span> <span class="n">MsSqlHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/mysql_hook.html
----------------------------------------------------------------------
diff --git a/_modules/mysql_hook.html b/_modules/mysql_hook.html
index 23d3726..941737f 100644
--- a/_modules/mysql_hook.html
+++ b/_modules/mysql_hook.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for mysql_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">MySQLdb</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">MySQLdb</span>
 <span class="kn">import</span> <span class="nn">MySQLdb.cursors</span>
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/mysql_operator.html
----------------------------------------------------------------------
diff --git a/_modules/mysql_operator.html b/_modules/mysql_operator.html
index 331f2b4..a1fc11a 100644
--- a/_modules/mysql_operator.html
+++ b/_modules/mysql_operator.html
@@ -137,9 +137,23 @@
            <div itemprop="articleBody">
             
   <h1>Source code for mysql_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">MySqlHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/mysql_to_hive.html
----------------------------------------------------------------------
diff --git a/_modules/mysql_to_hive.html b/_modules/mysql_to_hive.html
index fa6b20f..cc8496b 100644
--- a/_modules/mysql_to_hive.html
+++ b/_modules/mysql_to_hive.html
@@ -137,14 +137,29 @@
            <div itemprop="articleBody">
             
   <h1>Source code for mysql_to_hive</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
 <span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
 <span class="kn">import</span> <span class="nn">unicodecsv</span> <span class="kn">as</span> <span class="nn">csv</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span>
 <span class="kn">import</span> <span class="nn">MySQLdb</span>
 
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">MySqlHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/postgres_hook.html
----------------------------------------------------------------------
diff --git a/_modules/postgres_hook.html b/_modules/postgres_hook.html
index d8896ac..3ecfa69 100644
--- a/_modules/postgres_hook.html
+++ b/_modules/postgres_hook.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for postgres_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">psycopg2</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">psycopg2</span>
 <span class="kn">import</span> <span class="nn">psycopg2.extensions</span>
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/postgres_operator.html
----------------------------------------------------------------------
diff --git a/_modules/postgres_operator.html b/_modules/postgres_operator.html
index 4eb0e00..5ade37d 100644
--- a/_modules/postgres_operator.html
+++ b/_modules/postgres_operator.html
@@ -137,9 +137,23 @@
            <div itemprop="articleBody">
             
   <h1>Source code for postgres_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">PostgresHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.postgres_hook</span> <span class="kn">import</span> <span class="n">PostgresHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/presto_check_operator.html
----------------------------------------------------------------------
diff --git a/_modules/presto_check_operator.html b/_modules/presto_check_operator.html
index 52dc0e6..e24b271 100644
--- a/_modules/presto_check_operator.html
+++ b/_modules/presto_check_operator.html
@@ -137,8 +137,22 @@
            <div itemprop="articleBody">
             
   <h1>Source code for presto_check_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">PrestoHook</span>
-<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="kn">import</span> <span class="n">CheckOperator</span><span class="p">,</span> <span class="n">ValueCheckOperator</span><span class="p">,</span> <span class="n">IntervalCheckOperator</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.presto_hook</span> <span class="kn">import</span> <span class="n">PrestoHook</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.check_operator</span> <span class="kn">import</span> <span class="n">CheckOperator</span><span class="p">,</span> <span class="n">ValueCheckOperator</span><span class="p">,</span> <span class="n">IntervalCheckOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/presto_hook.html
----------------------------------------------------------------------
diff --git a/_modules/presto_hook.html b/_modules/presto_hook.html
index 73e4cc0..49b93a9 100644
--- a/_modules/presto_hook.html
+++ b/_modules/presto_hook.html
@@ -137,13 +137,28 @@
            <div itemprop="articleBody">
             
   <h1>Source code for presto_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span class="kn">import</span> <span class="nn">logging</span>
+
 <span class="kn">from</span> <span class="nn">pyhive</span> <span class="kn">import</span> <span class="n">presto</span>
 <span class="kn">from</span> <span class="nn">pyhive.exc</span> <span class="kn">import</span> <span class="n">DatabaseError</span>
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>
 
-<span class="kn">import</span> <span class="nn">logging</span>
 <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;pyhive&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">)</span>
 
 
@@ -204,8 +219,7 @@
             <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">PrestoHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">get_first</span><span class="p">(</span>
                 <span class="bp">self</span><span class="o">.</span><span class="n">_strip_sql</span><span class="p">(</span><span class="n">hql</span><span class="p">),</span> <span class="n">parameters</span><span class="p">)</span>
         <span class="k">except</span> <span class="n">DatabaseError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
-            <span class="n">obj</span> <span class="o">=</span> <span class="nb">eval</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
-            <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="n">obj</span><span class="p">[</span><span class="s1">&#39;message&#39;</span><span class="p">])</span></div>
+            <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="n">e</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">&#39;message&#39;</span><span class="p">])</span></div>
 
 <div class="viewcode-block" id="PrestoHook.get_pandas_df"><a class="viewcode-back" href="../code.html#airflow.hooks.PrestoHook.get_pandas_df">[docs]</a>    <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
@@ -217,8 +231,7 @@
             <span class="n">cursor</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_strip_sql</span><span class="p">(</span><span class="n">hql</span><span class="p">),</span> <span class="n">parameters</span><span class="p">)</span>
             <span class="n">data</span> <span class="o">=</span> <span class="n">cursor</span><span class="o">.</span><span class="n">fetchall</span><span class="p">()</span>
         <span class="k">except</span> <span class="n">DatabaseError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
-            <span class="n">obj</span> <span class="o">=</span> <span class="nb">eval</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
-            <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="n">obj</span><span class="p">[</span><span class="s1">&#39;message&#39;</span><span class="p">])</span>
+            <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="n">e</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">&#39;message&#39;</span><span class="p">])</span>
         <span class="n">column_descriptions</span> <span class="o">=</span> <span class="n">cursor</span><span class="o">.</span><span class="n">description</span>
         <span class="k">if</span> <span class="n">data</span><span class="p">:</span>
             <span class="n">df</span> <span class="o">=</span> <span class="n">pandas</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/python_operator.html
----------------------------------------------------------------------
diff --git a/_modules/python_operator.html b/_modules/python_operator.html
index 0f0de31..059ebb2 100644
--- a/_modules/python_operator.html
+++ b/_modules/python_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for python_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
 <span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/qubole_operator.html
----------------------------------------------------------------------
diff --git a/_modules/qubole_operator.html b/_modules/qubole_operator.html
index 07ea626..fb33047 100644
--- a/_modules/qubole_operator.html
+++ b/_modules/qubole_operator.html
@@ -137,9 +137,23 @@
            <div itemprop="articleBody">
             
   <h1>Source code for qubole_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks</span> <span class="kn">import</span> <span class="n">QuboleHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.qubole_hook</span> <span class="kn">import</span> <span class="n">QuboleHook</span>
 
 
 <div class="viewcode-block" id="QuboleOperator"><a class="viewcode-back" href="../code.html#airflow.contrib.operators.QuboleOperator">[docs]</a><span class="k">class</span> <span class="nc">QuboleOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -220,7 +234,7 @@
 
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;query&#39;</span><span class="p">,</span> <span class="s1">&#39;script_location&#39;</span><span class="p">,</span> <span class="s1">&#39;sub_command&#39;</span><span class="p">,</span> <span class="s1">&#39;script&#39;</span><span class="p">,</span> <span class="s1">&#39;files&#39;</span><span class="p">,</span> <span class="s1">&#39;archives&#39;</span><span class="p">,</span> <span class="s1">&#39;program&#39;</span><span class="p">,</span> <span class="s1">&#39;cmdline&#39;</span><span class="p">,</span>
                        <span class="s1">&#39;sql&#39;</span><span class="p">,</span> <span class="s1">&#39;where_clause&#39;</span><span class="p">,</span> <span class="s1">&#39;extract_query&#39;</span><span class="p">,</span> <span class="s1">&#39;boundary_query&#39;</span><span class="p">,</span> <span class="s1">&#39;macros&#39;</span><span class="p">,</span> <span class="s1">&#39;tags&#39;</span><span class="p">,</span> <span class="s1">&#39;name&#39;</span><span class="p">,</span> <span class="s1">&#39;parameters&#39;</span><span class="p">)</span>
-    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.txt&#39;</span><span class="p">)</span>
+    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.txt&#39;</span><span class="p">,)</span>
     <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#3064A1&#39;</span>
     <span class="n">ui_fgcolor</span> <span class="o">=</span> <span class="s1">&#39;#fff&#39;</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/s3_to_hive_operator.html
----------------------------------------------------------------------
diff --git a/_modules/s3_to_hive_operator.html b/_modules/s3_to_hive_operator.html
index dc23eb5..ad7715e 100644
--- a/_modules/s3_to_hive_operator.html
+++ b/_modules/s3_to_hive_operator.html
@@ -137,13 +137,28 @@
            <div itemprop="articleBody">
             
   <h1>Source code for s3_to_hive_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">next</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">next</span>
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">zip</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span>
 
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">S3Hook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.S3_hook</span> <span class="kn">import</span> <span class="n">S3Hook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/sensors.html
----------------------------------------------------------------------
diff --git a/_modules/sensors.html b/_modules/sensors.html
index 19336b2..fab6313 100644
--- a/_modules/sensors.html
+++ b/_modules/sensors.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for sensors</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
 <span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
 <span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
@@ -146,10 +160,11 @@
 <span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlparse</span>
 <span class="kn">from</span> <span class="nn">time</span> <span class="kn">import</span> <span class="n">sleep</span>
 
+<span class="kn">import</span> <span class="nn">airflow</span>
 <span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">hooks</span><span class="p">,</span> <span class="n">settings</span>
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span><span class="p">,</span> <span class="n">AirflowSensorTimeout</span><span class="p">,</span> <span class="n">AirflowSkipException</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span><span class="p">,</span> <span class="n">TaskInstance</span><span class="p">,</span> <span class="n">Connection</span> <span class="k">as</span> <span class="n">DB</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="kn">import</span> <span class="n">State</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
@@ -193,12 +208,12 @@
     <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="n">started_at</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span>
         <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">poke</span><span class="p">(</span><span class="n">context</span><span class="p">):</span>
-            <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">poke_interval</span><span class="p">)</span>
-            <span class="k">if</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">started_at</span><span class="p">)</span><span class="o">.</span><span class="n">seconds</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">:</span>
+            <span class="k">if</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">started_at</span><span class="p">)</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">()</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">:</span>
                 <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">soft_fail</span><span class="p">:</span>
                     <span class="k">raise</span> <span class="n">AirflowSkipException</span><span class="p">(</span><span class="s1">&#39;Snap. Time is OUT.&#39;</span><span class="p">)</span>
                 <span class="k">else</span><span class="p">:</span>
                     <span class="k">raise</span> <span class="n">AirflowSensorTimeout</span><span class="p">(</span><span class="s1">&#39;Snap. Time is OUT.&#39;</span><span class="p">)</span>
+            <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">poke_interval</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Success criteria met. Exiting.&quot;</span><span class="p">)</span>
 
 
@@ -303,8 +318,14 @@
 <span class="sd">    :type allowed_states: list</span>
 <span class="sd">    :param execution_delta: time difference with the previous execution to</span>
 <span class="sd">        look at, the default is the same execution_date as the current task.</span>
-<span class="sd">        For yesterday, use [positive!] datetime.timedelta(days=1)</span>
+<span class="sd">        For yesterday, use [positive!] datetime.timedelta(days=1). Either</span>
+<span class="sd">        execution_delta or execution_date_fn can be passed to</span>
+<span class="sd">        ExternalTaskSensor, but not both.</span>
 <span class="sd">    :type execution_delta: datetime.timedelta</span>
+<span class="sd">    :param execution_date_fn: function that receives the current execution date</span>
+<span class="sd">        and returns the desired execution date to query. Either execution_delta</span>
+<span class="sd">        or execution_date_fn can be passed to ExternalTaskSensor, but not both.</span>
+<span class="sd">    :type execution_date_fn: callable</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="nd">@apply_defaults</span>
@@ -314,16 +335,25 @@
             <span class="n">external_task_id</span><span class="p">,</span>
             <span class="n">allowed_states</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">execution_delta</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">execution_date_fn</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
         <span class="nb">super</span><span class="p">(</span><span class="n">ExternalTaskSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">allowed_states</span> <span class="o">=</span> <span class="n">allowed_states</span> <span class="ow">or</span> <span class="p">[</span><span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">execution_delta</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span> <span class="ow">and</span> <span class="n">execution_date_fn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s1">&#39;Only one of `execution_date` or `execution_date_fn` may&#39;</span>
+                <span class="s1">&#39;be provided to ExternalTaskSensor; not both.&#39;</span><span class="p">)</span>
+
         <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span> <span class="o">=</span> <span class="n">execution_delta</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span> <span class="o">=</span> <span class="n">execution_date_fn</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">external_dag_id</span> <span class="o">=</span> <span class="n">external_dag_id</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">external_task_id</span> <span class="o">=</span> <span class="n">external_task_id</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span><span class="p">:</span>
             <span class="n">dttm</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">&#39;execution_date&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span><span class="p">:</span>
+            <span class="n">dttm</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span><span class="p">(</span><span class="n">context</span><span class="p">[</span><span class="s1">&#39;execution_date&#39;</span><span class="p">])</span>
         <span class="k">else</span><span class="p">:</span>
             <span class="n">dttm</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">&#39;execution_date&#39;</span><span class="p">]</span>
 
@@ -346,15 +376,91 @@
         <span class="k">return</span> <span class="n">count</span></div>
 
 
+<span class="k">class</span> <span class="nc">NamedHivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Waits for a set of partitions to show up in Hive.</span>
+
+<span class="sd">    :param partition_names: List of fully qualified names of the</span>
+<span class="sd">        partitions to wait for. A fully qualified name is of the</span>
+<span class="sd">        form schema.table/pk1=pv1/pk2=pv2, for example,</span>
+<span class="sd">        default.users/ds=2016-01-01. This is passed as is to the metastore</span>
+<span class="sd">        Thrift client &quot;get_partitions_by_name&quot; method. Note that</span>
+<span class="sd">        you cannot use logical operators as in HivePartitionSensor.</span>
+<span class="sd">    :type partition_names: list of strings</span>
+<span class="sd">    :param metastore_conn_id: reference to the metastore thrift service</span>
+<span class="sd">        connection id</span>
+<span class="sd">    :type metastore_conn_id: str</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;partition_names&#39;</span><span class="p">,</span> <span class="p">)</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">partition_names</span><span class="p">,</span>
+            <span class="n">metastore_conn_id</span><span class="o">=</span><span class="s1">&#39;metastore_default&#39;</span><span class="p">,</span>
+            <span class="n">poke_interval</span><span class="o">=</span><span class="mi">60</span><span class="o">*</span><span class="mi">3</span><span class="p">,</span>
+            <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">NamedHivePartitionSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span>
+            <span class="n">poke_interval</span><span class="o">=</span><span class="n">poke_interval</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">partition_names</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
+            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;partition_names must be an array of strings&#39;</span><span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">partition_name</span> <span class="ow">in</span> <span class="n">partition_names</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">parse_partition_name</span><span class="p">(</span><span class="n">partition_name</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span> <span class="o">=</span> <span class="n">metastore_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span> <span class="o">=</span> <span class="n">partition_names</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">=</span> <span class="mi">0</span>
+
+    <span class="k">def</span> <span class="nf">parse_partition_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">partition</span><span class="p">):</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">schema</span><span class="p">,</span> <span class="n">table_partition</span> <span class="o">=</span> <span class="n">partition</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
+            <span class="n">table</span><span class="p">,</span> <span class="n">partition</span> <span class="o">=</span> <span class="n">table_partition</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span>
+        <span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Could not parse &#39;</span> <span class="o">+</span> <span class="n">partition</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">&#39;hook&#39;</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">hive_hooks</span><span class="o">.</span><span class="n">HiveMetastoreHook</span><span class="p">(</span>
+                <span class="n">metastore_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span><span class="p">)</span>
+
+        <span class="k">def</span> <span class="nf">poke_partition</span><span class="p">(</span><span class="n">partition</span><span class="p">):</span>
+
+            <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_partition_name</span><span class="p">(</span><span class="n">partition</span><span class="p">)</span>
+
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                <span class="s1">&#39;Poking for {schema}.{table}/{partition}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
+            <span class="p">)</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">check_for_named_partition</span><span class="p">(</span>
+                <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span><span class="p">)</span>
+
+        <span class="k">while</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span><span class="p">):</span>
+            <span class="k">if</span> <span class="n">poke_partition</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span><span class="p">]):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">+=</span> <span class="mi">1</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">return</span> <span class="bp">False</span>
+
+        <span class="k">return</span> <span class="bp">True</span>
+
+
 <div class="viewcode-block" id="HivePartitionSensor"><a class="viewcode-back" href="../code.html#airflow.operators.HivePartitionSensor">[docs]</a><span class="k">class</span> <span class="nc">HivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Waits for a partition to show up in Hive</span>
+<span class="sd">    Waits for a partition to show up in Hive.</span>
+
+<span class="sd">    Note: Because @partition supports general logical operators, it</span>
+<span class="sd">    can be inefficient. Consider using NamedHivePartitionSensor instead if</span>
+<span class="sd">    you don&#39;t need the full flexibility of HivePartitionSensor.</span>
 
 <span class="sd">    :param table: The name of the table to wait for, supports the dot</span>
 <span class="sd">        notation (my_database.my_table)</span>
 <span class="sd">    :type table: string</span>
 <span class="sd">    :param partition: The partition clause to wait for. This is passed as</span>
-<span class="sd">        is to the Metastore Thrift client &quot;get_partitions_by_filter&quot; method,</span>
+<span class="sd">        is to the metastore Thrift client &quot;get_partitions_by_filter&quot; method,</span>
 <span class="sd">        and apparently supports SQL like notation as in `ds=&#39;2015-01-01&#39;</span>
 <span class="sd">        AND type=&#39;value&#39;` and &gt; &lt; sings as in &quot;ds&gt;=2015-01-01&quot;</span>
 <span class="sd">    :type partition: string</span>
@@ -388,7 +494,7 @@
             <span class="s1">&#39;Poking for table {self.schema}.{self.table}, &#39;</span>
             <span class="s1">&#39;partition {self.partition}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
         <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">&#39;hook&#39;</span><span class="p">):</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HiveMetastoreHook</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">hive_hooks</span><span class="o">.</span><span class="n">HiveMetastoreHook</span><span class="p">(</span>
                 <span class="n">metastore_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span><span class="p">)</span>
         <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">check_for_partition</span><span class="p">(</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">partition</span><span class="p">)</span></div>
@@ -411,7 +517,8 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span> <span class="o">=</span> <span class="n">hdfs_conn_id</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">sb</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="kn">import</span> <span class="nn">airflow.hooks.hdfs_hook</span>
+        <span class="n">sb</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">hdfs_hook</span><span class="o">.</span><span class="n">HDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;snakebite&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">WARNING</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
             <span class="s1">&#39;Poking for file {self.filepath} &#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
@@ -439,7 +546,7 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span> <span class="o">=</span> <span class="n">webhdfs_conn_id</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">c</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">WebHDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span><span class="p">)</span>
+        <span class="n">c</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">webhdfs_hook</span><span class="o">.</span><span class="n">WebHDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
             <span class="s1">&#39;Poking for file {self.filepath} &#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
         <span class="k">return</span> <span class="n">c</span><span class="o">.</span><span class="n">check_for_path</span><span class="p">(</span><span class="n">hdfs_path</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">)</span></div>
@@ -495,7 +602,8 @@
         <span class="n">session</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
+        <span class="kn">import</span> <span class="nn">airflow.hooks.S3_hook</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">S3_hook</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
         <span class="n">full_url</span> <span class="o">=</span> <span class="s2">&quot;s3://&quot;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span> <span class="o">+</span> <span class="s2">&quot;/&quot;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_key</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Poking for key : {full_url}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">wildcard_match</span><span class="p">:</span>
@@ -547,7 +655,8 @@
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Poking for prefix : {self.prefix}</span><span class="se">\n</span><span class="s1">&#39;</span>
                      <span class="s1">&#39;in bucket s3://{self.bucket_name}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
-        <span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
+        <span class="kn">import</span> <span class="nn">airflow.hooks.S3_hook</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">S3_hook</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
         <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">check_for_prefix</span><span class="p">(</span>
             <span class="n">prefix</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prefix</span><span class="p">,</span>
             <span class="n">delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span><span class="p">,</span>
@@ -639,7 +748,7 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">extra_options</span> <span class="o">=</span> <span class="n">extra_options</span> <span class="ow">or</span> <span class="p">{}</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span> <span class="o">=</span> <span class="n">response_check</span>
 
-        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HttpHook</span><span class="p">(</span><span class="n">method</span><span class="o">=</span><span class="s1">&#39;GET&#39;</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="o">=</span><span class="n">http_conn_id</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">http_hook</span><span class="o">.</span><span class="n">HttpHook</span><span class="p">(</span><span class="n">method</span><span class="o">=</span><span class="s1">&#39;GET&#39;</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="o">=</span><span class="n">http_conn_id</span><span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Poking: &#39;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">endpoint</span><span class="p">)</span>
@@ -652,7 +761,7 @@
                 <span class="c1"># run content check on response</span>
                 <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
         <span class="k">except</span> <span class="n">AirflowException</span> <span class="k">as</span> <span class="n">ae</span><span class="p">:</span>
-            <span class="k">if</span> <span class="n">ae</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;404&quot;</span><span class="p">):</span>
+            <span class="k">if</span> <span class="nb">str</span><span class="p">(</span><span class="n">ae</span><span class="p">)</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;404&quot;</span><span class="p">):</span>
                 <span class="k">return</span> <span class="bp">False</span>
 
             <span class="k">raise</span> <span class="n">ae</span>


Mime
View raw message