airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [airflow-site] 09/30: 1.9.0
Date Wed, 27 Nov 2019 19:14:18 GMT
This is an automated email from the ASF dual-hosted git repository.

ash pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/airflow-site.git

commit 28a3eb6000f0d877c8c8088facb82bf2ab169c8d
Author: Chris Riccomini <criccomini@apache.org>
AuthorDate: Wed Jan 3 09:47:55 2018 -0800

    1.9.0
---
 _modules/S3_hook.html                              |  455 ++--
 .../airflow/contrib/executors/mesos_executor.html  |   58 +-
 .../{ => airflow/contrib/hooks}/bigquery_hook.html |  606 +++--
 _modules/airflow/contrib/hooks/datastore_hook.html |  410 +++
 _modules/airflow/contrib/hooks/emr_hook.html       |  295 ++
 .../airflow/contrib/hooks/gcp_dataflow_hook.html   |  395 +++
 .../airflow/contrib/hooks/gcp_mlengine_hook.html   |  500 ++++
 _modules/airflow/contrib/hooks/gcs_hook.html       |  264 +-
 _modules/airflow/contrib/hooks/wasb_hook.html      |  345 +++
 .../operators/bigquery_check_operator.html}        |  137 +-
 .../contrib/operators/bigquery_operator.html       |  144 +-
 ...ery_operator.html => bigquery_to_bigquery.html} |  149 +-
 .../airflow/contrib/operators/bigquery_to_gcs.html |  157 +-
 .../contrib/operators/databricks_operator.html     |  487 ++++
 .../contrib/operators/dataflow_operator.html       |  470 ++++
 .../contrib/operators/dataproc_operator.html       | 1139 ++++++++
 .../airflow/contrib/operators/ecs_operator.html    |  360 +++
 .../contrib/operators/emr_add_steps_operator.html  |  291 ++
 .../operators/emr_create_job_flow_operator.html    |  296 +++
 .../operators/emr_terminate_job_flow_operator.html |  285 ++
 .../{bigquery_operator.html => file_to_wasb.html}  |  143 +-
 .../contrib/operators/gcs_download_operator.html   |  134 +-
 _modules/airflow/contrib/operators/gcs_to_bq.html  |  429 +++
 .../contrib/operators/hipchat_operator.html        |   18 +-
 .../contrib/operators/mlengine_operator.html       |  795 ++++++
 _modules/airflow/contrib/sensors/wasb_sensor.html  |  327 +++
 _modules/airflow/executors/local_executor.html     |   20 +-
 .../airflow/executors/sequential_executor.html     |   13 +-
 _modules/airflow/hooks/S3_hook.html                |  489 ++++
 _modules/airflow/macros.html                       |   12 +-
 _modules/airflow/macros/hive.html                  |   11 +-
 _modules/airflow/models.html                       | 1466 +++++++---
 _modules/airflow/operators/docker_operator.html    |  197 +-
 .../airflow/operators/redshift_to_s3_operator.html |  337 +++
 .../operators/s3_file_transform_operator.html      |  342 +++
 .../airflow/operators/s3_to_hive_operator.html     |  482 ++++
 _modules/airflow/operators/sensors.html            |  168 +-
 _modules/bash_operator.html                        |   42 +-
 _modules/bigquery_hook.html                        |  544 +++-
 _modules/dagrun_operator.html                      |   17 +-
 _modules/dbapi_hook.html                           |  153 +-
 _modules/druid_hook.html                           |  271 +-
 _modules/dummy_operator.html                       |    9 +-
 _modules/email_operator.html                       |   11 +-
 _modules/ftp_hook.html                             |   29 +-
 _modules/gcs_hook.html                             |  256 +-
 _modules/generic_transfer.html                     |   24 +-
 _modules/hive_hooks.html                           |  329 ++-
 _modules/hive_operator.html                        |   80 +-
 _modules/hive_to_druid.html                        |  207 +-
 _modules/hive_to_mysql.html                        |   86 +-
 _modules/http_hook.html                            |   31 +-
 _modules/http_operator.html                        |   17 +-
 _modules/index.html                                |   53 +-
 _modules/mysql_hook.html                           |   31 +-
 _modules/mysql_operator.html                       |   22 +-
 _modules/mysql_to_hive.html                        |   90 +-
 _modules/postgres_hook.html                        |  105 +-
 _modules/postgres_operator.html                    |   76 +-
 _modules/presto_check_operator.html                |   13 +-
 _modules/presto_hook.html                          |   37 +-
 _modules/python_operator.html                      |  297 ++-
 _modules/qubole_operator.html                      |  144 +-
 _modules/s3_to_hive_operator.html                  |  259 +-
 _modules/sensors.html                              |  178 +-
 _modules/slack_operator.html                       |   77 +-
 _modules/sqlite_hook.html                          |    7 +-
 _sources/api.rst.txt                               |   20 +-
 _sources/code.rst.txt                              |   14 +-
 _sources/concepts.rst.txt                          |   10 +-
 _sources/configuration.rst.txt                     |   63 +-
 _sources/faq.rst.txt                               |   12 +-
 _sources/installation.rst.txt                      |  124 +-
 _sources/integration.rst.txt                       |  351 ++-
 _sources/scheduler.rst.txt                         |   22 +-
 _sources/security.rst.txt                          |   82 +-
 _sources/start.rst.txt                             |    2 +-
 _sources/tutorial.rst.txt                          |   19 +-
 _static/basic.css                                  |   15 +-
 _static/doctools.js                                |   54 +-
 _static/pygments.css                               |    4 +
 _static/searchtools.js                             |    5 +-
 _static/websupport.js                              |    2 +-
 api.html                                           |   28 +-
 cli.html                                           | 1413 ++++++----
 code.html                                          | 2806 ++++++++++++++++----
 concepts.html                                      |  105 +-
 configuration.html                                 |   83 +-
 faq.html                                           |   45 +-
 genindex.html                                      |  380 ++-
 index.html                                         |  192 +-
 installation.html                                  |   79 +-
 integration.html                                   | 1738 +++++++++++-
 license.html                                       |    7 +-
 objects.inv                                        |  Bin 2159 -> 3480 bytes
 plugins.html                                       |   21 +-
 profiling.html                                     |   11 +-
 project.html                                       |   19 +-
 py-modindex.html                                   |    7 +-
 scheduler.html                                     |   68 +-
 search.html                                        |    7 +-
 searchindex.js                                     |    2 +-
 security.html                                      |  140 +-
 start.html                                         |   23 +-
 tutorial.html                                      |   85 +-
 ui.html                                            |   15 +-
 106 files changed, 20079 insertions(+), 4115 deletions(-)

diff --git a/_modules/S3_hook.html b/_modules/S3_hook.html
index 40a14c3..ae1091a 100644
--- a/_modules/S3_hook.html
+++ b/_modules/S3_hook.html
@@ -13,6 +13,8 @@
 
   
   
+  
+  
 
   
 
@@ -30,6 +32,9 @@
   
 
   
+        <link rel="index" title="Index"
+              href="../genindex.html"/>
+        <link rel="search" title="Search" href="../search.html"/>
     <link rel="top" title="Airflow Documentation" href="../index.html"/>
         <link rel="up" title="Module code" href="index.html"/> 
 
@@ -40,6 +45,7 @@
 
 <body class="wy-body-for-nav" role="document">
 
+   
   <div class="wy-grid-for-nav">
 
     
@@ -76,7 +82,10 @@
           
             
             
-                <ul>
+              
+            
+            
+              <ul>
 <li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li>
@@ -90,6 +99,8 @@
 <li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling &amp; Triggers</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li>
 </ul>
@@ -104,8 +115,10 @@
 
       
       <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
-        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
-        <a href="../index.html">Airflow</a>
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../index.html">Airflow</a>
+        
       </nav>
 
 
@@ -118,19 +131,36 @@
 
 
 
+
+
+
+
+
+
+
+
+
+
 <div role="navigation" aria-label="breadcrumbs navigation">
+
   <ul class="wy-breadcrumbs">
-    <li><a href="../index.html">Docs</a> &raquo;</li>
-      
+    
+      <li><a href="../index.html">Docs</a> &raquo;</li>
+        
           <li><a href="index.html">Module code</a> &raquo;</li>
-      
-    <li>S3_hook</li>
+        
+      <li>S3_hook</li>
+    
+    
       <li class="wy-breadcrumbs-aside">
         
-          
+            
         
       </li>
+    
   </ul>
+
+  
   <hr/>
 </div>
           <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -151,135 +181,21 @@
 <span class="c1"># See the License for the specific language governing permissions and</span>
 <span class="c1"># limitations under the License.</span>
 
-<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">division</span>
-<span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
-<span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
-<span class="kn">import</span> <span class="nn">logging</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
+
+<span class="kn">from</span> <span class="nn">six</span> <span class="k">import</span> <span class="n">BytesIO</span>
+<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="k">import</span> <span class="n">urlparse</span>
 <span class="kn">import</span> <span class="nn">re</span>
 <span class="kn">import</span> <span class="nn">fnmatch</span>
-<span class="kn">import</span> <span class="nn">configparser</span>
-<span class="kn">import</span> <span class="nn">math</span>
-<span class="kn">import</span> <span class="nn">os</span>
-<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlparse</span>
-<span class="kn">import</span> <span class="nn">warnings</span>
-
-<span class="kn">import</span> <span class="nn">boto</span>
-<span class="kn">from</span> <span class="nn">boto.s3.connection</span> <span class="kn">import</span> <span class="n">S3Connection</span>
-<span class="kn">from</span> <span class="nn">boto.sts</span> <span class="kn">import</span> <span class="n">STSConnection</span>
-<span class="n">boto</span><span class="o">.</span><span class="n">set_stream_logger</span><span class="p">(</span><span class="s1">&#39;boto&#39;</span><span class="p">)</span>
-<span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;boto&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">)</span>
 
-<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
-
-
-<span class="k">def</span> <span class="nf">_parse_s3_config</span><span class="p">(</span><span class="n">config_file_name</span><span class="p">,</span> <span class="n">config_format</span><span class="o">=</span><span class="s1">&#39;boto&#39;</span><span class="p">,</span> <span class="n">profile</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+<div class="viewcode-block" id="S3Hook"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook">[docs]</a><span class="k">class</span> <span class="nc">S3Hook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Parses a config file for s3 credentials. Can currently</span>
-<span class="sd">    parse boto, s3cmd.conf and AWS SDK config formats</span>
-
-<span class="sd">    :param config_file_name: path to the config file</span>
-<span class="sd">    :type config_file_name: str</span>
-<span class="sd">    :param config_format: config type. One of &quot;boto&quot;, &quot;s3cmd&quot; or &quot;aws&quot;.</span>
-<span class="sd">        Defaults to &quot;boto&quot;</span>
-<span class="sd">    :type config_format: str</span>
-<span class="sd">    :param profile: profile name in AWS type config file</span>
-<span class="sd">    :type profile: str</span>
+<span class="sd">    Interact with AWS S3, using the boto3 library.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">Config</span> <span class="o">=</span> <span class="n">configparser</span><span class="o">.</span><span class="n">ConfigParser</span><span class="p">()</span>
-    <span class="k">if</span> <span class="n">Config</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="n">config_file_name</span><span class="p">):</span>  <span class="c1"># pragma: no cover</span>
-        <span class="n">sections</span> <span class="o">=</span> <span class="n">Config</span><span class="o">.</span><span class="n">sections</span><span class="p">()</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;Couldn&#39;t read {0}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">config_file_name</span><span class="p">))</span>
-    <span class="c1"># Setting option names depending on file format</span>
-    <span class="k">if</span> <span class="n">config_format</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
-        <span class="n">config_format</span> <span class="o">=</span> <span class="s1">&#39;boto&#39;</span>
-    <span class="n">conf_format</span> <span class="o">=</span> <span class="n">config_format</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
-    <span class="k">if</span> <span class="n">conf_format</span> <span class="o">==</span> <span class="s1">&#39;boto&#39;</span><span class="p">:</span>  <span class="c1"># pragma: no cover</span>
-        <span class="k">if</span> <span class="n">profile</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span> <span class="ow">and</span> <span class="s1">&#39;profile &#39;</span> <span class="o">+</span> <span class="n">profile</span> <span class="ow">in</span> <span class="n">sections</span><span class="p">:</span>
-            <span class="n">cred_section</span> <span class="o">=</span> <span class="s1">&#39;profile &#39;</span> <span class="o">+</span> <span class="n">profile</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="n">cred_section</span> <span class="o">=</span> <span class="s1">&#39;Credentials&#39;</span>
-    <span class="k">elif</span> <span class="n">conf_format</span> <span class="o">==</span> <span class="s1">&#39;aws&#39;</span> <span class="ow">and</span> <span class="n">profile</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
-        <span class="n">cred_section</span> <span class="o">=</span> <span class="n">profile</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="n">cred_section</span> <span class="o">=</span> <span class="s1">&#39;default&#39;</span>
-    <span class="c1"># Option names</span>
-    <span class="k">if</span> <span class="n">conf_format</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">&#39;boto&#39;</span><span class="p">,</span> <span class="s1">&#39;aws&#39;</span><span class="p">):</span>  <span class="c1"># pragma: no cover</span>
-        <span class="n">key_id_option</span> <span class="o">=</span> <span class="s1">&#39;aws_access_key_id&#39;</span>
-        <span class="n">secret_key_option</span> <span class="o">=</span> <span class="s1">&#39;aws_secret_access_key&#39;</span>
-        <span class="c1"># security_token_option = &#39;aws_security_token&#39;</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="n">key_id_option</span> <span class="o">=</span> <span class="s1">&#39;access_key&#39;</span>
-        <span class="n">secret_key_option</span> <span class="o">=</span> <span class="s1">&#39;secret_key&#39;</span>
-    <span class="c1"># Actual Parsing</span>
-    <span class="k">if</span> <span class="n">cred_section</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">sections</span><span class="p">:</span>
-        <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;This config file format is not recognized&quot;</span><span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="k">try</span><span class="p">:</span>
-            <span class="n">access_key</span> <span class="o">=</span> <span class="n">Config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cred_section</span><span class="p">,</span> <span class="n">key_id_option</span><span class="p">)</span>
-            <span class="n">secret_key</span> <span class="o">=</span> <span class="n">Config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cred_section</span><span class="p">,</span> <span class="n">secret_key_option</span><span class="p">)</span>
-            <span class="n">calling_format</span> <span class="o">=</span> <span class="bp">None</span>
-            <span class="k">if</span> <span class="n">Config</span><span class="o">.</span><span class="n">has_option</span><span class="p">(</span><span class="n">cred_section</span><span class="p">,</span> <span class="s1">&#39;calling_format&#39;</span><span class="p">):</span>
-                <span class="n">calling_format</span> <span class="o">=</span> <span class="n">Config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cred_section</span><span class="p">,</span> <span class="s1">&#39;calling_format&#39;</span><span class="p">)</span>
-        <span class="k">except</span><span class="p">:</span>
-            <span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;Option Error in parsing s3 config file&quot;</span><span class="p">)</span>
-            <span class="k">raise</span>
-        <span class="k">return</span> <span class="p">(</span><span class="n">access_key</span><span class="p">,</span> <span class="n">secret_key</span><span class="p">,</span> <span class="n">calling_format</span><span class="p">)</span>
 
-
-<div class="viewcode-block" id="S3Hook"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook">[docs]</a><span class="k">class</span> <span class="nc">S3Hook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
-    <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Interact with S3. This class is a wrapper around the boto library.</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
-            <span class="bp">self</span><span class="p">,</span>
-            <span class="n">s3_conn_id</span><span class="o">=</span><span class="s1">&#39;s3_default&#39;</span><span class="p">):</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span> <span class="o">=</span> <span class="n">s3_conn_id</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">s3_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">extra_params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">s3_conn</span><span class="o">.</span><span class="n">extra_dejson</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">profile</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">extra_params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;profile&#39;</span><span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">calling_format</span> <span class="o">=</span> <span class="bp">None</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">_creds_in_conn</span> <span class="o">=</span> <span class="s1">&#39;aws_secret_access_key&#39;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">extra_params</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">_creds_in_config_file</span> <span class="o">=</span> <span class="s1">&#39;s3_config_file&#39;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">extra_params</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">_default_to_boto</span> <span class="o">=</span> <span class="bp">False</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_creds_in_conn</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">_a_key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">extra_params</span><span class="p">[</span><span class="s1">&#39;aws_access_key_id&#39;</span><span class="p">]</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">_s_key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">extra_params</span><span class="p">[</span><span class="s1">&#39;aws_secret_access_key&#39;</span><span class="p">]</span>
-            <span class="k">if</span> <span class="s1">&#39;calling_format&#39;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">extra_params</span><span class="p">:</span>
-                <span class="bp">self</span><span class="o">.</span><span class="n">calling_format</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">extra_params</span><span class="p">[</span><span class="s1">&#39;calling_format&#39;</span><span class="p">]</span>
-        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_creds_in_config_file</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">s3_config_file</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">extra_params</span><span class="p">[</span><span class="s1">&#39;s3_config_file&#39;</span><span class="p">]</span>
-            <span class="c1"># The format can be None and will default to boto in the parser</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">s3_config_format</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">extra_params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;s3_config_format&#39;</span><span class="p">)</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">_default_to_boto</span> <span class="o">=</span> <span class="bp">True</span>
-        <span class="c1"># STS support for cross account resource access</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">_sts_conn_required</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;aws_account_id&#39;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">extra_params</span> <span class="ow">or</span>
-                                   <span class="s1">&#39;role_arn&#39;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">extra_params</span><span class="p">)</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sts_conn_required</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">role_arn</span> <span class="o">=</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">extra_params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;role_arn&#39;</span><span class="p">)</span> <span class="ow">or</span>
-                             <span class="s2">&quot;arn:aws:iam::&quot;</span> <span class="o">+</span>
-                             <span class="bp">self</span><span class="o">.</span><span class="n">extra_params</span><span class="p">[</span><span class="s1">&#39;aws_account_id&#39;</span><span class="p">]</span> <span class="o">+</span>
-                             <span class="s2">&quot;:role/&quot;</span> <span class="o">+</span>
-                             <span class="bp">self</span><span class="o">.</span><span class="n">extra_params</span><span class="p">[</span><span class="s1">&#39;aws_iam_role&#39;</span><span class="p">])</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
-
-    <span class="k">def</span> <span class="nf">__getstate__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-        <span class="n">pickled_dict</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__dict__</span><span class="p">)</span>
-        <span class="k">del</span> <span class="n">pickled_dict</span><span class="p">[</span><span class="s1">&#39;connection&#39;</span><span class="p">]</span>
-        <span class="k">return</span> <span class="n">pickled_dict</span>
-
-    <span class="k">def</span> <span class="nf">__setstate__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">d</span><span class="p">):</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">d</span><span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">__dict__</span><span class="p">[</span><span class="s1">&#39;connection&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
-
-    <span class="k">def</span> <span class="nf">_parse_s3_url</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">s3url</span><span class="p">):</span>
-        <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
-            <span class="s1">&#39;Please note: S3Hook._parse_s3_url() is now &#39;</span>
-            <span class="s1">&#39;S3Hook.parse_s3_url() (no leading underscore).&#39;</span><span class="p">,</span>
-            <span class="ne">DeprecationWarning</span><span class="p">)</span>
-        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">s3url</span><span class="p">)</span>
+    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">&#39;s3&#39;</span><span class="p">)</span>
 
     <span class="nd">@staticmethod</span>
     <span class="k">def</span> <span class="nf">parse_s3_url</span><span class="p">(</span><span class="n">s3url</span><span class="p">):</span>
@@ -291,47 +207,6 @@
             <span class="n">key</span> <span class="o">=</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span>
             <span class="k">return</span> <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span>
 
-<div class="viewcode-block" id="S3Hook.get_conn"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-        <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Returns the boto S3Connection object.</span>
-<span class="sd">        &quot;&quot;&quot;</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_default_to_boto</span><span class="p">:</span>
-            <span class="k">return</span> <span class="n">S3Connection</span><span class="p">(</span><span class="n">profile_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">profile</span><span class="p">)</span>
-        <span class="n">a_key</span> <span class="o">=</span> <span class="n">s_key</span> <span class="o">=</span> <span class="bp">None</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_creds_in_config_file</span><span class="p">:</span>
-            <span class="n">a_key</span><span class="p">,</span> <span class="n">s_key</span><span class="p">,</span> <span class="n">calling_format</span> <span class="o">=</span> <span class="n">_parse_s3_config</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_config_file</span><span class="p">,</span>
-                                                <span class="bp">self</span><span class="o">.</span><span class="n">s3_config_format</span><span class="p">,</span>
-                                                <span class="bp">self</span><span class="o">.</span><span class="n">profile</span><span class="p">)</span>
-        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_creds_in_conn</span><span class="p">:</span>
-            <span class="n">a_key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_a_key</span>
-            <span class="n">s_key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_s_key</span>
-            <span class="n">calling_format</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">calling_format</span>
-
-        <span class="k">if</span> <span class="n">calling_format</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
-            <span class="n">calling_format</span> <span class="o">=</span> <span class="s1">&#39;boto.s3.connection.SubdomainCallingFormat&#39;</span>
-
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sts_conn_required</span><span class="p">:</span>
-            <span class="n">sts_connection</span> <span class="o">=</span> <span class="n">STSConnection</span><span class="p">(</span><span class="n">aws_access_key_id</span><span class="o">=</span><span class="n">a_key</span><span class="p">,</span>
-                                           <span class="n">aws_secret_access_key</span><span class="o">=</span><span class="n">s_key</span><span class="p">,</span>
-                                           <span class="n">profile_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">profile</span><span class="p">)</span>
-            <span class="n">assumed_role_object</span> <span class="o">=</span> <span class="n">sts_connection</span><span class="o">.</span><span class="n">assume_role</span><span class="p">(</span>
-                <span class="n">role_arn</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">role_arn</span><span class="p">,</span>
-                <span class="n">role_session_name</span><span class="o">=</span><span class="s2">&quot;Airflow_&quot;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span>
-                <span class="p">)</span>
-            <span class="n">creds</span> <span class="o">=</span> <span class="n">assumed_role_object</span><span class="o">.</span><span class="n">credentials</span>
-            <span class="n">connection</span> <span class="o">=</span> <span class="n">S3Connection</span><span class="p">(</span>
-                <span class="n">aws_access_key_id</span><span class="o">=</span><span class="n">creds</span><span class="o">.</span><span class="n">access_key</span><span class="p">,</span>
-                <span class="n">aws_secret_access_key</span><span class="o">=</span><span class="n">creds</span><span class="o">.</span><span class="n">secret_key</span><span class="p">,</span>
-                <span class="n">calling_format</span><span class="o">=</span><span class="n">calling_format</span><span class="p">,</span>
-                <span class="n">security_token</span><span class="o">=</span><span class="n">creds</span><span class="o">.</span><span class="n">session_token</span>
-                <span class="p">)</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="n">connection</span> <span class="o">=</span> <span class="n">S3Connection</span><span class="p">(</span><span class="n">aws_access_key_id</span><span class="o">=</span><span class="n">a_key</span><span class="p">,</span>
-                                      <span class="n">aws_secret_access_key</span><span class="o">=</span><span class="n">s_key</span><span class="p">,</span>
-                                      <span class="n">calling_format</span><span class="o">=</span><span class="n">calling_format</span><span class="p">,</span>
-                                      <span class="n">profile_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">profile</span><span class="p">)</span>
-        <span class="k">return</span> <span class="n">connection</span></div>
-
 <div class="viewcode-block" id="S3Hook.check_for_bucket"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.check_for_bucket">[docs]</a>    <span class="k">def</span> <span class="nf">check_for_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Check if bucket_name exists.</span>
@@ -339,20 +214,35 @@
 <span class="sd">        :param bucket_name: the name of the bucket</span>
 <span class="sd">        :type bucket_name: str</span>
 <span class="sd">        &quot;&quot;&quot;</span>
-        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">lookup</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span></div>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">head_bucket</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">)</span>
+            <span class="k">return</span> <span class="kc">True</span>
+        <span class="k">except</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">False</span></div>
 
 <div class="viewcode-block" id="S3Hook.get_bucket"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.get_bucket">[docs]</a>    <span class="k">def</span> <span class="nf">get_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Returns a boto.s3.bucket.Bucket object</span>
+<span class="sd">        Returns a boto3.S3.Bucket object</span>
 
 <span class="sd">        :param bucket_name: the name of the bucket</span>
 <span class="sd">        :type bucket_name: str</span>
 <span class="sd">        &quot;&quot;&quot;</span>
-        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">get_bucket</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">)</span></div>
+        <span class="n">s3</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_resource_type</span><span class="p">(</span><span class="s1">&#39;s3&#39;</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">s3</span><span class="o">.</span><span class="n">Bucket</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">)</span></div>
 
-<div class="viewcode-block" id="S3Hook.list_keys"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.list_keys">[docs]</a>    <span class="k">def</span> <span class="nf">list_keys</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="n">delimiter</span><span class= [...]
+<div class="viewcode-block" id="S3Hook.check_for_prefix"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.check_for_prefix">[docs]</a>    <span class="k">def</span> <span class="nf">check_for_prefix</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Lists keys in a bucket under prefix and not containing delimiter</span>
+<span class="sd">        Checks that a prefix exists in a bucket</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">prefix</span> <span class="o">=</span> <span class="n">prefix</span> <span class="o">+</span> <span class="n">delimiter</span> <span class="k">if</span> <span class="n">prefix</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">!=</span> <span class="n">delimiter</span> <span class="k">else</span> <span class="n">prefix</span>
+        <span class="n">prefix_split</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">r</span><span class="s1">&#39;(\w+[</span><span class="si">{d}</span><span class="s1">])$&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">d</span><span class="o">=</span><span class="n">delimiter</span><span class="p">),</span> <span class="n">prefi [...]
+        <span class="n">previous_level</span> <span class="o">=</span> <span class="n">prefix_split</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+        <span class="n">plist</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">list_prefixes</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">previous_level</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">)</span>
+        <span class="k">return</span> <span class="kc">False</span> <span class="k">if</span> <span class="n">plist</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">prefix</span> <span class="ow">in</span> <span class="n">plist</span></div>
+
+<div class="viewcode-block" id="S3Hook.list_prefixes"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.list_prefixes">[docs]</a>    <span class="k">def</span> <span class="nf">list_prefixes</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="n">delimiter</span> [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Lists prefixes in a bucket under prefix</span>
 
 <span class="sd">        :param bucket_name: the name of the bucket</span>
 <span class="sd">        :type bucket_name: str</span>
@@ -361,13 +251,14 @@
 <span class="sd">        :param delimiter: the delimiter marks key hierarchy.</span>
 <span class="sd">        :type delimiter: str</span>
 <span class="sd">        &quot;&quot;&quot;</span>
-        <span class="n">b</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_bucket</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">)</span>
-        <span class="n">keylist</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">b</span><span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="n">prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">))</span>
-        <span class="k">return</span> <span class="p">[</span><span class="n">k</span><span class="o">.</span><span class="n">name</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">keylist</span><span class="p">]</span> <span class="k">if</span> <span class="n">keylist</span> <span class="o">!=</span> <span class="p">[]</span> <span class="k">else</span> <span class="bp">None</span></div>
+        <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">list_objects_v2</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span> 
+                                                   <span class="n">Prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span> 
+                                                   <span class="n">Delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span>
+        <span class="k">return</span> <span class="p">[</span><span class="n">p</span><span class="p">[</span><span class="s1">&#39;Prefix&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;CommonPrefixes&#39;</span><span class="p">]]</span> <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">get</span><s [...]
 
-<div class="viewcode-block" id="S3Hook.list_prefixes"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.list_prefixes">[docs]</a>    <span class="k">def</span> <span class="nf">list_prefixes</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="n">delimiter</span> [...]
+<div class="viewcode-block" id="S3Hook.list_keys"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.list_keys">[docs]</a>    <span class="k">def</span> <span class="nf">list_keys</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="n">delimiter</span><span class= [...]
         <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Lists prefixes in a bucket under prefix</span>
+<span class="sd">        Lists keys in a bucket under prefix and not containing delimiter</span>
 
 <span class="sd">        :param bucket_name: the name of the bucket</span>
 <span class="sd">        :type bucket_name: str</span>
@@ -376,24 +267,32 @@
 <span class="sd">        :param delimiter: the delimiter marks key hierarchy.</span>
 <span class="sd">        :type delimiter: str</span>
 <span class="sd">        &quot;&quot;&quot;</span>
-        <span class="n">b</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_bucket</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">)</span>
-        <span class="n">plist</span> <span class="o">=</span> <span class="n">b</span><span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="n">prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span>
-        <span class="n">prefix_names</span> <span class="o">=</span> <span class="p">[</span><span class="n">p</span><span class="o">.</span><span class="n">name</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">plist</span>
-                        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">p</span><span class="p">,</span> <span class="n">boto</span><span class="o">.</span><span class="n">s3</span><span class="o">.</span><span class="n">prefix</span><span class="o">.</span><span class="n">Prefix</span><span class="p">)]</span>
-        <span class="k">return</span> <span class="n">prefix_names</span> <span class="k">if</span> <span class="n">prefix_names</span> <span class="o">!=</span> <span class="p">[]</span> <span class="k">else</span> <span class="bp">None</span></div>
+        <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">list_objects_v2</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span>
+                                                   <span class="n">Prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span>
+                                                   <span class="n">Delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span>
+        <span class="k">return</span> <span class="p">[</span><span class="n">k</span><span class="p">[</span><span class="s1">&#39;Key&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;Contents&#39;</span><span class="p">]]</span> <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">get</span><span class [...]
 
-<div class="viewcode-block" id="S3Hook.check_for_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.check_for_key">[docs]</a>    <span class="k">def</span> <span class="nf">check_for_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+<div class="viewcode-block" id="S3Hook.check_for_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.check_for_key">[docs]</a>    <span class="k">def</span> <span class="nf">check_for_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Checks that a key exists in a bucket</span>
+<span class="sd">        Checks if a key exists in a bucket</span>
+
+<span class="sd">        :param key: S3 key that will point to the file</span>
+<span class="sd">        :type key: str</span>
+<span class="sd">        :param bucket_name: Name of the bucket in which the file is stored</span>
+<span class="sd">        :type bucket_name: str</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span>
             <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
-        <span class="n">bucket</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_bucket</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">)</span>
-        <span class="k">return</span> <span class="n">bucket</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="n">key</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span></div>
 
-<div class="viewcode-block" id="S3Hook.get_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.get_key">[docs]</a>    <span class="k">def</span> <span class="nf">get_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">head_object</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">Key</span><span class="o">=</span><span class="n">key</span><span class="p">)</span>
+            <span class="k">return</span> <span class="kc">True</span>
+        <span class="k">except</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">False</span></div>
+
+<div class="viewcode-block" id="S3Hook.get_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.get_key">[docs]</a>    <span class="k">def</span> <span class="nf">get_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Returns a boto.s3.key.Key object</span>
+<span class="sd">        Returns a boto3.s3.Object</span>
 
 <span class="sd">        :param key: the path to the key</span>
 <span class="sd">        :type key: str</span>
@@ -402,21 +301,36 @@
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span>
             <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
-        <span class="n">bucket</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_bucket</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">)</span>
-        <span class="k">return</span> <span class="n">bucket</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="n">key</span><span class="p">)</span></div>
+
+        <span class="n">obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_resource_type</span><span class="p">(</span><span class="s1">&#39;s3&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">Object</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span>
+        <span class="n">obj</span><span class="o">.</span><span class="n">load</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">obj</span></div>
+
+<div class="viewcode-block" id="S3Hook.read_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.read_key">[docs]</a>    <span class="k">def</span> <span class="nf">read_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Reads a key from S3</span>
+
+<span class="sd">        :param key: S3 key that will point to the file</span>
+<span class="sd">        :type key: str</span>
+<span class="sd">        :param bucket_name: Name of the bucket in which the file is stored</span>
+<span class="sd">        :type bucket_name: str</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="n">obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">obj</span><span class="o">.</span><span class="n">get</span><span class="p">()[</span><span class="s1">&#39;Body&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span></div>
 
 <div class="viewcode-block" id="S3Hook.check_for_wildcard_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.check_for_wildcard_key">[docs]</a>    <span class="k">def</span> <span class="nf">check_for_wildcard_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
-                               <span class="n">wildcard_key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">):</span>
+                               <span class="n">wildcard_key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Checks that a key matching a wildcard expression exists in a bucket</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_wildcard_key</span><span class="p">(</span><span class="n">wildcard_key</span><span class="o">=</span><span class="n">wildcard_key</span><span class="p">,</span>
                                      <span class="n">bucket_name</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span>
-                                     <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span></div>
+                                     <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span></div>
 
-<div class="viewcode-block" id="S3Hook.get_wildcard_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.get_wildcard_key">[docs]</a>    <span class="k">def</span> <span class="nf">get_wildcard_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">delimit [...]
+<div class="viewcode-block" id="S3Hook.get_wildcard_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.get_wildcard_key">[docs]</a>    <span class="k">def</span> <span class="nf">get_wildcard_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">delimit [...]
         <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Returns a boto.s3.key.Key object matching the regular expression</span>
+<span class="sd">        Returns a boto3.s3.Object object matching the regular expression</span>
 
 <span class="sd">        :param regex_key: the path to the key</span>
 <span class="sd">        :type regex_key: str</span>
@@ -425,31 +339,20 @@
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span>
             <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">wildcard_key</span><span class="p">)</span>
-        <span class="n">bucket</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_bucket</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">)</span>
-        <span class="n">prefix</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">r&#39;[*]&#39;</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
-        <span class="n">klist</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">list_keys</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span>
-        <span class="k">if</span> <span class="ow">not</span> <span class="n">klist</span><span class="p">:</span>
-            <span class="k">return</span> <span class="bp">None</span>
-        <span class="n">key_matches</span> <span class="o">=</span> <span class="p">[</span><span class="n">k</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">klist</span> <span class="k">if</span> <span class="n">fnmatch</span><span class="o">.</span><span class="n">fnmatch</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">)]</span>
-        <span class="k">return</span> <span class="n">bucket</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="n">key_matches</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="k">if</span> <span class="n">key_matches</span> <span class="k">else</span> <span class="bp">None</span></div>
 
-<div class="viewcode-block" id="S3Hook.check_for_prefix"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.check_for_prefix">[docs]</a>    <span class="k">def</span> <span class="nf">check_for_prefix</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">):</span>
-        <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Checks that a prefix exists in a bucket</span>
-<span class="sd">        &quot;&quot;&quot;</span>
-        <span class="n">prefix</span> <span class="o">=</span> <span class="n">prefix</span> <span class="o">+</span> <span class="n">delimiter</span> <span class="k">if</span> <span class="n">prefix</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">!=</span> <span class="n">delimiter</span> <span class="k">else</span> <span class="n">prefix</span>
-        <span class="n">prefix_split</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">r&#39;(\w+[{d}])$&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">d</span><span class="o">=</span><span class="n">delimiter</span><span class="p">),</span> <span class="n">prefix</span><span class="p">,</span> <span class="mi">1</span><span class="p [...]
-        <span class="n">previous_level</span> <span class="o">=</span> <span class="n">prefix_split</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
-        <span class="n">plist</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">list_prefixes</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">previous_level</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">)</span>
-        <span class="k">return</span> <span class="bp">False</span> <span class="k">if</span> <span class="n">plist</span> <span class="ow">is</span> <span class="bp">None</span> <span class="k">else</span> <span class="n">prefix</span> <span class="ow">in</span> <span class="n">plist</span></div>
-
-<div class="viewcode-block" id="S3Hook.load_file"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.load_file">[docs]</a>    <span class="k">def</span> <span class="nf">load_file</span><span class="p">(</span>
-            <span class="bp">self</span><span class="p">,</span>
-            <span class="n">filename</span><span class="p">,</span>
-            <span class="n">key</span><span class="p">,</span>
-            <span class="n">bucket_name</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
-            <span class="n">replace</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
-            <span class="n">multipart_bytes</span><span class="o">=</span><span class="mi">5</span> <span class="o">*</span> <span class="p">(</span><span class="mi">1024</span> <span class="o">**</span> <span class="mi">3</span><span class="p">)):</span>
+        <span class="n">prefix</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">r</span><span class="s1">&#39;[*]&#39;</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
+        <span class="n">klist</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">list_keys</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">klist</span><span class="p">:</span>
+            <span class="n">key_matches</span> <span class="o">=</span> <span class="p">[</span><span class="n">k</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">klist</span> <span class="k">if</span> <span class="n">fnmatch</span><span class="o">.</span><span class="n">fnmatch</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">)]</span>
+            <span class="k">if</span> <span class="n">key_matches</span><span class="p">:</span>
+                <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="n">key_matches</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">bucket_name</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="S3Hook.load_file"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.load_file">[docs]</a>    <span class="k">def</span> <span class="nf">load_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                  <span class="n">filename</span><span class="p">,</span>
+                  <span class="n">key</span><span class="p">,</span>
+                  <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                  <span class="n">replace</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                  <span class="n">encrypt</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Loads a local file to S3</span>
 
@@ -463,60 +366,35 @@
 <span class="sd">            if it already exists. If replace is False and the key exists, an</span>
 <span class="sd">            error will be raised.</span>
 <span class="sd">        :type replace: bool</span>
-<span class="sd">        :param multipart_bytes: If provided, the file is uploaded in parts of</span>
-<span class="sd">            this size (minimum 5242880). The default value is 5GB, since S3</span>
-<span class="sd">            cannot accept non-multipart uploads for files larger than 5GB. If</span>
-<span class="sd">            the file is smaller than the specified limit, the option will be</span>
-<span class="sd">            ignored.</span>
-<span class="sd">        :type multipart_bytes: int</span>
+<span class="sd">        :param encrypt: If True, the file will be encrypted on the server-side</span>
+<span class="sd">            by S3 and will be stored in an encrypted form while at rest in S3.</span>
+<span class="sd">        :type encrypt: bool</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span>
             <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
-        <span class="n">bucket</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_bucket</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">)</span>
-        <span class="n">key_obj</span> <span class="o">=</span> <span class="n">bucket</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
-        <span class="k">if</span> <span class="ow">not</span> <span class="n">replace</span> <span class="ow">and</span> <span class="n">key_obj</span><span class="p">:</span>
-            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;The key {key} already exists.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
-                <span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
-
-        <span class="n">key_size</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">getsize</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span>
-        <span class="k">if</span> <span class="n">multipart_bytes</span> <span class="ow">and</span> <span class="n">key_size</span> <span class="o">&gt;=</span> <span class="n">multipart_bytes</span><span class="p">:</span>
-            <span class="c1"># multipart upload</span>
-            <span class="kn">from</span> <span class="nn">filechunkio</span> <span class="kn">import</span> <span class="n">FileChunkIO</span>
-            <span class="n">mp</span> <span class="o">=</span> <span class="n">bucket</span><span class="o">.</span><span class="n">initiate_multipart_upload</span><span class="p">(</span><span class="n">key_name</span><span class="o">=</span><span class="n">key</span><span class="p">)</span>
-            <span class="n">total_chunks</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">math</span><span class="o">.</span><span class="n">ceil</span><span class="p">(</span><span class="n">key_size</span> <span class="o">/</span> <span class="n">multipart_bytes</span><span class="p">))</span>
-            <span class="n">sent_bytes</span> <span class="o">=</span> <span class="mi">0</span>
-            <span class="k">try</span><span class="p">:</span>
-                <span class="k">for</span> <span class="n">chunk</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">total_chunks</span><span class="p">):</span>
-                    <span class="n">offset</span> <span class="o">=</span> <span class="n">chunk</span> <span class="o">*</span> <span class="n">multipart_bytes</span>
-                    <span class="nb">bytes</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">multipart_bytes</span><span class="p">,</span> <span class="n">key_size</span> <span class="o">-</span> <span class="n">offset</span><span class="p">)</span>
-                    <span class="k">with</span> <span class="n">FileChunkIO</span><span class="p">(</span>
-                            <span class="n">filename</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">,</span> <span class="n">offset</span><span class="o">=</span><span class="n">offset</span><span class="p">,</span> <span class="nb">bytes</span><span class="o">=</span><span class="nb">bytes</span><span class="p">)</span> <span class="k">as</span> <span class="n">fp</span><span class="p">:</span>
-                        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Sending chunk {c} of {tc}...&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
-                            <span class="n">c</span><span class="o">=</span><span class="n">chunk</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">tc</span><span class="o">=</span><span class="n">total_chunks</span><span class="p">))</span>
-                        <span class="n">mp</span><span class="o">.</span><span class="n">upload_part_from_file</span><span class="p">(</span><span class="n">fp</span><span class="p">,</span> <span class="n">part_num</span><span class="o">=</span><span class="n">chunk</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
-            <span class="k">except</span><span class="p">:</span>
-                <span class="n">mp</span><span class="o">.</span><span class="n">cancel_upload</span><span class="p">()</span>
-                <span class="k">raise</span>
-            <span class="n">mp</span><span class="o">.</span><span class="n">complete_upload</span><span class="p">()</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="c1"># regular upload</span>
-            <span class="k">if</span> <span class="ow">not</span> <span class="n">key_obj</span><span class="p">:</span>
-                <span class="n">key_obj</span> <span class="o">=</span> <span class="n">bucket</span><span class="o">.</span><span class="n">new_key</span><span class="p">(</span><span class="n">key_name</span><span class="o">=</span><span class="n">key</span><span class="p">)</span>
-            <span class="n">key_size</span> <span class="o">=</span> <span class="n">key_obj</span><span class="o">.</span><span class="n">set_contents_from_filename</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span>
-                                                      <span class="n">replace</span><span class="o">=</span><span class="n">replace</span><span class="p">)</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;The key {key} now contains&quot;</span>
-                     <span class="s2">&quot; {key_size} bytes&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span></div>
-
-<div class="viewcode-block" id="S3Hook.load_string"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.load_string">[docs]</a>    <span class="k">def</span> <span class="nf">load_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">string_data</span><span class="p">,</span>
-                    <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
-                    <span class="n">replace</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
-                    <span class="n">encrypt</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">replace</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">check_for_key</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;The key </span><span class="si">{key}</span><span class="s2"> already exists.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">))</span>
+
+        <span class="n">extra_args</span><span class="o">=</span><span class="p">{}</span>
+        <span class="k">if</span> <span class="n">encrypt</span><span class="p">:</span>
+            <span class="n">extra_args</span><span class="p">[</span><span class="s1">&#39;ServerSideEncryption&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;AES256&quot;</span>
+
+        <span class="n">client</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="n">client</span><span class="o">.</span><span class="n">upload_file</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">ExtraArgs</span><span class="o">=</span><span class="n">extra_args</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="S3Hook.load_string"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.load_string">[docs]</a>    <span class="k">def</span> <span class="nf">load_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> 
+                    <span class="n">string_data</span><span class="p">,</span>
+                    <span class="n">key</span><span class="p">,</span> 
+                    <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                    <span class="n">replace</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                    <span class="n">encrypt</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                    <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Loads a local file to S3</span>
+<span class="sd">        Loads a string to S3</span>
 
-<span class="sd">        This is provided as a convenience to drop a file in S3. It uses the</span>
-<span class="sd">        boto infrastructure to ship a file to s3. It is currently using only</span>
-<span class="sd">        a single part download, and should not be used to move large files.</span>
+<span class="sd">        This is provided as a convenience to drop a string in S3. It uses the</span>
+<span class="sd">        boto infrastructure to ship a file to s3. </span>
 
 <span class="sd">        :param string_data: string to set as content for the key.</span>
 <span class="sd">        :type string_data: str</span>
@@ -527,24 +405,30 @@
 <span class="sd">        :param replace: A flag to decide whether or not to overwrite the key</span>
 <span class="sd">            if it already exists</span>
 <span class="sd">        :type replace: bool</span>
+<span class="sd">        :param encrypt: If True, the file will be encrypted on the server-side</span>
+<span class="sd">            by S3 and will be stored in an encrypted form while at rest in S3.</span>
+<span class="sd">        :type encrypt: bool</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span>
             <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
-        <span class="n">bucket</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_bucket</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">)</span>
-        <span class="n">key_obj</span> <span class="o">=</span> <span class="n">bucket</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
-        <span class="k">if</span> <span class="ow">not</span> <span class="n">replace</span> <span class="ow">and</span> <span class="n">key_obj</span><span class="p">:</span>
-            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;The key {key} already exists.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
-                <span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
-        <span class="k">if</span> <span class="ow">not</span> <span class="n">key_obj</span><span class="p">:</span>
-            <span class="n">key_obj</span> <span class="o">=</span> <span class="n">bucket</span><span class="o">.</span><span class="n">new_key</span><span class="p">(</span><span class="n">key_name</span><span class="o">=</span><span class="n">key</span><span class="p">)</span>
-        <span class="n">key_size</span> <span class="o">=</span> <span class="n">key_obj</span><span class="o">.</span><span class="n">set_contents_from_string</span><span class="p">(</span><span class="n">string_data</span><span class="p">,</span>
-                                                    <span class="n">replace</span><span class="o">=</span><span class="n">replace</span><span class="p">,</span>
-                                                    <span class="n">encrypt_key</span><span class="o">=</span><span class="n">encrypt</span><span class="p">)</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;The key {key} now contains&quot;</span>
-                     <span class="s2">&quot; {key_size} bytes&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span></div></div>
+        
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">replace</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">check_for_key</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;The key </span><span class="si">{key}</span><span class="s2"> already exists.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">))</span>
+        
+        <span class="n">extra_args</span><span class="o">=</span><span class="p">{}</span>
+        <span class="k">if</span> <span class="n">encrypt</span><span class="p">:</span>
+            <span class="n">extra_args</span><span class="p">[</span><span class="s1">&#39;ServerSideEncryption&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;AES256&quot;</span>
+        
+        <span class="n">filelike_buffer</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="n">string_data</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">encoding</span><span class="p">))</span>
+        
+        <span class="n">client</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="n">client</span><span class="o">.</span><span class="n">upload_fileobj</span><span class="p">(</span><span class="n">filelike_buffer</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">ExtraArgs</span><span class="o">=</span><span class="n">extra_args</span><span class="p">)</span></div></div>
 </pre></div>
 
            </div>
+           <div class="articleComments">
+            
+           </div>
           </div>
           <footer>
   
@@ -577,7 +461,8 @@
             VERSION:'',
             COLLAPSE_INDEX:false,
             FILE_SUFFIX:'.html',
-            HAS_SOURCE:  true
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
         };
     </script>
       <script type="text/javascript" src="../_static/jquery.js"></script>
diff --git a/_modules/airflow/contrib/executors/mesos_executor.html b/_modules/airflow/contrib/executors/mesos_executor.html
index 311d1a8..f23df55 100644
--- a/_modules/airflow/contrib/executors/mesos_executor.html
+++ b/_modules/airflow/contrib/executors/mesos_executor.html
@@ -13,6 +13,8 @@
 
   
   
+  
+  
 
   
 
@@ -80,7 +82,10 @@
           
             
             
-                <ul>
+              
+            
+            
+              <ul>
 <li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
@@ -177,9 +182,12 @@
 <span class="c1"># limitations under the License.</span>
 
 <span class="kn">from</span> <span class="nn">future</span> <span class="k">import</span> <span class="n">standard_library</span>
+
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+<span class="kn">from</span> <span class="nn">airflow.www.utils</span> <span class="k">import</span> <span class="n">LoginMixin</span>
+
 <span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
-<span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">from</span> <span class="nn">queue</span> <span class="k">import</span> <span class="n">Queue</span>
 
 <span class="kn">import</span> <span class="nn">mesos.interface</span>
@@ -205,7 +213,7 @@
 
 <span class="c1"># AirflowMesosScheduler, implements Mesos Scheduler interface</span>
 <span class="c1"># To schedule airflow jobs on mesos</span>
-<span class="k">class</span> <span class="nc">AirflowMesosScheduler</span><span class="p">(</span><span class="n">mesos</span><span class="o">.</span><span class="n">interface</span><span class="o">.</span><span class="n">Scheduler</span><span class="p">):</span>
+<span class="k">class</span> <span class="nc">AirflowMesosScheduler</span><span class="p">(</span><span class="n">mesos</span><span class="o">.</span><span class="n">interface</span><span class="o">.</span><span class="n">Scheduler</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Airflow Mesos scheduler implements mesos scheduler interface</span>
 <span class="sd">    to schedule airflow tasks on mesos.</span>
@@ -213,7 +221,6 @@
 <span class="sd">    &#39;airflow run &lt;dag_id&gt; &lt;task_instance_id&gt; &lt;start_date&gt; --local -p=&lt;pickle&gt;&#39;</span>
 <span class="sd">    to run on a mesos slave.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                  <span class="n">task_queue</span><span class="p">,</span>
                  <span class="n">result_queue</span><span class="p">,</span>
@@ -227,7 +234,7 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span> <span class="o">=</span> <span class="p">{}</span>
 
     <span class="k">def</span> <span class="nf">registered</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">frameworkId</span><span class="p">,</span> <span class="n">masterInfo</span><span class="p">):</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler registered to mesos with framework ID </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler registered to Mesos with framework ID </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
 
         <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;CHECKPOINT&#39;</span><span class="p">)</span> <span class="ow">and</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span  [...]
             <span class="c1"># Import here to work around a circular import error</span>
@@ -248,25 +255,25 @@
             <span class="n">Session</span><span class="o">.</span><span class="n">remove</span><span class="p">()</span>
 
     <span class="k">def</span> <span class="nf">reregistered</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">masterInfo</span><span class="p">):</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler re-registered to mesos&quot;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler re-registered to mesos&quot;</span><span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">disconnected</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">):</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler disconnected from mesos&quot;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler disconnected from mesos&quot;</span><span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">offerRescinded</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">offerId</span><span class="p">):</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler offer </span><span class="si">%s</span><span class="s2"> rescinded&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">offerId</span><span class="p">))</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler offer </span><span class="si">%s</span><span class="s2"> rescinded&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">offerId</span><span class="p">))</span>
 
     <span class="k">def</span> <span class="nf">frameworkMessage</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">executorId</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">,</span> <span class="n">message</span><span class="p">):</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler received framework message </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler received framework message </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">executorLost</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">executorId</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">,</span> <span class="n">status</span><span class="p">):</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler executor </span><span class="si">%s</span><span class="s2"> lost&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">executorId</span><span class="p">))</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler executor </span><span class="si">%s</span><span class="s2"> lost&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">executorId</span><span class="p">))</span>
 
     <span class="k">def</span> <span class="nf">slaveLost</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">):</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler slave </span><span class="si">%s</span><span class="s2"> lost&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">slaveId</span><span class="p">))</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler slave </span><span class="si">%s</span><span class="s2"> lost&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">slaveId</span><span class="p">))</span>
 
     <span class="k">def</span> <span class="nf">error</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">message</span><span class="p">):</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler driver aborted </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler driver aborted </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
         <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler driver aborted </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">message</span><span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">resourceOffers</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">offers</span><span class="p">):</span>
@@ -280,7 +287,7 @@
                 <span class="k">elif</span> <span class="n">resource</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="s2">&quot;mem&quot;</span><span class="p">:</span>
                     <span class="n">offerMem</span> <span class="o">+=</span> <span class="n">resource</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span>
 
-            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Received offer </span><span class="si">%s</span><span class="s2"> with cpus: </span><span class="si">%s</span><span class="s2"> and mem: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</spa [...]
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Received offer </span><span class="si">%s</span><span class="s2"> with cpus: </span><span class="si">%s</span><span class="s2"> and mem: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">offer</span><span class="o">.</span><span class="n">id</span><s [...]
 
             <span class="n">remainingCpus</span> <span class="o">=</span> <span class="n">offerCpus</span>
             <span class="n">remainingMem</span> <span class="o">=</span> <span class="n">offerMem</span>
@@ -293,7 +300,7 @@
                 <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span> <span class="o">+=</span> <span class="mi">1</span>
                 <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span><span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">tid</span><span class="p">)]</span> <span class="o">=</span> <span class="n">key</span>
 
-                <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Launching task </span><span class="si">%d</span><span class="s2"> using offer </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">tid</span><span class="p">,</span> <span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span><s [...]
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Launching task </span><span class="si">%d</span><span class="s2"> using offer </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">tid</span><span class="p">,</span> <span class="n">offer</span><span class="o">.</span><span class="n">id</span><span  [...]
 
                 <span class="n">task</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TaskInfo</span><span class="p">()</span>
                 <span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">tid</span><span class="p">)</span>
@@ -323,15 +330,17 @@
             <span class="n">driver</span><span class="o">.</span><span class="n">launchTasks</span><span class="p">(</span><span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="p">,</span> <span class="n">tasks</span><span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">statusUpdate</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">update</span><span class="p">):</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Task </span><span class="si">%s</span><span class="s2"> is in state </span><span class="si">%s</span><span class="s2">, data </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span>
-                     <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">,</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TaskState</span><span class="o">.</span><span class="n">Name</span><span class="p">(</span><span class="n">update</span><span class="o">.</span><span class="n">state</span><span class="p">),</span> <span class="nb">str</span><span class="p [...]
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+            <span class="s2">&quot;Task </span><span class="si">%s</span><span class="s2"> is in state </span><span class="si">%s</span><span class="s2">, data </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span>
+            <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">,</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TaskState</span><span class="o">.</span><span class="n">Name</span><span class="p">(</span><span class="n">update</span><span class="o">.</span><span class="n">state</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span [...]
+        <span class="p">)</span>
 
         <span class="k">try</span><span class="p">:</span>
             <span class="n">key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span><span class="p">[</span><span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">]</span>
         <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
             <span class="c1"># The map may not contain an item if the framework re-registered after a failover.</span>
             <span class="c1"># Discard these tasks.</span>
-            <span class="n">logging</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;Unrecognised task key </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;Unrecognised task key </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
             <span class="k">return</span>
 
         <span class="k">if</span> <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_FINISHED</span><span class="p">:</span>
@@ -345,7 +354,7 @@
             <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span>
 
 
-<div class="viewcode-block" id="MesosExecutor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor">[docs]</a><span class="k">class</span> <span class="nc">MesosExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">):</span>
+<div class="viewcode-block" id="MesosExecutor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor">[docs]</a><span class="k">class</span> <span class="nc">MesosExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">,</span> <span class="n">LoginMixin</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    MesosExecutor allows distributing the execution of task</span>
 <span class="sd">    instances to multiple mesos workers.</span>
@@ -356,7 +365,6 @@
 <span class="sd">    elastic distributed systems to easily be built and run effectively.</span>
 <span class="sd">    See http://mesos.apache.org/</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-
     <span class="k">def</span> <span class="nf">start</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
@@ -364,7 +372,7 @@
         <span class="n">framework</span><span class="o">.</span><span class="n">user</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
 
         <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;MASTER&#39;</span><span class="p">):</span>
-            <span class="n">logging</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Expecting mesos master URL for mesos executor&quot;</span><span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Expecting mesos master URL for mesos executor&quot;</span><span class="p">)</span>
             <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;mesos.master not provided for mesos executor&quot;</span><span class="p">)</span>
 
         <span class="n">master</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;MASTER&#39;</span><span class="p">)</span>
@@ -400,17 +408,19 @@
         <span class="k">else</span><span class="p">:</span>
             <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">False</span>
 
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;MesosFramework master : </span><span class="si">%s</span><span class="s1">, name : </span><span class="si">%s</span><span class="s1">, cpu : </span><span class="si">%s</span><span class="s1">, mem : </span><span class="si">%s</span><span class="s1">, checkpoint : </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
-            <span class="n">master</span><span class="p">,</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_cpu</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_memory</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">framework</span><span class="o"> [...]
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+            <span class="s1">&#39;MesosFramework master : </span><span class="si">%s</span><span class="s1">, name : </span><span class="si">%s</span><span class="s1">, cpu : </span><span class="si">%s</span><span class="s1">, mem : </span><span class="si">%s</span><span class="s1">, checkpoint : </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+            <span class="n">master</span><span class="p">,</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_cpu</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_memory</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">framework</span><span class="o"> [...]
+        <span class="p">)</span>
 
         <span class="n">implicit_acknowledgements</span> <span class="o">=</span> <span class="mi">1</span>
 
         <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;AUTHENTICATE&#39;</span><span class="p">):</span>
             <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;DEFAULT_PRINCIPAL&#39;</span><span class="p">):</span>
-                <span class="n">logging</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Expecting authentication principal in the environment&quot;</span><span class="p">)</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Expecting authentication principal in the environment&quot;</span><span class="p">)</span>
                 <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;mesos.default_principal not provided in authenticated mode&quot;</span><span class="p">)</span>
             <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;DEFAULT_SECRET&#39;</span><span class="p">):</span>
-                <span class="n">logging</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Expecting authentication secret in the environment&quot;</span><span class="p">)</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Expecting authentication secret in the environment&quot;</span><span class="p">)</span>
                 <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;mesos.default_secret not provided in authenticated mode&quot;</span><span class="p">)</span>
 
             <span class="n">credential</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Credential</span><span class="p">()</span>
diff --git a/_modules/bigquery_hook.html b/_modules/airflow/contrib/hooks/bigquery_hook.html
similarity index 60%
copy from _modules/bigquery_hook.html
copy to _modules/airflow/contrib/hooks/bigquery_hook.html
index 1f8170a..1926c79 100644
--- a/_modules/bigquery_hook.html
+++ b/_modules/airflow/contrib/hooks/bigquery_hook.html
@@ -8,11 +8,13 @@
   
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   
-  <title>bigquery_hook &mdash; Airflow Documentation</title>
+  <title>airflow.contrib.hooks.bigquery_hook &mdash; Airflow Documentation</title>
   
 
   
   
+  
+  
 
   
 
@@ -24,22 +26,26 @@
 
   
   
-    <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
   
 
   
 
   
-    <link rel="top" title="Airflow Documentation" href="../index.html"/>
-        <link rel="up" title="Module code" href="index.html"/> 
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
 
   
-  <script src="../_static/js/modernizr.min.js"></script>
+  <script src="../../../../_static/js/modernizr.min.js"></script>
 
 </head>
 
 <body class="wy-body-for-nav" role="document">
 
+   
   <div class="wy-grid-for-nav">
 
     
@@ -49,7 +55,7 @@
           
 
           
-            <a href="../index.html" class="icon icon-home"> Airflow
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
           
 
           
@@ -62,7 +68,7 @@
 
           
 <div role="search">
-  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
     <input type="text" name="q" placeholder="Search docs" />
     <input type="hidden" name="check_keywords" value="yes" />
     <input type="hidden" name="area" value="default" />
@@ -76,22 +82,27 @@
           
             
             
-                <ul>
-<li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../configuration.html">Configuration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../ui.html">UI / Screenshots</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../concepts.html">Concepts</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../profiling.html">Data Profiling</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../cli.html">Command Line Interface</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling &amp; Triggers</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li>
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
 </ul>
 
             
@@ -104,8 +115,10 @@
 
       
       <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
-        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
-        <a href="../index.html">Airflow</a>
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
       </nav>
 
 
@@ -118,25 +131,42 @@
 
 
 
+
+
+
+
+
+
+
+
+
+
 <div role="navigation" aria-label="breadcrumbs navigation">
+
   <ul class="wy-breadcrumbs">
-    <li><a href="../index.html">Docs</a> &raquo;</li>
-      
-          <li><a href="index.html">Module code</a> &raquo;</li>
-      
-    <li>bigquery_hook</li>
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.bigquery_hook</li>
+    
+    
       <li class="wy-breadcrumbs-aside">
         
-          
+            
         
       </li>
+    
   </ul>
+
+  
   <hr/>
 </div>
           <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
            <div itemprop="articleBody">
             
-  <h1>Source code for bigquery_hook</h1><div class="highlight"><pre>
+  <h1>Source code for airflow.contrib.hooks.bigquery_hook</h1><div class="highlight"><pre>
 <span></span><span class="c1"># -*- coding: utf-8 -*-</span>
 <span class="c1">#</span>
 <span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
@@ -157,25 +187,24 @@
 <span class="sd">implementation for BigQuery.</span>
 <span class="sd">&quot;&quot;&quot;</span>
 
-<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">range</span>
-<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="nb">basestring</span>
-
-<span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">time</span>
 
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="kn">import</span> <span class="n">GoogleCloudBaseHook</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>
-<span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="kn">import</span> <span class="n">build</span>
-<span class="kn">from</span> <span class="nn">pandas.io.gbq</span> <span class="kn">import</span> <span class="n">GbqConnector</span><span class="p">,</span> \
+<span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="k">import</span> <span class="n">build</span><span class="p">,</span> <span class="n">HttpError</span>
+<span class="kn">from</span> <span class="nn">googleapiclient</span> <span class="k">import</span> <span class="n">errors</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">range</span>
+<span class="kn">from</span> <span class="nn">pandas_gbq.gbq</span> <span class="k">import</span> <span class="n">GbqConnector</span><span class="p">,</span> \
     <span class="n">_parse_data</span> <span class="k">as</span> <span class="n">gbq_parse_data</span><span class="p">,</span> \
     <span class="n">_check_google_client_version</span> <span class="k">as</span> <span class="n">gbq_check_google_client_version</span><span class="p">,</span> \
     <span class="n">_test_google_api_imports</span> <span class="k">as</span> <span class="n">gbq_test_google_api_imports</span>
-<span class="kn">from</span> <span class="nn">pandas.tools.merge</span> <span class="kn">import</span> <span class="n">concat</span>
+<span class="kn">from</span> <span class="nn">pandas.tools.merge</span> <span class="k">import</span> <span class="n">concat</span>
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="k">import</span> <span class="n">basestring</span>
 
-<span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;bigquery&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">)</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="k">import</span> <span class="n">DbApiHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
 
 
-<div class="viewcode-block" id="BigQueryHook"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.BigQueryHook">[docs]</a><span class="k">class</span> <span class="nc">BigQueryHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">,</span> <span class="n">DbApiHook</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook">[docs]</a><span class="k">class</span> <span class="nc">BigQueryHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">,</span> <span class="n">DbApiHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Interact with BigQuery. This hook uses the Google Cloud Platform</span>
 <span class="sd">    connection.</span>
@@ -184,12 +213,12 @@
 
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                  <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">&#39;bigquery_default&#39;</span><span class="p">,</span>
-                 <span class="n">delegate_to</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span>
+                 <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
             <span class="n">conn_id</span><span class="o">=</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
             <span class="n">delegate_to</span><span class="o">=</span><span class="n">delegate_to</span><span class="p">)</span>
 
-<div class="viewcode-block" id="BigQueryHook.get_conn"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.BigQueryHook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Returns a BigQuery PEP 249 connection object.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
@@ -197,14 +226,14 @@
         <span class="n">project</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">&#39;project&#39;</span><span class="p">)</span>
         <span class="k">return</span> <span class="n">BigQueryConnection</span><span class="p">(</span><span class="n">service</span><span class="o">=</span><span class="n">service</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="n">project</span><span class="p">)</span></div>
 
-<div class="viewcode-block" id="BigQueryHook.get_service"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.BigQueryHook.get_service">[docs]</a>    <span class="k">def</span> <span class="nf">get_service</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryHook.get_service"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.get_service">[docs]</a>    <span class="k">def</span> <span class="nf">get_service</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Returns a BigQuery service object.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
         <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">&#39;bigquery&#39;</span><span class="p">,</span> <span class="s1">&#39;v2&#39;</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">)</span></div>
 
-<div class="viewcode-block" id="BigQueryHook.insert_rows"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.BigQueryHook.insert_rows">[docs]</a>    <span class="k">def</span> <span class="nf">insert_rows</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">rows</span><span class="p">,</span> <span class="n">target_fields</span><span class="o">=</span><span class="bp">None</s [...]
+<div class="viewcode-block" id="BigQueryHook.insert_rows"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.insert_rows">[docs]</a>    <span class="k">def</span> <span class="nf">insert_rows</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">rows</span><span class="p">,</span> <span class="n">target_fields</span><span class="o">=< [...]
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Insertion is currently unsupported. Theoretically, you could use</span>
 <span class="sd">        BigQuery&#39;s streaming API to insert rows into a table, but this hasn&#39;t</span>
@@ -212,7 +241,7 @@
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span></div>
 
-<div class="viewcode-block" id="BigQueryHook.get_pandas_df"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.BigQueryHook.get_pandas_df">[docs]</a>    <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryHook.get_pandas_df"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.get_pandas_df">[docs]</a>    <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">, [...]
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Returns a Pandas DataFrame for the results produced by a BigQuery</span>
 <span class="sd">        query. The DbApiHook method must be overridden because Pandas</span>
@@ -223,10 +252,14 @@
 
 <span class="sd">        :param bql: The BigQuery SQL to execute.</span>
 <span class="sd">        :type bql: string</span>
+<span class="sd">        :param parameters: The parameters to render the SQL query with (not used, leave to override superclass method)</span>
+<span class="sd">        :type parameters: mapping or iterable</span>
+<span class="sd">        :param dialect: Dialect of BigQuery SQL – legacy SQL or standard SQL</span>
+<span class="sd">        :type dialect: string in {&#39;legacy&#39;, &#39;standard&#39;}, default &#39;legacy&#39;</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_service</span><span class="p">()</span>
         <span class="n">project</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">&#39;project&#39;</span><span class="p">)</span>
-        <span class="n">connector</span> <span class="o">=</span> <span class="n">BigQueryPandasConnector</span><span class="p">(</span><span class="n">project</span><span class="p">,</span> <span class="n">service</span><span class="p">)</span>
+        <span class="n">connector</span> <span class="o">=</span> <span class="n">BigQueryPandasConnector</span><span class="p">(</span><span class="n">project</span><span class="p">,</span> <span class="n">service</span><span class="p">,</span> <span class="n">dialect</span><span class="o">=</span><span class="n">dialect</span><span class="p">)</span>
         <span class="n">schema</span><span class="p">,</span> <span class="n">pages</span> <span class="o">=</span> <span class="n">connector</span><span class="o">.</span><span class="n">run_query</span><span class="p">(</span><span class="n">bql</span><span class="p">)</span>
         <span class="n">dataframe_list</span> <span class="o">=</span> <span class="p">[]</span>
 
@@ -235,9 +268,35 @@
             <span class="n">dataframe_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">gbq_parse_data</span><span class="p">(</span><span class="n">schema</span><span class="p">,</span> <span class="n">page</span><span class="p">))</span>
 
         <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">dataframe_list</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
-            <span class="k">return</span> <span class="n">concat</span><span class="p">(</span><span class="n">dataframe_list</span><span class="p">,</span> <span class="n">ignore_index</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">concat</span><span class="p">(</span><span class="n">dataframe_list</span><span class="p">,</span> <span class="n">ignore_index</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
         <span class="k">else</span><span class="p">:</span>
-            <span class="k">return</span> <span class="n">gbq_parse_data</span><span class="p">(</span><span class="n">schema</span><span class="p">,</span> <span class="p">[])</span></div></div>
+            <span class="k">return</span> <span class="n">gbq_parse_data</span><span class="p">(</span><span class="n">schema</span><span class="p">,</span> <span class="p">[])</span></div>
+
+<div class="viewcode-block" id="BigQueryHook.table_exists"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.table_exists">[docs]</a>    <span class="k">def</span> <span class="nf">table_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span cla [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Checks for the existence of a table in Google BigQuery.</span>
+
+<span class="sd">        :param project_id: The Google cloud project in which to look for the table. The connection supplied to the hook</span>
+<span class="sd">        must provide access to the specified project.</span>
+<span class="sd">        :type project_id: string</span>
+<span class="sd">        :param dataset_id: The name of the dataset in which to look for the table.</span>
+<span class="sd">            storage bucket.</span>
+<span class="sd">        :type dataset_id: string</span>
+<span class="sd">        :param table_id: The name of the table to check the existence of.</span>
+<span class="sd">        :type table_id: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_service</span><span class="p">()</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+                <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+                <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+                <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span>
+            <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="k">return</span> <span class="kc">True</span>
+        <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;404&#39;</span><span class="p">:</span>
+                <span class="k">return</span> <span class="kc">False</span>
+            <span class="k">raise</span></div></div>
 
 
 <span class="k">class</span> <span class="nc">BigQueryPandasConnector</span><span class="p">(</span><span class="n">GbqConnector</span><span class="p">):</span>
@@ -248,13 +307,14 @@
 <span class="sd">    without forcing a three legged OAuth connection. Instead, we can inject</span>
 <span class="sd">    service account credentials into the binding.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">service</span><span class="p">,</span> <span class="n">reauth</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">service</span><span class="p">,</span> <span class="n">reauth</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">dialect< [...]
         <span class="n">gbq_check_google_client_version</span><span class="p">()</span>
         <span class="n">gbq_test_google_api_imports</span><span class="p">()</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">reauth</span> <span class="o">=</span> <span class="n">reauth</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">service</span> <span class="o">=</span> <span class="n">service</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dialect</span> <span class="o">=</span> <span class="n">dialect</span>
 
 
 <span class="k">class</span> <span class="nc">BigQueryConnection</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
@@ -285,22 +345,25 @@
             <span class="s2">&quot;BigQueryConnection does not have transactions&quot;</span><span class="p">)</span>
 
 
-<span class="k">class</span> <span class="nc">BigQueryBaseCursor</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+<span class="k">class</span> <span class="nc">BigQueryBaseCursor</span><span class="p">(</span><span class="n">LoggingMixin</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    The BigQuery base cursor contains helper methods to execute queries against</span>
 <span class="sd">    BigQuery. The methods can be used directly by operators, in cases where a</span>
 <span class="sd">    PEP 249 cursor isn&#39;t needed.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">service</span><span class="p">,</span> <span class="n">project_id</span><span class="p">):</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">service</span> <span class="o">=</span> <span class="n">service</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
 
     <span class="k">def</span> <span class="nf">run_query</span><span class="p">(</span>
-            <span class="bp">self</span><span class="p">,</span> <span class="n">bql</span><span class="p">,</span> <span class="n">destination_dataset_table</span> <span class="o">=</span> <span class="bp">False</span><span class="p">,</span>
+            <span class="bp">self</span><span class="p">,</span> <span class="n">bql</span><span class="p">,</span> <span class="n">destination_dataset_table</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
             <span class="n">write_disposition</span> <span class="o">=</span> <span class="s1">&#39;WRITE_EMPTY&#39;</span><span class="p">,</span>
-            <span class="n">allow_large_results</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
-            <span class="n">udf_config</span> <span class="o">=</span> <span class="bp">False</span><span class="p">):</span>
+            <span class="n">allow_large_results</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+            <span class="n">udf_config</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+            <span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+            <span class="n">maximum_billing_tier</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">create_disposition</span><span class="o">=</span><span class="s1">&#39;CREATE_IF_NEEDED&#39;</span><span class="p">,</span>
+            <span class="n">query_params</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Executes a BigQuery SQL query. Optionally persists results in a BigQuery</span>
 <span class="sd">        table. See here:</span>
@@ -315,29 +378,40 @@
 <span class="sd">            BigQuery table to save the query results.</span>
 <span class="sd">        :param write_disposition: What to do if the table already exists in</span>
 <span class="sd">            BigQuery.</span>
+<span class="sd">        :type write_disposition: string</span>
+<span class="sd">        :param create_disposition: Specifies whether the job is allowed to create new tables.</span>
+<span class="sd">        :type create_disposition: string</span>
 <span class="sd">        :param allow_large_results: Whether to allow large results.</span>
 <span class="sd">        :type allow_large_results: boolean</span>
 <span class="sd">        :param udf_config: The User Defined Function configuration for the query.</span>
 <span class="sd">            See https://cloud.google.com/bigquery/user-defined-functions for details.</span>
 <span class="sd">        :type udf_config: list</span>
+<span class="sd">        :param use_legacy_sql: Whether to use legacy SQL (true) or standard SQL (false).</span>
+<span class="sd">        :type use_legacy_sql: boolean</span>
+<span class="sd">        :param maximum_billing_tier: Positive integer that serves as a multiplier of the basic price.</span>
+<span class="sd">        :type maximum_billing_tier: integer</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
             <span class="s1">&#39;query&#39;</span><span class="p">:</span> <span class="p">{</span>
                 <span class="s1">&#39;query&#39;</span><span class="p">:</span> <span class="n">bql</span><span class="p">,</span>
+                <span class="s1">&#39;useLegacySql&#39;</span><span class="p">:</span> <span class="n">use_legacy_sql</span><span class="p">,</span>
+                <span class="s1">&#39;maximumBillingTier&#39;</span><span class="p">:</span> <span class="n">maximum_billing_tier</span>
             <span class="p">}</span>
         <span class="p">}</span>
 
         <span class="k">if</span> <span class="n">destination_dataset_table</span><span class="p">:</span>
             <span class="k">assert</span> <span class="s1">&#39;.&#39;</span> <span class="ow">in</span> <span class="n">destination_dataset_table</span><span class="p">,</span> <span class="p">(</span>
                 <span class="s1">&#39;Expected destination_dataset_table in the format of &#39;</span>
-                <span class="s1">&#39;&lt;dataset&gt;.&lt;table&gt;. Got: {}&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">destination_dataset_table</span><span class="p">)</span>
-            <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
-                <span class="n">destination_dataset_table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+                <span class="s1">&#39;&lt;dataset&gt;.&lt;table&gt;. Got: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">destination_dataset_table</span><span class="p">)</span>
+            <span class="n">destination_project</span><span class="p">,</span> <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
+                <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">destination_dataset_table</span><span class="p">,</span>
+                                 <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
             <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
                 <span class="s1">&#39;allowLargeResults&#39;</span><span class="p">:</span> <span class="n">allow_large_results</span><span class="p">,</span>
                 <span class="s1">&#39;writeDisposition&#39;</span><span class="p">:</span> <span class="n">write_disposition</span><span class="p">,</span>
+                <span class="s1">&#39;createDisposition&#39;</span><span class="p">:</span> <span class="n">create_disposition</span><span class="p">,</span>
                 <span class="s1">&#39;destinationTable&#39;</span><span class="p">:</span> <span class="p">{</span>
-                    <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                    <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="n">destination_project</span><span class="p">,</span>
                     <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">destination_dataset</span><span class="p">,</span>
                     <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">destination_table</span><span class="p">,</span>
                 <span class="p">}</span>
@@ -348,12 +422,15 @@
                 <span class="s1">&#39;userDefinedFunctionResources&#39;</span><span class="p">:</span> <span class="n">udf_config</span>
             <span class="p">})</span>
 
+        <span class="k">if</span> <span class="n">query_params</span><span class="p">:</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">][</span><span class="s1">&#39;queryParameters&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_params</span>
+
         <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">run_extract</span><span class="p">(</span>  <span class="c1"># noqa</span>
             <span class="bp">self</span><span class="p">,</span> <span class="n">source_project_dataset_table</span><span class="p">,</span> <span class="n">destination_cloud_storage_uris</span><span class="p">,</span>
             <span class="n">compression</span><span class="o">=</span><span class="s1">&#39;NONE&#39;</span><span class="p">,</span> <span class="n">export_format</span><span class="o">=</span><span class="s1">&#39;CSV&#39;</span><span class="p">,</span> <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">&#39;,&#39;</span><span class="p">,</span>
-            <span class="n">print_header</span><span class="o">=</span><span class="bp">True</span><span class="p">):</span>
+            <span class="n">print_header</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Executes a BigQuery extract command to copy data from BigQuery to</span>
 <span class="sd">        Google Cloud Storage. See here:</span>
@@ -379,9 +456,12 @@
 <span class="sd">        :param print_header: Whether to print a header for a CSV file extract.</span>
 <span class="sd">        :type print_header: boolean</span>
 <span class="sd">        &quot;&quot;&quot;</span>
+
         <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span><span class="p">,</span> <span class="n">source_table</span> <span class="o">=</span> \
-            <span class="bp">self</span><span class="o">.</span><span class="n">_split_project_dataset_table_input</span><span class="p">(</span>
-                <span class="s1">&#39;source_project_dataset_table&#39;</span><span class="p">,</span> <span class="n">source_project_dataset_table</span><span class="p">)</span>
+            <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">source_project_dataset_table</span><span class="p">,</span>
+                             <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                             <span class="n">var_name</span><span class="o">=</span><span class="s1">&#39;source_project_dataset_table&#39;</span><span class="p">)</span>
+
         <span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
             <span class="s1">&#39;extract&#39;</span><span class="p">:</span> <span class="p">{</span>
                 <span class="s1">&#39;sourceTable&#39;</span><span class="p">:</span> <span class="p">{</span>
@@ -418,14 +498,14 @@
 <span class="sd">        For more details about these parameters.</span>
 
 <span class="sd">        :param source_project_dataset_tables: One or more dotted</span>
-<span class="sd">            (&lt;project&gt;.)&lt;dataset&gt;.&lt;table&gt;</span>
+<span class="sd">            (project:|project.)&lt;dataset&gt;.&lt;table&gt;</span>
 <span class="sd">            BigQuery tables to use as the source data. Use a list if there are</span>
 <span class="sd">            multiple source tables.</span>
 <span class="sd">            If &lt;project&gt; is not included, project will be the project defined</span>
 <span class="sd">            in the connection json.</span>
 <span class="sd">        :type source_project_dataset_tables: list|string</span>
 <span class="sd">        :param destination_project_dataset_table: The destination BigQuery</span>
-<span class="sd">            table. Format is: &lt;project&gt;.&lt;dataset&gt;.&lt;table&gt;</span>
+<span class="sd">            table. Format is: (project:|project.)&lt;dataset&gt;.&lt;table&gt;</span>
 <span class="sd">        :type destination_project_dataset_table: string</span>
 <span class="sd">        :param write_disposition: The write disposition if the table already exists.</span>
 <span class="sd">        :type write_disposition: string</span>
@@ -440,21 +520,18 @@
         <span class="n">source_project_dataset_tables_fixup</span> <span class="o">=</span> <span class="p">[]</span>
         <span class="k">for</span> <span class="n">source_project_dataset_table</span> <span class="ow">in</span> <span class="n">source_project_dataset_tables</span><span class="p">:</span>
             <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span><span class="p">,</span> <span class="n">source_table</span> <span class="o">=</span> \
-                <span class="bp">self</span><span class="o">.</span><span class="n">_split_project_dataset_table_input</span><span class="p">(</span>
-                    <span class="s1">&#39;source_project_dataset_table&#39;</span><span class="p">,</span> <span class="n">source_project_dataset_table</span><span class="p">)</span>
+                <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">source_project_dataset_table</span><span class="p">,</span>
+                                 <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                                 <span class="n">var_name</span><span class="o">=</span><span class="s1">&#39;source_project_dataset_table&#39;</span><span class="p">)</span>
             <span class="n">source_project_dataset_tables_fixup</span><span class="o">.</span><span class="n">append</span><span class="p">({</span>
                 <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="n">source_project</span><span class="p">,</span>
                 <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">source_dataset</span><span class="p">,</span>
                 <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">source_table</span>
             <span class="p">})</span>
 
-        <span class="k">assert</span> <span class="mi">3</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">destination_project_dataset_table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)),</span> <span class="p">(</span>
-            <span class="s1">&#39;Expected destination_project_dataset_table in the format of &#39;</span>
-            <span class="s1">&#39;&lt;project&gt;.&lt;dataset&gt;.&lt;table&gt;. &#39;</span>
-            <span class="s1">&#39;Got: {}&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">destination_project_dataset_table</span><span class="p">)</span>
-
         <span class="n">destination_project</span><span class="p">,</span> <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
-            <span class="n">destination_project_dataset_table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
+            <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">destination_project_dataset_table</span><span class="p">,</span>
+                             <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
         <span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
             <span class="s1">&#39;copy&#39;</span><span class="p">:</span> <span class="p">{</span>
                 <span class="s1">&#39;createDisposition&#39;</span><span class="p">:</span> <span class="n">create_disposition</span><span class="p">,</span>
@@ -477,7 +554,13 @@
                  <span class="n">create_disposition</span><span class="o">=</span><span class="s1">&#39;CREATE_IF_NEEDED&#39;</span><span class="p">,</span>
                  <span class="n">skip_leading_rows</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
                  <span class="n">write_disposition</span><span class="o">=</span><span class="s1">&#39;WRITE_EMPTY&#39;</span><span class="p">,</span>
-                 <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">&#39;,&#39;</span><span class="p">):</span>
+                 <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">&#39;,&#39;</span><span class="p">,</span>
+                 <span class="n">max_bad_records</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+                 <span class="n">quote_character</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                 <span class="n">allow_jagged_rows</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                 <span class="n">schema_update_options</span><span class="o">=</span><span class="p">(),</span>
+                 <span class="n">src_fmt_configs</span><span class="o">=</span><span class="p">{}):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Executes a BigQuery load command to load data from Google Cloud Storage</span>
 <span class="sd">        to BigQuery. See here:</span>
@@ -487,9 +570,9 @@
 <span class="sd">        For more details about these parameters.</span>
 
 <span class="sd">        :param destination_project_dataset_table:</span>
-<span class="sd">            The dotted (&lt;project&gt;.)&lt;dataset&gt;.&lt;table&gt; BigQuery table to load data into.</span>
-<span class="sd">            If &lt;project&gt; is not included, project will be the project defined in</span>
-<span class="sd">            the connection json.</span>
+<span class="sd">            The dotted (&lt;project&gt;.|&lt;project&gt;:)&lt;dataset&gt;.&lt;table&gt; BigQuery table to load</span>
+<span class="sd">            data into. If &lt;project&gt; is not included, project will be the project defined</span>
+<span class="sd">            in the connection json.</span>
 <span class="sd">        :type destination_project_dataset_table: string</span>
 <span class="sd">        :param schema_fields: The schema field list as defined here:</span>
 <span class="sd">            https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load</span>
@@ -508,10 +591,56 @@
 <span class="sd">        :type write_disposition: string</span>
 <span class="sd">        :param field_delimiter: The delimiter to use when loading from a CSV.</span>
 <span class="sd">        :type field_delimiter: string</span>
+<span class="sd">        :param max_bad_records: The maximum number of bad records that BigQuery can</span>
+<span class="sd">            ignore when running the job.</span>
+<span class="sd">        :type max_bad_records: int</span>
+<span class="sd">        :param quote_character: The value that is used to quote data sections in a CSV file.</span>
+<span class="sd">        :type quote_character: string</span>
+<span class="sd">        :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not (false).</span>
+<span class="sd">        :type allow_quoted_newlines: boolean</span>
+<span class="sd">        :param allow_jagged_rows: Accept rows that are missing trailing optional columns.</span>
+<span class="sd">            The missing values are treated as nulls. If false, records with missing trailing columns</span>
+<span class="sd">            are treated as bad records, and if there are too many bad records, an invalid error is</span>
+<span class="sd">            returned in the job result. Only applicable when soure_format is CSV.</span>
+<span class="sd">        :type allow_jagged_rows: bool</span>
+<span class="sd">        :param schema_update_options: Allows the schema of the desitination</span>
+<span class="sd">            table to be updated as a side effect of the load job.</span>
+<span class="sd">        :type schema_update_options: list</span>
+<span class="sd">        :param src_fmt_configs: configure optional fields specific to the source format</span>
+<span class="sd">        :type src_fmt_configs: dict</span>
 <span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="c1"># bigquery only allows certain source formats</span>
+        <span class="c1"># we check to make sure the passed source format is valid</span>
+        <span class="c1"># if it&#39;s not, we raise a ValueError</span>
+        <span class="c1"># Refer to this link for more details:</span>
+        <span class="c1">#   https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.tableDefinitions.(key).sourceFormat</span>
+        <span class="n">source_format</span> <span class="o">=</span> <span class="n">source_format</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
+        <span class="n">allowed_formats</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;CSV&quot;</span><span class="p">,</span> <span class="s2">&quot;NEWLINE_DELIMITED_JSON&quot;</span><span class="p">,</span> <span class="s2">&quot;AVRO&quot;</span><span class="p">,</span> <span class="s2">&quot;GOOGLE_SHEETS&quot;</span><span class="p">,</span> <span class="s2">&quot;DATASTORE_BACKUP&quot;</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">source_format</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">allowed_formats</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">{0}</span><span class="s2"> is not a valid source format. &quot;</span>
+                    <span class="s2">&quot;Please use one of the following types: </span><span class="si">{1}</span><span class="s2">&quot;</span>
+                    <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">source_format</span><span class="p">,</span> <span class="n">allowed_formats</span><span class="p">))</span>
+
+        <span class="c1"># bigquery also allows you to define how you want a table&#39;s schema to change</span>
+        <span class="c1"># as a side effect of a load</span>
+        <span class="c1"># for more details:</span>
+        <span class="c1">#   https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schemaUpdateOptions</span>
+        <span class="n">allowed_schema_update_options</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="s1">&#39;ALLOW_FIELD_ADDITION&#39;</span><span class="p">,</span>
+            <span class="s2">&quot;ALLOW_FIELD_RELAXATION&quot;</span>
+        <span class="p">]</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="nb">set</span><span class="p">(</span><span class="n">allowed_schema_update_options</span><span class="p">)</span><span class="o">.</span><span class="n">issuperset</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">)):</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2"> contains invalid schema update options. &quot;</span>
+                <span class="s2">&quot;Please only use one or more of the following options: </span><span class="si">{1}</span><span class="s2">&quot;</span>
+                <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">,</span> <span class="n">allowed_schema_update_options</span><span class="p">)</span>
+            <span class="p">)</span>
+
         <span class="n">destination_project</span><span class="p">,</span> <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
-            <span class="bp">self</span><span class="o">.</span><span class="n">_split_project_dataset_table_input</span><span class="p">(</span>
-                <span class="s1">&#39;destination_project_dataset_table&#39;</span><span class="p">,</span> <span class="n">destination_project_dataset_table</span><span class="p">)</span>
+            <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">destination_project_dataset_table</span><span class="p">,</span>
+                             <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                             <span class="n">var_name</span><span class="o">=</span><span class="s1">&#39;destination_project_dataset_table&#39;</span><span class="p">)</span>
 
         <span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
             <span class="s1">&#39;load&#39;</span><span class="p">:</span> <span class="p">{</span>
@@ -521,43 +650,62 @@
                     <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">destination_dataset</span><span class="p">,</span>
                     <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">destination_table</span><span class="p">,</span>
                 <span class="p">},</span>
-                <span class="s1">&#39;schema&#39;</span><span class="p">:</span> <span class="p">{</span>
-                    <span class="s1">&#39;fields&#39;</span><span class="p">:</span> <span class="n">schema_fields</span>
-                <span class="p">},</span>
                 <span class="s1">&#39;sourceFormat&#39;</span><span class="p">:</span> <span class="n">source_format</span><span class="p">,</span>
                 <span class="s1">&#39;sourceUris&#39;</span><span class="p">:</span> <span class="n">source_uris</span><span class="p">,</span>
                 <span class="s1">&#39;writeDisposition&#39;</span><span class="p">:</span> <span class="n">write_disposition</span><span class="p">,</span>
             <span class="p">}</span>
         <span class="p">}</span>
+        <span class="k">if</span> <span class="n">schema_fields</span><span class="p">:</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">][</span><span class="s1">&#39;schema&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
+                <span class="s1">&#39;fields&#39;</span><span class="p">:</span> <span class="n">schema_fields</span>
+            <span class="p">}</span>
+
+        <span class="k">if</span> <span class="n">schema_update_options</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">write_disposition</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;WRITE_APPEND&quot;</span><span class="p">,</span> <span class="s2">&quot;WRITE_TRUNCATE&quot;</span><span class="p">]:</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                    <span class="s2">&quot;schema_update_options is only &quot;</span>
+                    <span class="s2">&quot;allowed if write_disposition is &quot;</span>
+                    <span class="s2">&quot;&#39;WRITE_APPEND&#39; or &#39;WRITE_TRUNCATE&#39;.&quot;</span>
+                <span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                    <span class="s2">&quot;Adding experimental &quot;</span>
+                    <span class="s2">&quot;&#39;schemaUpdateOptions&#39;: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">)</span>
+                <span class="p">)</span>
+                <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">][</span><span class="s1">&#39;schemaUpdateOptions&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">schema_update_options</span>
+
+        <span class="k">if</span> <span class="n">max_bad_records</span><span class="p">:</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">][</span><span class="s1">&#39;maxBadRecords&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">max_bad_records</span>
+
+        <span class="c1"># if following fields are not specified in src_fmt_configs,</span>
+        <span class="c1"># honor the top-level params for backward-compatibility</span>
+        <span class="k">if</span> <span class="s1">&#39;skipLeadingRows&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+            <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">&#39;skipLeadingRows&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">skip_leading_rows</span>
+        <span class="k">if</span> <span class="s1">&#39;fieldDelimiter&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+            <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">&#39;fieldDelimiter&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">field_delimiter</span>
+        <span class="k">if</span> <span class="n">quote_character</span><span class="p">:</span>
+            <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">&#39;quote&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">quote_character</span>
+        <span class="k">if</span> <span class="n">allow_quoted_newlines</span><span class="p">:</span>
+            <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">&#39;allowQuotedNewlines&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">allow_quoted_newlines</span>
+
+        <span class="n">src_fmt_to_configs_mapping</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;CSV&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;allowJaggedRows&#39;</span><span class="p">,</span> <span class="s1">&#39;allowQuotedNewlines&#39;</span><span class="p">,</span> <span class="s1">&#39;autodetect&#39;</span><span class="p">,</span>
+                    <span class="s1">&#39;fieldDelimiter&#39;</span><span class="p">,</span> <span class="s1">&#39;skipLeadingRows&#39;</span><span class="p">,</span> <span class="s1">&#39;ignoreUnknownValues&#39;</span><span class="p">,</span>
+                    <span class="s1">&#39;nullMarker&#39;</span><span class="p">,</span> <span class="s1">&#39;quote&#39;</span><span class="p">],</span>
+            <span class="s1">&#39;DATASTORE_BACKUP&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;projectionFields&#39;</span><span class="p">],</span>
+            <span class="s1">&#39;NEWLINE_DELIMITED_JSON&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;autodetect&#39;</span><span class="p">,</span> <span class="s1">&#39;ignoreUnknownValues&#39;</span><span class="p">],</span>
+            <span class="s1">&#39;AVRO&#39;</span><span class="p">:</span> <span class="p">[],</span>
+        <span class="p">}</span>
+        <span class="n">valid_configs</span> <span class="o">=</span> <span class="n">src_fmt_to_configs_mapping</span><span class="p">[</span><span class="n">source_format</span><span class="p">]</span>
+        <span class="n">src_fmt_configs</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="o">.</span><span class="n">items</span><span class="p">()</span>
+                           <span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">valid_configs</span><span class="p">}</span>
+        <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">src_fmt_configs</span><span class="p">)</span>
 
-        <span class="k">if</span> <span class="n">source_format</span> <span class="o">==</span> <span class="s1">&#39;CSV&#39;</span><span class="p">:</span>
-            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">][</span><span class="s1">&#39;skipLeadingRows&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">skip_leading_rows</span>
-            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">][</span><span class="s1">&#39;fieldDelimiter&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">field_delimiter</span>
+        <span class="k">if</span> <span class="n">allow_jagged_rows</span><span class="p">:</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">][</span><span class="s1">&#39;allowJaggedRows&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">allow_jagged_rows</span>
 
         <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
 
-    <span class="k">def</span> <span class="nf">_split_project_dataset_table_input</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">var_name</span><span class="p">,</span> <span class="n">project_dataset_table</span><span class="p">):</span>
-        <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        :param var_name: the name of the variable input, for logging and erroring purposes.</span>
-<span class="sd">        :type var_name: str</span>
-<span class="sd">        :param project_dataset_table: input string in (&lt;project&gt;.)&lt;dataset&gt;.&lt;project&gt; format.</span>
-<span class="sd">            if project is not included in the string, self.project_id will be returned in the tuple.</span>
-<span class="sd">        :type project_dataset_table: str</span>
-<span class="sd">        :return: (project, dataset, table) tuple</span>
-<span class="sd">        &quot;&quot;&quot;</span>
-        <span class="n">table_split</span> <span class="o">=</span> <span class="n">project_dataset_table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
-        <span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">table_split</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span> <span class="ow">or</span> <span class="nb">len</span><span class="p">(</span><span class="n">table_split</span><span class="p">)</span> <span class="o">==</span> <span class="mi">3</span><span class="p">,</span> <span class="p">(</span>
-            <span class="s1">&#39;Expected {var} in the format of (&lt;project.)&lt;dataset&gt;.&lt;table&gt;, &#39;</span>
-            <span class="s1">&#39;got {input}&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">var</span><span class="o">=</span><span class="n">var_name</span><span class="p">,</span> <span class="nb">input</span><span class="o">=</span><span class="n">project_dataset_table</span><span class="p">)</span>
-
-        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">table_split</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span>
-            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;project not included in {var}: {input}; using project &quot;{project}&quot;&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">var</span><span class="o">=</span><span class="n">var_name</span><span class="p">,</span> <span class="nb">input</span><span class="o">=</span><span class="n">project_datase [...]
-            <span class="n">dataset</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table_split</span>
-            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">table</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="n">project</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table_split</span>
-            <span class="k">return</span> <span class="n">project</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">table</span>
-
     <span class="k">def</span> <span class="nf">run_with_configuration</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">configuration</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Executes a BigQuery SQL query. See here:</span>
@@ -581,18 +729,32 @@
             <span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">job_data</span><span class="p">)</span> \
             <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
         <span class="n">job_id</span> <span class="o">=</span> <span class="n">query_reply</span><span class="p">[</span><span class="s1">&#39;jobReference&#39;</span><span class="p">][</span><span class="s1">&#39;jobId&#39;</span><span class="p">]</span>
-        <span class="n">job</span> <span class="o">=</span> <span class="n">jobs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">jobId</span><span class="o">=</span><span class="n">job_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span  [...]
 
         <span class="c1"># Wait for query to finish.</span>
-        <span class="k">while</span> <span class="ow">not</span> <span class="n">job</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">][</span><span class="s1">&#39;state&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;DONE&#39;</span><span class="p">:</span>
-            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Waiting for job to complete: </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job_id</span><span class="p">)</span>
-            <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
-            <span class="n">job</span> <span class="o">=</span> <span class="n">jobs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">jobId</span><span class="o">=</span><span class="n">job_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><s [...]
-
-        <span class="c1"># Check if job had errors.</span>
-        <span class="k">if</span> <span class="s1">&#39;errorResult&#39;</span> <span class="ow">in</span> <span class="n">job</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">]:</span>
-            <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
-                <span class="s1">&#39;BigQuery job failed. Final error was: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">job</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">][</span><span class="s1">&#39;errorResult&#39;</span><span class="p">])</span>
+        <span class="n">keep_polling_job</span> <span class="o">=</span> <span class="kc">True</span>
+        <span class="k">while</span> <span class="p">(</span><span class="n">keep_polling_job</span><span class="p">):</span>
+            <span class="k">try</span><span class="p">:</span>
+                <span class="n">job</span> <span class="o">=</span> <span class="n">jobs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">jobId</span><span class="o">=</span><span class="n">job_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</spa [...]
+                <span class="k">if</span> <span class="p">(</span><span class="n">job</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">][</span><span class="s1">&#39;state&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;DONE&#39;</span><span class="p">):</span>
+                    <span class="n">keep_polling_job</span> <span class="o">=</span> <span class="kc">False</span>
+                    <span class="c1"># Check if job had errors.</span>
+                    <span class="k">if</span> <span class="s1">&#39;errorResult&#39;</span> <span class="ow">in</span> <span class="n">job</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">]:</span>
+                        <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
+                            <span class="s1">&#39;BigQuery job failed. Final error was: </span><span class="si">{}</span><span class="s1">. The job was: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                                <span class="n">job</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">][</span><span class="s1">&#39;errorResult&#39;</span><span class="p">],</span> <span class="n">job</span>
+                            <span class="p">)</span>
+                        <span class="p">)</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Waiting for job to complete : </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">jo [...]
+                    <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
+
+            <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+                <span class="k">if</span> <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span> <span class="ow">in</span> <span class="p">[</span><span class="mi">500</span><span class="p">,</span> <span class="mi">503</span><span class="p">]:</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">%s</span><span class="s1">: Retryable error, waiting for job to complete: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class [...]
+                    <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
+                        <span class="s1">&#39;BigQuery job status check failed. Final error was: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span><span class="p">)</span>
 
         <span class="k">return</span> <span class="n">job_id</span>
 
@@ -611,7 +773,7 @@
         <span class="k">return</span> <span class="n">tables_resource</span><span class="p">[</span><span class="s1">&#39;schema&#39;</span><span class="p">]</span>
 
     <span class="k">def</span> <span class="nf">get_tabledata</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">,</span>
-                      <span class="n">max_results</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">page_token</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">start_index</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+                      <span class="n">max_results</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">page_token</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">start_index</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Get the data of a given dataset.table.</span>
 <span class="sd">        see https://cloud.google.com/bigquery/docs/reference/v2/tabledata/list</span>
@@ -639,15 +801,56 @@
             <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
         <span class="p">)</span>
 
-    <span class="k">def</span> <span class="nf">run_table_upsert</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_resource</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+    <span class="k">def</span> <span class="nf">run_table_delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">deletion_dataset_table</span><span class="p">,</span> <span class="n">ignore_if_missing</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Delete an existing table from the dataset;</span>
+<span class="sd">        If the table does not exist, return an error unless ignore_if_missing</span>
+<span class="sd">        is set to True.</span>
+
+<span class="sd">        :param deletion_dataset_table: A dotted</span>
+<span class="sd">        (&lt;project&gt;.|&lt;project&gt;:)&lt;dataset&gt;.&lt;table&gt; that indicates which table</span>
+<span class="sd">        will be deleted.</span>
+<span class="sd">        :type deletion_dataset_table: str</span>
+<span class="sd">        :param ignore_if_missing: if True, then return success even if the</span>
+<span class="sd">        requested table does not exist.</span>
+<span class="sd">        :type ignore_if_missing: boolean</span>
+<span class="sd">        :return:</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="k">assert</span> <span class="s1">&#39;.&#39;</span> <span class="ow">in</span> <span class="n">deletion_dataset_table</span><span class="p">,</span> <span class="p">(</span>
+            <span class="s1">&#39;Expected deletion_dataset_table in the format of &#39;</span>
+            <span class="s1">&#39;&lt;dataset&gt;.&lt;table&gt;. Got: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">deletion_dataset_table</span><span class="p">)</span>
+        <span class="n">deletion_project</span><span class="p">,</span> <span class="n">deletion_dataset</span><span class="p">,</span> <span class="n">deletion_table</span> <span class="o">=</span> \
+            <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">deletion_dataset_table</span><span class="p">,</span>
+                             <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">tables_resource</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span> \
+                <span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">deletion_project</span><span class="p">,</span>
+                        <span class="n">datasetId</span><span class="o">=</span><span class="n">deletion_dataset</span><span class="p">,</span>
+                        <span class="n">tableId</span><span class="o">=</span><span class="n">deletion_table</span><span class="p">)</span> \
+                <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Deleted table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1">.&#39;</span><span class="p">,</span>
+                          <span class="n">deletion_project</span><span class="p">,</span> <span class="n">deletion_dataset</span><span class="p">,</span> <span class="n">deletion_table</span><span class="p">)</span>
+        <span class="k">except</span> <span class="n">HttpError</span><span class="p">:</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="n">ignore_if_missing</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
+                    <span class="s1">&#39;Table deletion failed. Table does not exist.&#39;</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Table does not exist. Skipping.&#39;</span><span class="p">)</span>
+
+
+    <span class="k">def</span> <span class="nf">run_table_upsert</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_resource</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        creates a new, empty table in the dataset;</span>
 <span class="sd">        If the table already exists, update the existing table.</span>
 <span class="sd">        Since BigQuery does not natively allow table upserts, this is not an</span>
 <span class="sd">        atomic operation.</span>
+
 <span class="sd">        :param dataset_id: the dataset to upsert the table into.</span>
 <span class="sd">        :type dataset_id: str</span>
-<span class="sd">        :param table_resource: a table resource. see https://cloud.google.com/bigquery/docs/reference/v2/tables#resource</span>
+<span class="sd">        :param table_resource: a table resource. see</span>
+<span class="sd">            https://cloud.google.com/bigquery/docs/reference/v2/tables#resource</span>
 <span class="sd">        :type table_resource: dict</span>
 <span class="sd">        :param project_id: the project to upsert the table into.  If None,</span>
 <span class="sd">        project will be self.project_id.</span>
@@ -655,15 +858,17 @@
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="c1"># check to see if the table exists</span>
         <span class="n">table_id</span> <span class="o">=</span> <span class="n">table_resource</span><span class="p">[</span><span class="s1">&#39;tableReference&#39;</span><span class="p">][</span><span class="s1">&#39;tableId&#39;</span><span class="p">]</span>
-        <span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+        <span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
         <span class="n">tables_list_resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
                                                       <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
-        <span class="k">while</span> <span class="bp">True</span><span class="p">:</span>
+        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
             <span class="k">for</span> <span class="n">table</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;tables&#39;</span><span class="p">,</span> <span class="p">[]):</span>
                 <span class="k">if</span> <span class="n">table</span><span class="p">[</span><span class="s1">&#39;tableReference&#39;</span><span class="p">][</span><span class="s1">&#39;tableId&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="n">table_id</span><span class="p">:</span>
                     <span class="c1"># found the table, do update</span>
-                    <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> exists, updating.&#39;</span><span class="p">,</span>
-                                 <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                        <span class="s1">&#39;Table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> exists, updating.&#39;</span><span class="p">,</span>
+                        <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span>
+                    <span class="p">)</span>
                     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
                                                         <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
                                                         <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">,</span>
@@ -678,8 +883,10 @@
             <span class="c1"># If there is no next page, then the table doesn&#39;t exist.</span>
             <span class="k">else</span><span class="p">:</span>
                 <span class="c1"># do insert</span>
-                <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> does not exist. creating.&#39;</span><span class="p">,</span>
-                             <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                    <span class="s1">&#39;Table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> does not exist. creating.&#39;</span><span class="p">,</span>
+                    <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span>
+                <span class="p">)</span>
                 <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
                                                     <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
                                                     <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
@@ -688,12 +895,13 @@
                                       <span class="n">source_dataset</span><span class="p">,</span>
                                       <span class="n">view_dataset</span><span class="p">,</span>
                                       <span class="n">view_table</span><span class="p">,</span>
-                                      <span class="n">source_project</span> <span class="o">=</span> <span class="bp">None</span><span class="p">,</span>
-                                      <span class="n">view_project</span> <span class="o">=</span> <span class="bp">None</span><span class="p">):</span>
+                                      <span class="n">source_project</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+                                      <span class="n">view_project</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Grant authorized view access of a dataset to a view table.</span>
 <span class="sd">        If this view has already been granted access to the dataset, do nothing.</span>
 <span class="sd">        This method is not atomic.  Running it may clobber a simultaneous update.</span>
+
 <span class="sd">        :param source_dataset: the source dataset</span>
 <span class="sd">        :type source_dataset: str</span>
 <span class="sd">        :param view_dataset: the dataset that the view is in</span>
@@ -723,18 +931,20 @@
                                 <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">view_table</span><span class="p">}}</span>
         <span class="c1"># check to see if the view we want to add already exists.</span>
         <span class="k">if</span> <span class="n">view_access</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">access</span><span class="p">:</span>
-            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;granting table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> authorized view access to </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1"> dataset.&#39;</span><span class="p">,</span>
-                         <span class="n">view_project</span><span class="p">,</span> <span class="n">view_dataset</span><span class="p">,</span> <span class="n">view_table</span><span class="p">,</span>
-                         <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span><span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                <span class="s1">&#39;Granting table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> authorized view access to </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1"> dataset.&#39;</span><span class="p">,</span>
+                <span class="n">view_project</span><span class="p">,</span> <span class="n">view_dataset</span><span class="p">,</span> <span class="n">view_table</span><span class="p">,</span> <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span>
+            <span class="p">)</span>
             <span class="n">access</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">view_access</span><span class="p">)</span>
             <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">datasets</span><span class="p">()</span><span class="o">.</span><span class="n">patch</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">source_project</span><span class="p">,</span>
                                                  <span class="n">datasetId</span><span class="o">=</span><span class="n">source_dataset</span><span class="p">,</span>
                                                  <span class="n">body</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;access&#39;</span><span class="p">:</span> <span class="n">access</span><span class="p">})</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
         <span class="k">else</span><span class="p">:</span>
             <span class="c1"># if view is already in access, do nothing.</span>
-            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> already has authorized view access to </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1"> dataset.&#39;</span><span class="p">,</span>
-                         <span class="n">view_project</span><span class="p">,</span> <span class="n">view_dataset</span><span class="p">,</span> <span class="n">view_table</span><span class="p">,</span>
-                         <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span><span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                <span class="s1">&#39;Table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> already has authorized view access to </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1"> dataset.&#39;</span><span class="p">,</span>
+                <span class="n">view_project</span><span class="p">,</span> <span class="n">view_dataset</span><span class="p">,</span> <span class="n">view_table</span><span class="p">,</span> <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span>
+            <span class="p">)</span>
             <span class="k">return</span> <span class="n">source_dataset_resource</span>
 
 
@@ -748,12 +958,12 @@
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">service</span><span class="p">,</span> <span class="n">project_id</span><span class="p">):</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryCursor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">service</span><span class="o">=</span><span class="n">service</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="n">project_id</span><span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">buffersize</span> <span class="o">=</span> <span class="bp">None</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span> <span class="o">=</span> <span class="bp">None</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="bp">None</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryCursor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">service</span><span class="o">=</span><span class="n">service</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="n">project_id</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">buffersize</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="kc">None</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">buffer</span> <span class="o">=</span> <span class="p">[]</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">all_pages_loaded</span> <span class="o">=</span> <span class="bp">False</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">all_pages_loaded</span> <span class="o">=</span> <span class="kc">False</span>
 
     <span class="nd">@property</span>
     <span class="k">def</span> <span class="nf">description</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
@@ -769,7 +979,7 @@
         <span class="sd">&quot;&quot;&quot; By default, return -1 to indicate that this is not supported. &quot;&quot;&quot;</span>
         <span class="k">return</span> <span class="o">-</span><span class="mi">1</span>
 
-    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">operation</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">operation</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Executes a BigQuery query, and returns the job ID.</span>
 
@@ -805,11 +1015,11 @@
 <span class="sd">        the next page, and load it into the buffer.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span><span class="p">:</span>
-            <span class="k">return</span> <span class="bp">None</span>
+            <span class="k">return</span> <span class="kc">None</span>
 
         <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">buffer</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
             <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">all_pages_loaded</span><span class="p">:</span>
-                <span class="k">return</span> <span class="bp">None</span>
+                <span class="k">return</span> <span class="kc">None</span>
 
             <span class="n">query_results</span> <span class="o">=</span> <span class="p">(</span>
                 <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span>
@@ -834,18 +1044,18 @@
                     <span class="bp">self</span><span class="o">.</span><span class="n">buffer</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">typed_row</span><span class="p">)</span>
 
                 <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span><span class="p">:</span>
-                    <span class="bp">self</span><span class="o">.</span><span class="n">all_pages_loaded</span> <span class="o">=</span> <span class="bp">True</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">all_pages_loaded</span> <span class="o">=</span> <span class="kc">True</span>
 
             <span class="k">else</span><span class="p">:</span>
                 <span class="c1"># Reset all state since we&#39;ve exhausted the results.</span>
-                <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span> <span class="o">=</span> <span class="bp">None</span>
-                <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="bp">None</span>
-                <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span> <span class="o">=</span> <span class="bp">None</span>
-                <span class="k">return</span> <span class="bp">None</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="k">return</span> <span class="kc">None</span>
 
         <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">buffer</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
 
-    <span class="k">def</span> <span class="nf">fetchmany</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+    <span class="k">def</span> <span class="nf">fetchmany</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Fetch the next set of rows of a query result, returning a sequence of sequences (e.g. a</span>
 <span class="sd">        list of tuples). An empty sequence is returned when no more rows are available.</span>
@@ -856,12 +1066,12 @@
 <span class="sd">        An :py:class:`~pyhive.exc.Error` (or subclass) exception is raised if the previous call to</span>
 <span class="sd">        :py:meth:`execute` did not produce any result set or no call was issued yet.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
-        <span class="k">if</span> <span class="n">size</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
+        <span class="k">if</span> <span class="n">size</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
             <span class="n">size</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">arraysize</span>
         <span class="n">result</span> <span class="o">=</span> <span class="p">[]</span>
         <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">size</span><span class="p">):</span>
             <span class="n">one</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span>
-            <span class="k">if</span> <span class="n">one</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">one</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
                 <span class="k">break</span>
             <span class="k">else</span><span class="p">:</span>
                 <span class="n">result</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">one</span><span class="p">)</span>
@@ -873,9 +1083,9 @@
 <span class="sd">        (e.g. a list of tuples).</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">result</span> <span class="o">=</span> <span class="p">[]</span>
-        <span class="k">while</span> <span class="bp">True</span><span class="p">:</span>
+        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
             <span class="n">one</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span>
-            <span class="k">if</span> <span class="n">one</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">one</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
                 <span class="k">break</span>
             <span class="k">else</span><span class="p">:</span>
                 <span class="n">result</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">one</span><span class="p">)</span>
@@ -895,7 +1105,7 @@
         <span class="sd">&quot;&quot;&quot; Does nothing by default &quot;&quot;&quot;</span>
         <span class="k">pass</span>
 
-    <span class="k">def</span> <span class="nf">setoutputsize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">size</span><span class="p">,</span> <span class="n">column</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+    <span class="k">def</span> <span class="nf">setoutputsize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">size</span><span class="p">,</span> <span class="n">column</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot; Does nothing by default &quot;&quot;&quot;</span>
         <span class="k">pass</span>
 
@@ -905,9 +1115,9 @@
     <span class="c1"># inspired by MySQL Python Connector (conversion.py)</span>
     <span class="n">string_parameters</span> <span class="o">=</span> <span class="p">{}</span>
     <span class="k">for</span> <span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span> <span class="ow">in</span> <span class="n">parameters</span><span class="o">.</span><span class="n">iteritems</span><span class="p">():</span>
-        <span class="k">if</span> <span class="n">value</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
+        <span class="k">if</span> <span class="n">value</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
             <span class="n">string_parameters</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;NULL&#39;</span>
-        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
+        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">basestring</span><span class="p">):</span>
             <span class="n">string_parameters</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;&#39;&quot;</span> <span class="o">+</span> <span class="n">_escape</span><span class="p">(</span><span class="n">value</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot;&#39;&quot;</span>
         <span class="k">else</span><span class="p">:</span>
             <span class="n">string_parameters</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
@@ -930,8 +1140,8 @@
 <span class="sd">    Helper method that casts a BigQuery row to the appropriate data types.</span>
 <span class="sd">    This is useful because BigQuery returns all fields as strings.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="k">if</span> <span class="n">string_field</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
-        <span class="k">return</span> <span class="bp">None</span>
+    <span class="k">if</span> <span class="n">string_field</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="k">return</span> <span class="kc">None</span>
     <span class="k">elif</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">&#39;INTEGER&#39;</span> <span class="ow">or</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">&#39;TIMESTAMP&#39;</span><span class="p">:</span>
         <span class="k">return</span> <span class="nb">int</span><span class="p">(</span><span class="n">string_field</span><span class="p">)</span>
     <span class="k">elif</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">&#39;FLOAT&#39;</span><span class="p">:</span>
@@ -941,9 +1151,74 @@
         <span class="k">return</span> <span class="n">string_field</span> <span class="o">==</span> <span class="s1">&#39;true&#39;</span>
     <span class="k">else</span><span class="p">:</span>
         <span class="k">return</span> <span class="n">string_field</span>
+
+
+<span class="k">def</span> <span class="nf">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="p">,</span> <span class="n">default_project_id</span><span class="p">,</span> <span class="n">var_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="k">assert</span> <span class="n">default_project_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">,</span> <span class="s2">&quot;INTERNAL: No default project is specified&quot;</span>
+
+    <span class="k">def</span> <span class="nf">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">var_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">return</span> <span class="s2">&quot;&quot;</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">return</span> <span class="s2">&quot;Format exception for </span><span class="si">{var}</span><span class="s2">: &quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">var</span><span class="o">=</span><span class="n">var_name</span><span class="p">)</span>
+
+    <span class="k">if</span> <span class="n">table_input</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span> <span class="o">+</span> <span class="n">table_input</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s1">&#39;:&#39;</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">3</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="ne">Exception</span><span class="p">((</span>
+            <span class="s1">&#39;</span><span class="si">{var}</span><span class="s1">Use either : or . to specify project &#39;</span>
+            <span class="s1">&#39;got </span><span class="si">{input}</span><span class="s1">&#39;</span>
+        <span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">var</span><span class="o">=</span><span class="n">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">),</span> <span class="nb">input</span><span class="o">=</span><span class="n">table_input</span><span class="p">))</span>
+
+    <span class="n">cmpt</span> <span class="o">=</span> <span class="n">table_input</span><span class="o">.</span><span class="n">rsplit</span><span class="p">(</span><span class="s1">&#39;:&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+    <span class="n">project_id</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="n">rest</span> <span class="o">=</span> <span class="n">table_input</span>
+    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">cmpt</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+        <span class="n">project_id</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="n">rest</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+    <span class="k">elif</span> <span class="nb">len</span><span class="p">(</span><span class="n">cmpt</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span> <span class="ow">and</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s1">&#39;:&#39;</span><span class="p">)</span> <span class="o">&lt;=</span> <span class= [...]
+        <span class="k">if</span> <span class="n">cmpt</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">2</span><span class="p">:</span>
+            <span class="n">project_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+            <span class="n">rest</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="ne">Exception</span><span class="p">((</span>
+            <span class="s1">&#39;</span><span class="si">{var}</span><span class="s1">Expect format of (&lt;project:)&lt;dataset&gt;.&lt;table&gt;, &#39;</span>
+            <span class="s1">&#39;got </span><span class="si">{input}</span><span class="s1">&#39;</span>
+        <span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">var</span><span class="o">=</span><span class="n">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">),</span> <span class="nb">input</span><span class="o">=</span><span class="n">table_input</span><span class="p">))</span>
+
+    <span class="n">cmpt</span> <span class="o">=</span> <span class="n">rest</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
+    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">cmpt</span><span class="p">)</span> <span class="o">==</span> <span class="mi">3</span><span class="p">:</span>
+        <span class="k">assert</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">,</span> <span class="p">(</span>
+            <span class="s2">&quot;</span><span class="si">{var}</span><span class="s2">Use either : or . to specify project&quot;</span>
+        <span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">var</span><span class="o">=</span><span class="n">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">))</span>
+        <span class="n">project_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+        <span class="n">dataset_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+        <span class="n">table_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span>
+
+    <span class="k">elif</span> <span class="nb">len</span><span class="p">(</span><span class="n">cmpt</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span>
+        <span class="n">dataset_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+        <span class="n">table_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="ne">Exception</span><span class="p">((</span>
+            <span class="s1">&#39;</span><span class="si">{var}</span><span class="s1">Expect format of (&lt;project.|&lt;project:)&lt;dataset&gt;.&lt;table&gt;, &#39;</span>
+            <span class="s1">&#39;got </span><span class="si">{input}</span><span class="s1">&#39;</span>
+        <span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">var</span><span class="o">=</span><span class="n">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">),</span> <span class="nb">input</span><span class="o">=</span><span class="n">table_input</span><span class="p">))</span>
+
+    <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="k">if</span> <span class="n">var_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
+            <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                <span class="s1">&#39;Project not included in </span><span class="si">{var}</span><span class="s1">: </span><span class="si">{input}</span><span class="s1">; using project &quot;</span><span class="si">{project}</span><span class="s1">&quot;&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                    <span class="n">var</span><span class="o">=</span><span class="n">var_name</span><span class="p">,</span> <span class="nb">input</span><span class="o">=</span><span class="n">table_input</span><span class="p">,</span> <span class="n">project</span><span class="o">=</span><span class="n">default_project_id</span>
+                <span class="p">)</span>
+            <span class="p">)</span>
+        <span class="n">project_id</span> <span class="o">=</span> <span class="n">default_project_id</span>
+
+    <span class="k">return</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span>
 </pre></div>
 
            </div>
+           <div class="articleComments">
+            
+           </div>
           </div>
           <footer>
   
@@ -972,22 +1247,23 @@
 
     <script type="text/javascript">
         var DOCUMENTATION_OPTIONS = {
-            URL_ROOT:'../',
+            URL_ROOT:'../../../../',
             VERSION:'',
             COLLAPSE_INDEX:false,
             FILE_SUFFIX:'.html',
-            HAS_SOURCE:  true
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
         };
     </script>
-      <script type="text/javascript" src="../_static/jquery.js"></script>
-      <script type="text/javascript" src="../_static/underscore.js"></script>
-      <script type="text/javascript" src="../_static/doctools.js"></script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
 
   
 
   
   
-    <script type="text/javascript" src="../_static/js/theme.js"></script>
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
   
 
   
diff --git a/_modules/airflow/contrib/hooks/datastore_hook.html b/_modules/airflow/contrib/hooks/datastore_hook.html
new file mode 100644
index 0000000..21363c3
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/datastore_hook.html
@@ -0,0 +1,410 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.datastore_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.datastore_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.datastore_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">import</span> <span class="nn">json</span>
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="k">import</span> <span class="n">build</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
+
+
+<div class="viewcode-block" id="DatastoreHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook">[docs]</a><span class="k">class</span> <span class="nc">DatastoreHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Interact with Google Cloud Datastore. This hook uses the Google Cloud Platform</span>
+<span class="sd">    connection.</span>
+
+<span class="sd">    This object is not threads safe. If you want to make multiple requests</span>
+<span class="sd">    simultaniously, you will need to create a hook per thread.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">datastore_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_datastore_default&#39;</span><span class="p">,</span>
+                 <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">DatastoreHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">datastore_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">admin_connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">(</span><span class="s1">&#39;v1beta1&#39;</span><span class="p">)</span>
+
+<div class="viewcode-block" id="DatastoreHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">version</span><span class="o">=</span><span class="s1">&#39;v1&#39;</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Returns a Google Cloud Storage service object.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">&#39;datastore&#39;</span><span class="p">,</span> <span class="n">version</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.allocate_ids"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.allocate_ids">[docs]</a>    <span class="k">def</span> <span class="nf">allocate_ids</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">partialKeys</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Allocate IDs for incomplete keys.</span>
+<span class="sd">        see https://cloud.google.com/datastore/docs/reference/rest/v1/projects/allocateIds</span>
+
+<span class="sd">        :param partialKeys: a list of partial keys</span>
+<span class="sd">        :return: a list of full keys.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">allocateIds</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">bo [...]
+        <span class="k">return</span> <span class="n">resp</span><span class="p">[</span><span class="s1">&#39;keys&#39;</span><span class="p">]</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.begin_transaction"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.begin_transaction">[docs]</a>    <span class="k">def</span> <span class="nf">begin_transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Get a new transaction handle</span>
+<span class="sd">        see https://cloud.google.com/datastore/docs/reference/rest/v1/projects/beginTransaction</span>
+
+<span class="sd">        :return: a transaction handle</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">beginTransaction</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class=" [...]
+        <span class="k">return</span> <span class="n">resp</span><span class="p">[</span><span class="s1">&#39;transaction&#39;</span><span class="p">]</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.commit"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.commit">[docs]</a>    <span class="k">def</span> <span class="nf">commit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">body</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Commit a transaction, optionally creating, deleting or modifying some entities.</span>
+<span class="sd">        see https://cloud.google.com/datastore/docs/reference/rest/v1/projects/commit</span>
+
+<span class="sd">        :param body: the body of the commit request</span>
+<span class="sd">        :return: the response body of the commit request</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">commit</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</s [...]
+        <span class="k">return</span> <span class="n">resp</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.lookup"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.lookup">[docs]</a>    <span class="k">def</span> <span class="nf">lookup</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">keys</span><span class="p">,</span> <span class="n">read_consistency</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <sp [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Lookup some entities by key</span>
+<span class="sd">        see https://cloud.google.com/datastore/docs/reference/rest/v1/projects/lookup</span>
+<span class="sd">        :param keys: the keys to lookup</span>
+<span class="sd">        :param read_consistency: the read consistency to use. default, strong or eventual.</span>
+<span class="sd">                Cannot be used with a transaction.</span>
+<span class="sd">        :param transaction: the transaction to use, if any.</span>
+<span class="sd">        :return: the response body of the lookup request.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">body</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;keys&#39;</span><span class="p">:</span> <span class="n">keys</span><span class="p">}</span>
+        <span class="k">if</span> <span class="n">read_consistency</span><span class="p">:</span>
+            <span class="n">body</span><span class="p">[</span><span class="s1">&#39;readConsistency&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">read_consistency</span>
+        <span class="k">if</span> <span class="n">transaction</span><span class="p">:</span>
+            <span class="n">body</span><span class="p">[</span><span class="s1">&#39;transaction&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">transaction</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">lookup</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</ [...]
+
+<div class="viewcode-block" id="DatastoreHook.rollback"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.rollback">[docs]</a>    <span class="k">def</span> <span class="nf">rollback</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">transaction</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Roll back a transaction</span>
+<span class="sd">        see https://cloud.google.com/datastore/docs/reference/rest/v1/projects/rollback</span>
+<span class="sd">        :param transaction: the transaction to roll back</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">rollback</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="p">{</span [...]
+            <span class="o">.</span><span class="n">execute</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.run_query"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.run_query">[docs]</a>    <span class="k">def</span> <span class="nf">run_query</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">body</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Run a query for entities.</span>
+<span class="sd">        see https://cloud.google.com/datastore/docs/reference/rest/v1/projects/runQuery</span>
+<span class="sd">        :param body: the body of the query request</span>
+<span class="sd">        :return: the batch of query results.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">runQuery</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body< [...]
+        <span class="k">return</span> <span class="n">resp</span><span class="p">[</span><span class="s1">&#39;batch&#39;</span><span class="p">]</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.get_operation"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.get_operation">[docs]</a>    <span class="k">def</span> <span class="nf">get_operation</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Gets the latest state of a long-running operation</span>
+
+<span class="sd">        :param name: the name of the operation resource</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">operations</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">name</span><span class="p">)</span><span cl [...]
+        <span class="k">return</span> <span class="n">resp</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.delete_operation"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.delete_operation">[docs]</a>    <span class="k">def</span> <span class="nf">delete_operation</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Deletes the long-running operation</span>
+
+<span class="sd">        :param name: the name of the operation resource</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">operations</span><span class="p">()</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">name</span><span class="p">)</span><span [...]
+        <span class="k">return</span> <span class="n">resp</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.poll_operation_until_done"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.poll_operation_until_done">[docs]</a>    <span class="k">def</span> <span class="nf">poll_operation_until_done</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">polling_interval_in_seconds</span><span class= [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Poll backup operation state until it&#39;s completed</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+            <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_operation</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
+            <span class="n">state</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;metadata&#39;</span><span class="p">][</span><span class="s1">&#39;common&#39;</span><span class="p">][</span><span class="s1">&#39;state&#39;</span><span class="p">]</span>
+            <span class="k">if</span> <span class="n">state</span> <span class="o">==</span> <span class="s1">&#39;PROCESSING&#39;</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Operation is processing. Re-polling state in </span><span class="si">{}</span><span class="s1"> seconds&#39;</span>
+                              <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">polling_interval_in_seconds</span><span class="p">))</span>
+                <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">polling_interval_in_seconds</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">return</span> <span class="n">result</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.export_to_storage_bucket"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.export_to_storage_bucket">[docs]</a>    <span class="k">def</span> <span class="nf">export_to_storage_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">namespace</span><span class="o">=</span><span c [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Export entities from Cloud Datastore to Cloud Storage for backup</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">output_uri_prefix</span> <span class="o">=</span> <span class="s1">&#39;gs://&#39;</span> <span class="o">+</span> <span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="p">[</span><span class="n">bucket</span><span class="p">,</span> <span clas [...]
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">entity_filter</span><span class="p">:</span>
+            <span class="n">entity_filter</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">labels</span><span class="p">:</span>
+            <span class="n">labels</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;outputUrlPrefix&#39;</span><span class="p">:</span> <span class="n">output_uri_prefix</span><span class="p">,</span>
+            <span class="s1">&#39;entityFilter&#39;</span><span class="p">:</span> <span class="n">entity_filter</span><span class="p">,</span>
+            <span class="s1">&#39;labels&#39;</span><span class="p">:</span> <span class="n">labels</span><span class="p">,</span>
+        <span class="p">}</span>
+        <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">admin_connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">export</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">b [...]
+        <span class="k">return</span> <span class="n">resp</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.import_from_storage_bucket"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.import_from_storage_bucket">[docs]</a>    <span class="k">def</span> <span class="nf">import_from_storage_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">file</span><span class="p">,</span> <span [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Import a backup from Cloud Storage to Cloud Datastore</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">input_url</span> <span class="o">=</span> <span class="s1">&#39;gs://&#39;</span> <span class="o">+</span> <span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="p">[</span><span class="n">bucket</span><span class="p">,</span> <span class="n">na [...]
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">entity_filter</span><span class="p">:</span>
+            <span class="n">entity_filter</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">labels</span><span class="p">:</span>
+            <span class="n">labels</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;inputUrl&#39;</span><span class="p">:</span> <span class="n">input_url</span><span class="p">,</span>
+            <span class="s1">&#39;entityFilter&#39;</span><span class="p">:</span> <span class="n">entity_filter</span><span class="p">,</span>
+            <span class="s1">&#39;labels&#39;</span><span class="p">:</span> <span class="n">labels</span><span class="p">,</span>
+        <span class="p">}</span>
+        <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">admin_connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">import_</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n"> [...]
+        <span class="k">return</span> <span class="n">resp</span></div></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/emr_hook.html b/_modules/airflow/contrib/hooks/emr_hook.html
new file mode 100644
index 0000000..cc158a2
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/emr_hook.html
@@ -0,0 +1,295 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.emr_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.emr_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.emr_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
+
+
+<div class="viewcode-block" id="EmrHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.emr_hook.EmrHook">[docs]</a><span class="k">class</span> <span class="nc">EmrHook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Interact with AWS EMR. emr_conn_id is only neccessary for using the create_job_flow method.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">emr_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">emr_conn_id</span> <span class="o">=</span> <span class="n">emr_conn_id</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">EmrHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">&#39;emr&#39;</span><span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span>
+
+    <span class="k">def</span> <span class="nf">create_job_flow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">job_flow_overrides</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Creates a job flow using the config from the EMR connection.</span>
+<span class="sd">        Keys of the json extra hash may have the arguments of the boto3 run_job_flow method.</span>
+<span class="sd">        Overrides for this config may be passed as the job_flow_overrides.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">emr_conn_id</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">&#39;emr_conn_id must be present to use create_job_flow&#39;</span><span class="p">)</span>
+
+        <span class="n">emr_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">emr_conn_id</span><span class="p">)</span>
+
+        <span class="n">config</span> <span class="o">=</span> <span class="n">emr_conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+        <span class="n">config</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">job_flow_overrides</span><span class="p">)</span>
+
+        <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">run_job_flow</span><span class="p">(</span>
+            <span class="n">Name</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Name&#39;</span><span class="p">),</span>
+            <span class="n">LogUri</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;LogUri&#39;</span><span class="p">),</span>
+            <span class="n">ReleaseLabel</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;ReleaseLabel&#39;</span><span class="p">),</span>
+            <span class="n">Instances</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Instances&#39;</span><span class="p">),</span>
+            <span class="n">Steps</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Steps&#39;</span><span class="p">,</span> <span class="p">[]),</span>
+            <span class="n">BootstrapActions</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;BootstrapActions&#39;</span><span class="p">,</span> <span class="p">[]),</span>
+            <span class="n">Applications</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Applications&#39;</span><span class="p">),</span>
+            <span class="n">Configurations</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Configurations&#39;</span><span class="p">,</span> <span class="p">[]),</span>
+            <span class="n">VisibleToAllUsers</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;VisibleToAllUsers&#39;</span><span class="p">),</span>
+            <span class="n">JobFlowRole</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;JobFlowRole&#39;</span><span class="p">),</span>
+            <span class="n">ServiceRole</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;ServiceRole&#39;</span><span class="p">),</span>
+            <span class="n">Tags</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Tags&#39;</span><span class="p">)</span>
+        <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">response</span></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/gcp_dataflow_hook.html b/_modules/airflow/contrib/hooks/gcp_dataflow_hook.html
new file mode 100644
index 0000000..82e3ccd
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/gcp_dataflow_hook.html
@@ -0,0 +1,395 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.gcp_dataflow_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.gcp_dataflow_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.gcp_dataflow_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="kn">import</span> <span class="nn">select</span>
+<span class="kn">import</span> <span class="nn">subprocess</span>
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">import</span> <span class="nn">uuid</span>
+
+<span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="k">import</span> <span class="n">build</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<span class="k">class</span> <span class="nc">_DataflowJob</span><span class="p">(</span><span class="n">LoggingMixin</span><span class="p">):</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">,</span> <span class="n">project_number</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_dataflow</span> <span class="o">=</span> <span class="n">dataflow</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_project_number</span> <span class="o">=</span> <span class="n">project_number</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_job_name</span> <span class="o">=</span> <span class="n">name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_job</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_job</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">_get_job_id_from_name</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">jobs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dataflow</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span>
+            <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_project_number</span>
+        <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="k">for</span> <span class="n">job</span> <span class="ow">in</span> <span class="n">jobs</span><span class="p">[</span><span class="s1">&#39;jobs&#39;</span><span class="p">]:</span>
+            <span class="k">if</span> <span class="n">job</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job_name</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span> <span class="o">=</span> <span class="n">job</span><span class="p">[</span><span class="s1">&#39;id&#39;</span><span class="p">]</span>
+                <span class="k">return</span> <span class="n">job</span>
+        <span class="k">return</span> <span class="kc">None</span>
+
+    <span class="k">def</span> <span class="nf">_get_job</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">job</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_job_id_from_name</span><span class="p">()</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">job</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dataflow</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span  [...]
+                                                       <span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="k">if</span> <span class="s1">&#39;currentState&#39;</span> <span class="ow">in</span> <span class="n">job</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                <span class="s1">&#39;Google Cloud DataFlow job </span><span class="si">%s</span><span class="s1"> is </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+                <span class="n">job</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">],</span> <span class="n">job</span><span class="p">[</span><span class="s1">&#39;currentState&#39;</span><span class="p">]</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                <span class="s1">&#39;Google Cloud DataFlow with job_id </span><span class="si">%s</span><span class="s1"> has name </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span><span class="p">,</span> <span class="n">job</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]</span>
+            <span class="p">)</span>
+        <span class="k">return</span> <span class="n">job</span>
+
+    <span class="k">def</span> <span class="nf">wait_for_done</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+            <span class="k">if</span> <span class="s1">&#39;currentState&#39;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">:</span>
+                <span class="k">if</span> <span class="s1">&#39;JOB_STATE_DONE&#39;</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">[</span><span class="s1">&#39;currentState&#39;</span><span class="p">]:</span>
+                    <span class="k">return</span> <span class="kc">True</span>
+                <span class="k">elif</span> <span class="s1">&#39;JOB_STATE_FAILED&#39;</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">[</span><span class="s1">&#39;currentState&#39;</span><span class="p">]:</span>
+                    <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s2">&quot;Google Cloud Dataflow job </span><span class="si">{}</span><span class="s2"> has failed.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                        <span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]))</span>
+                <span class="k">elif</span> <span class="s1">&#39;JOB_STATE_CANCELLED&#39;</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">[</span><span class="s1">&#39;currentState&#39;</span><span class="p">]:</span>
+                    <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s2">&quot;Google Cloud Dataflow job </span><span class="si">{}</span><span class="s2"> was cancelled.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                        <span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]))</span>
+                <span class="k">elif</span> <span class="s1">&#39;JOB_STATE_RUNNING&#39;</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">[</span><span class="s1">&#39;currentState&#39;</span><span class="p">]:</span>
+                    <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span>
+                <span class="k">elif</span> <span class="s1">&#39;JOB_STATE_PENDING&#39;</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">[</span><span class="s1">&#39;currentState&#39;</span><span class="p">]:</span>
+                    <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">15</span><span class="p">)</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">))</span>
+                    <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
+                        <span class="s2">&quot;Google Cloud Dataflow job </span><span class="si">{}</span><span class="s2"> was unknown state: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                            <span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">[</span><span class="s1">&#39;currentState&#39;</span><span class="p">]))</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">15</span><span class="p">)</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">_job</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_job</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job</span>
+
+
+<span class="k">class</span> <span class="nc">_Dataflow</span><span class="p">(</span><span class="n">LoggingMixin</span><span class="p">):</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cmd</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span><span class="n">cmd</span><span class="p">,</span> <span class="n">shell</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><s [...]
+                                      <span class="n">stderr</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_line</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fd</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">fd</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stderr</span><span class="o">.</span><span class="n">fileno</span><span class="p">():</span>
+            <span class="n">line</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stderr</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
+            <span class="k">return</span> <span class="n">line</span>
+        <span class="k">if</span> <span class="n">fd</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">fileno</span><span class="p">():</span>
+            <span class="n">line</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
+            <span class="k">return</span> <span class="n">line</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_extract_job</span><span class="p">(</span><span class="n">line</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">line</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">line</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;Submitted job: &quot;</span><span class="p">):</span>
+                <span class="k">return</span> <span class="n">line</span><span class="p">[</span><span class="mi">15</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+
+    <span class="k">def</span> <span class="nf">wait_for_done</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">reads</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stderr</span><span class="o">.</span><span class="n">fileno</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">fileno</span><span c [...]
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Start waiting for DataFlow process to complete.&quot;</span><span class="p">)</span>
+        <span class="k">while</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">poll</span><span class="p">()</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">ret</span> <span class="o">=</span> <span class="n">select</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">reads</span><span class="p">,</span> <span class="p">[],</span> <span class="p">[],</span> <span class="mi">5</span><span class="p">)</span>
+            <span class="k">if</span> <span class="n">ret</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="k">for</span> <span class="n">fd</span> <span class="ow">in</span> <span class="n">ret</span><span class="p">[</span><span class="mi">0</span><span class="p">]:</span>
+                    <span class="n">line</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="p">(</span><span class="n">fd</span><span class="p">)</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">line</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Waiting for DataFlow process to complete.&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">returncode</span> <span class="ow">is</span> <span class="ow">not</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s2">&quot;DataFlow failed with return code </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">returncode</span><span class="p">))</span>
+
+
+<div class="viewcode-block" id="DataFlowHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_dataflow_hook.DataFlowHook">[docs]</a><span class="k">class</span> <span class="nc">DataFlowHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">):</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_default&#39;</span><span class="p">,</span>
+                 <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">DataFlowHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="p">)</span>
+
+<div class="viewcode-block" id="DataFlowHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_dataflow_hook.DataFlowHook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Returns a Google Cloud Storage service object.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">&#39;dataflow&#39;</span><span class="p">,</span> <span class="s1">&#39;v1b3&#39;</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">)</span></div>
+
+    <span class="k">def</span> <span class="nf">_start_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">command_prefix</span><span class="p">):</span>
+        <span class="n">cmd</span> <span class="o">=</span> <span class="n">command_prefix</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_cmd</span><span class="p">(</span><span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">)</span>
+        <span class="n">_Dataflow</span><span class="p">(</span><span class="n">cmd</span><span class="p">)</span><span class="o">.</span><span class="n">wait_for_done</span><span class="p">()</span>
+        <span class="n">_DataflowJob</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">(),</span> <span class="n">variables</span><span class="p">[</span><span class="s1">&#39;project&#39;</span><span class="p">],</span> <span class="n">name</span><span class="p">)</span><span class="o">.</span><span class="n">wait_for_done</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">start_java_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">):</span>
+        <span class="n">name</span> <span class="o">=</span> <span class="n">task_id</span> <span class="o">+</span> <span class="s2">&quot;-&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid1</span><span class="p">())[:</span><span class="mi">8</span><span class="p">]</span>
+        <span class="n">variables</span><span class="p">[</span><span class="s1">&#39;jobName&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_start_dataflow</span><span class="p">(</span>
+            <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;java&quot;</span><span class="p">,</span> <span class="s2">&quot;-jar&quot;</span><span class="p">])</span>
+
+    <span class="k">def</span> <span class="nf">start_python_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">,</span> <span class="n">py_options</span><span class="p">):</span>
+        <span class="n">name</span> <span class="o">=</span> <span class="n">task_id</span> <span class="o">+</span> <span class="s2">&quot;-&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid1</span><span class="p">())[:</span><span class="mi">8</span><span class="p">]</span>
+        <span class="n">variables</span><span class="p">[</span><span class="s2">&quot;job_name&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_start_dataflow</span><span class="p">(</span>
+            <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;python&quot;</span><span class="p">]</span> <span class="o">+</span> <span class="n">py_options</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_build_cmd</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">):</span>
+        <span class="n">command</span> <span class="o">=</span> <span class="p">[</span><span class="n">dataflow</span><span class="p">,</span> <span class="s2">&quot;--runner=DataflowRunner&quot;</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">variables</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">attr</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">variables</span><span class="o">.</span><span class="n">iteritems</span><span class="p">():</span>
+                <span class="n">command</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;--&quot;</span> <span class="o">+</span> <span class="n">attr</span> <span class="o">+</span> <span class="s2">&quot;=&quot;</span> <span class="o">+</span> <span class="n">value</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">command</span></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/gcp_mlengine_hook.html b/_modules/airflow/contrib/hooks/gcp_mlengine_hook.html
new file mode 100644
index 0000000..82450ae
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/gcp_mlengine_hook.html
@@ -0,0 +1,500 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.gcp_mlengine_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.gcp_mlengine_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.gcp_mlengine_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
+<span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
+<span class="c1"># this work for additional information regarding copyright ownership.</span>
+<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
+<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
+<span class="c1"># the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="kn">import</span> <span class="nn">random</span>
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">from</span> <span class="nn">apiclient</span> <span class="k">import</span> <span class="n">errors</span>
+<span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="k">import</span> <span class="n">build</span>
+<span class="kn">from</span> <span class="nn">oauth2client.client</span> <span class="k">import</span> <span class="n">GoogleCredentials</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<span class="k">def</span> <span class="nf">_poll_with_exponential_delay</span><span class="p">(</span><span class="n">request</span><span class="p">,</span> <span class="n">max_n</span><span class="p">,</span> <span class="n">is_done_func</span><span class="p">,</span> <span class="n">is_error_func</span><span class="p">):</span>
+    <span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
+
+    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">max_n</span><span class="p">):</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">response</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="k">if</span> <span class="n">is_error_func</span><span class="p">(</span><span class="n">response</span><span class="p">):</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                    <span class="s1">&#39;The response contained an error: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
+                <span class="p">)</span>
+            <span class="k">elif</span> <span class="n">is_done_func</span><span class="p">(</span><span class="n">response</span><span class="p">):</span>
+                <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Operation is done: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">response</span><span class="p">)</span>
+                <span class="k">return</span> <span class="n">response</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">((</span><span class="mi">2</span><span class="o">**</span><span class="n">i</span><span class="p">)</span> <span class="o">+</span> <span class="p">(</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1000</span><span class="p">)</span> <span class="o [...]
+        <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span> <span class="o">!=</span> <span class="mi">429</span><span class="p">:</span>
+                <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Something went wrong. Not retrying: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="nb">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
+                <span class="k">raise</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">((</span><span class="mi">2</span><span class="o">**</span><span class="n">i</span><span class="p">)</span> <span class="o">+</span> <span class="p">(</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1000</span><span class="p">)</span> <span class="o [...]
+
+
+<div class="viewcode-block" id="MLEngineHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_mlengine_hook.MLEngineHook">[docs]</a><span class="k">class</span> <span class="nc">MLEngineHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">):</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_default&#39;</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">MLEngineHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_mlengine</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+<div class="viewcode-block" id="MLEngineHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_mlengine_hook.MLEngineHook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Returns a Google MLEngine service object.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">credentials</span> <span class="o">=</span> <span class="n">GoogleCredentials</span><span class="o">.</span><span class="n">get_application_default</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">&#39;ml&#39;</span><span class="p">,</span> <span class="s1">&#39;v1&#39;</span><span class="p">,</span> <span class="n">credentials</span><span class="o">=</span><span class="n">credentials</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="MLEngineHook.create_job"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_mlengine_hook.MLEngineHook.create_job">[docs]</a>    <span class="k">def</span> <span class="nf">create_job</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="p">,</span> <span class="n">use_existing_job_fn</span><span c [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Launches a MLEngine job and wait for it to reach a terminal state.</span>
+
+<span class="sd">        :param project_id: The Google Cloud project id within which MLEngine</span>
+<span class="sd">            job will be launched.</span>
+<span class="sd">        :type project_id: string</span>
+
+<span class="sd">        :param job: MLEngine Job object that should be provided to the MLEngine</span>
+<span class="sd">            API, such as:</span>
+<span class="sd">            {</span>
+<span class="sd">              &#39;jobId&#39;: &#39;my_job_id&#39;,</span>
+<span class="sd">              &#39;trainingInput&#39;: {</span>
+<span class="sd">                &#39;scaleTier&#39;: &#39;STANDARD_1&#39;,</span>
+<span class="sd">                ...</span>
+<span class="sd">              }</span>
+<span class="sd">            }</span>
+<span class="sd">        :type job: dict</span>
+
+<span class="sd">        :param use_existing_job_fn: In case that a MLEngine job with the same</span>
+<span class="sd">            job_id already exist, this method (if provided) will decide whether</span>
+<span class="sd">            we should use this existing job, continue waiting for it to finish</span>
+<span class="sd">            and returning the job object. It should accepts a MLEngine job</span>
+<span class="sd">            object, and returns a boolean value indicating whether it is OK to</span>
+<span class="sd">            reuse the existing job. If &#39;use_existing_job_fn&#39; is not provided,</span>
+<span class="sd">            we by default reuse the existing MLEngine job.</span>
+<span class="sd">        :type use_existing_job_fn: function</span>
+
+<span class="sd">        :return: The MLEngine job object if the job successfully reach a</span>
+<span class="sd">            terminal state (which might be FAILED or CANCELLED state).</span>
+<span class="sd">        :rtype: dict</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">request</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mlengine</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
+            <span class="n">parent</span><span class="o">=</span><span class="s1">&#39;projects/</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">project_id</span><span class="p">),</span>
+            <span class="n">body</span><span class="o">=</span><span class="n">job</span><span class="p">)</span>
+        <span class="n">job_id</span> <span class="o">=</span> <span class="n">job</span><span class="p">[</span><span class="s1">&#39;jobId&#39;</span><span class="p">]</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">request</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="c1"># 409 means there is an existing job with the same job ID.</span>
+            <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span> <span class="o">==</span> <span class="mi">409</span><span class="p">:</span>
+                <span class="k">if</span> <span class="n">use_existing_job_fn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                    <span class="n">existing_job</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_job</span><span class="p">(</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job_id</span><span class="p">)</span>
+                    <span class="k">if</span> <span class="ow">not</span> <span class="n">use_existing_job_fn</span><span class="p">(</span><span class="n">existing_job</span><span class="p">):</span>
+                        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
+                            <span class="s1">&#39;Job with job_id </span><span class="si">%s</span><span class="s1"> already exist, but it does &#39;</span>
+                            <span class="s1">&#39;not match our expectation: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+                            <span class="n">job_id</span><span class="p">,</span> <span class="n">existing_job</span>
+                        <span class="p">)</span>
+                        <span class="k">raise</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                    <span class="s1">&#39;Job with job_id </span><span class="si">%s</span><span class="s1"> already exist. Will waiting for it to finish&#39;</span><span class="p">,</span>
+                    <span class="n">job_id</span>
+                <span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">&#39;Failed to create MLEngine job: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
+                <span class="k">raise</span>
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_job_done</span><span class="p">(</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job_id</span><span class="p">)</span></div>
+
+    <span class="k">def</span> <span class="nf">_get_job</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">job_id</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Gets a MLEngine job based on the job name.</span>
+
+<span class="sd">        :return: MLEngine job object if succeed.</span>
+<span class="sd">        :rtype: dict</span>
+
+<span class="sd">        Raises:</span>
+<span class="sd">            apiclient.errors.HttpError: if HTTP error is returned from server</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">job_name</span> <span class="o">=</span> <span class="s1">&#39;projects/</span><span class="si">{}</span><span class="s1">/jobs/</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job_id</span><span class="p">)</span>
+        <span class="n">request</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mlengine</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">job_name</span><span class="p">)</span>
+        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+            <span class="k">try</span><span class="p">:</span>
+                <span class="k">return</span> <span class="n">request</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+                <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span> <span class="o">==</span> <span class="mi">429</span><span class="p">:</span>
+                    <span class="c1"># polling after 30 seconds when quota failure occurs</span>
+                    <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">30</span><span class="p">)</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">&#39;Failed to get MLEngine job: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
+                    <span class="k">raise</span>
+
+    <span class="k">def</span> <span class="nf">_wait_for_job_done</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">job_id</span><span class="p">,</span> <span class="n">interval</span><span class="o">=</span><span class="mi">30</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Waits for the Job to reach a terminal state.</span>
+
+<span class="sd">        This method will periodically check the job state until the job reach</span>
+<span class="sd">        a terminal state.</span>
+
+<span class="sd">        Raises:</span>
+<span class="sd">            apiclient.errors.HttpError: if HTTP error is returned when getting</span>
+<span class="sd">            the job</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">assert</span> <span class="n">interval</span> <span class="o">&gt;</span> <span class="mi">0</span>
+        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+            <span class="n">job</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_job</span><span class="p">(</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job_id</span><span class="p">)</span>
+            <span class="k">if</span> <span class="n">job</span><span class="p">[</span><span class="s1">&#39;state&#39;</span><span class="p">]</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;SUCCEEDED&#39;</span><span class="p">,</span> <span class="s1">&#39;FAILED&#39;</span><span class="p">,</span> <span class="s1">&#39;CANCELLED&#39;</span><span class="p">]:</span>
+                <span class="k">return</span> <span class="n">job</span>
+            <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">interval</span><span class="p">)</span>
+
+<div class="viewcode-block" id="MLEngineHook.create_version"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_mlengine_hook.MLEngineHook.create_version">[docs]</a>    <span class="k">def</span> <span class="nf">create_version</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">model_name</span><span class="p">,</span> <span class="n">version_spec</ [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Creates the Version on Google Cloud ML Engine.</span>
+
+<span class="sd">        Returns the operation if the version was created successfully and</span>
+<span class="sd">        raises an error otherwise.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">parent_name</span> <span class="o">=</span> <span class="s1">&#39;projects/</span><span class="si">{}</span><span class="s1">/models/</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">project_id</span><span class="p">,</span> <span class="n">model_name</span><span class="p">)</span>
+        <span class="n">create_request</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mlengine</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">models</span><span class="p">()</span><span class="o">.</span><span class="n">versions</span><span class="p">()</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
+            <span class="n">parent</span><span class="o">=</span><span class="n">parent_name</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">version_spec</span><span class="p">)</span>
+        <span class="n">response</span> <span class="o">=</span> <span class="n">create_request</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="n">get_request</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mlengine</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">operations</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+            <span class="n">name</span><span class="o">=</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">])</span>
+
+        <span class="k">return</span> <span class="n">_poll_with_exponential_delay</span><span class="p">(</span>
+            <span class="n">request</span><span class="o">=</span><span class="n">get_request</span><span class="p">,</span>
+            <span class="n">max_n</span><span class="o">=</span><span class="mi">9</span><span class="p">,</span>
+            <span class="n">is_done_func</span><span class="o">=</span><span class="k">lambda</span> <span class="n">resp</span><span class="p">:</span> <span class="n">resp</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;done&#39;</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+            <span class="n">is_error_func</span><span class="o">=</span><span class="k">lambda</span> <span class="n">resp</span><span class="p">:</span> <span class="n">resp</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;error&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="MLEngineHook.set_default_version"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_mlengine_hook.MLEngineHook.set_default_version">[docs]</a>    <span class="k">def</span> <span class="nf">set_default_version</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">model_name</span><span class="p">,</span> <span class="n" [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Sets a version to be the default. Blocks until finished.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">full_version_name</span> <span class="o">=</span> <span class="s1">&#39;projects/</span><span class="si">{}</span><span class="s1">/models/</span><span class="si">{}</span><span class="s1">/versions/</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+            <span class="n">project_id</span><span class="p">,</span> <span class="n">model_name</span><span class="p">,</span> <span class="n">version_name</span><span class="p">)</span>
+        <span class="n">request</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mlengine</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">models</span><span class="p">()</span><span class="o">.</span><span class="n">versions</span><span class="p">()</span><span class="o">.</span><span class="n">setDefault</span><span class="p">(</span>
+            <span class="n">name</span><span class="o">=</span><span class="n">full_version_name</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="p">{})</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">response</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Successfully set version: </span><span class="si">%s</span><span class="s1"> to default&#39;</span><span class="p">,</span> <span class="n">response</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">response</span>
+        <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">&#39;Something went wrong: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
+            <span class="k">raise</span></div>
+
+<div class="viewcode-block" id="MLEngineHook.list_versions"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_mlengine_hook.MLEngineHook.list_versions">[docs]</a>    <span class="k">def</span> <span class="nf">list_versions</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">model_name</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Lists all available versions of a model. Blocks until finished.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">result</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="n">full_parent_name</span> <span class="o">=</span> <span class="s1">&#39;projects/</span><span class="si">{}</span><span class="s1">/models/</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+            <span class="n">project_id</span><span class="p">,</span> <span class="n">model_name</span><span class="p">)</span>
+        <span class="n">request</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mlengine</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">models</span><span class="p">()</span><span class="o">.</span><span class="n">versions</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span>
+            <span class="n">parent</span><span class="o">=</span><span class="n">full_parent_name</span><span class="p">,</span> <span class="n">pageSize</span><span class="o">=</span><span class="mi">100</span><span class="p">)</span>
+
+        <span class="n">response</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="n">next_page_token</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;nextPageToken&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+        <span class="n">result</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;versions&#39;</span><span class="p">,</span> <span class="p">[]))</span>
+        <span class="k">while</span> <span class="n">next_page_token</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">next_request</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mlengine</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">models</span><span class="p">()</span><span class="o">.</span><span class="n">versions</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span>
+                <span class="n">parent</span><span class="o">=</span><span class="n">full_parent_name</span><span class="p">,</span>
+                <span class="n">pageToken</span><span class="o">=</span><span class="n">next_page_token</span><span class="p">,</span>
+                <span class="n">pageSize</span><span class="o">=</span><span class="mi">100</span><span class="p">)</span>
+            <span class="n">response</span> <span class="o">=</span> <span class="n">next_request</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="n">next_page_token</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;nextPageToken&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+            <span class="n">result</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;versions&#39;</span><span class="p">,</span> <span class="p">[]))</span>
+            <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">result</span></div>
+
+<div class="viewcode-block" id="MLEngineHook.delete_version"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_mlengine_hook.MLEngineHook.delete_version">[docs]</a>    <span class="k">def</span> <span class="nf">delete_version</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">model_name</span><span class="p">,</span> <span class="n">version_name</ [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Deletes the given version of a model. Blocks until finished.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">full_name</span> <span class="o">=</span> <span class="s1">&#39;projects/</span><span class="si">{}</span><span class="s1">/models/</span><span class="si">{}</span><span class="s1">/versions/</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+            <span class="n">project_id</span><span class="p">,</span> <span class="n">model_name</span><span class="p">,</span> <span class="n">version_name</span><span class="p">)</span>
+        <span class="n">delete_request</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mlengine</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">models</span><span class="p">()</span><span class="o">.</span><span class="n">versions</span><span class="p">()</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span>
+            <span class="n">name</span><span class="o">=</span><span class="n">full_name</span><span class="p">)</span>
+        <span class="n">response</span> <span class="o">=</span> <span class="n">delete_request</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="n">get_request</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mlengine</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">operations</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+            <span class="n">name</span><span class="o">=</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">])</span>
+
+        <span class="k">return</span> <span class="n">_poll_with_exponential_delay</span><span class="p">(</span>
+            <span class="n">request</span><span class="o">=</span><span class="n">get_request</span><span class="p">,</span>
+            <span class="n">max_n</span><span class="o">=</span><span class="mi">9</span><span class="p">,</span>
+            <span class="n">is_done_func</span><span class="o">=</span><span class="k">lambda</span> <span class="n">resp</span><span class="p">:</span> <span class="n">resp</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;done&#39;</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+            <span class="n">is_error_func</span><span class="o">=</span><span class="k">lambda</span> <span class="n">resp</span><span class="p">:</span> <span class="n">resp</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;error&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="MLEngineHook.create_model"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_mlengine_hook.MLEngineHook.create_model">[docs]</a>    <span class="k">def</span> <span class="nf">create_model</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">model</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Create a Model. Blocks until finished.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">assert</span> <span class="n">model</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">model</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="s1">&#39;&#39;</span>
+        <span class="n">project</span> <span class="o">=</span> <span class="s1">&#39;projects/</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">project_id</span><span class="p">)</span>
+
+        <span class="n">request</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mlengine</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">models</span><span class="p">()</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
+            <span class="n">parent</span><span class="o">=</span><span class="n">project</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">model</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">request</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="MLEngineHook.get_model"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_mlengine_hook.MLEngineHook.get_model">[docs]</a>    <span class="k">def</span> <span class="nf">get_model</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">model_name</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Gets a Model. Blocks until finished.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">assert</span> <span class="n">model_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">model_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="s1">&#39;&#39;</span>
+        <span class="n">full_model_name</span> <span class="o">=</span> <span class="s1">&#39;projects/</span><span class="si">{}</span><span class="s1">/models/</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+            <span class="n">project_id</span><span class="p">,</span> <span class="n">model_name</span><span class="p">)</span>
+        <span class="n">request</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mlengine</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">models</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">full_model_name</span><span class="p">)</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">request</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span> <span class="o">==</span> <span class="mi">404</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">&#39;Model was not found: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
+                <span class="k">return</span> <span class="kc">None</span>
+            <span class="k">raise</span></div></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/gcs_hook.html b/_modules/airflow/contrib/hooks/gcs_hook.html
index 8edc584..6f3bc49 100644
--- a/_modules/airflow/contrib/hooks/gcs_hook.html
+++ b/_modules/airflow/contrib/hooks/gcs_hook.html
@@ -13,6 +13,8 @@
 
   
   
+  
+  
 
   
 
@@ -30,6 +32,9 @@
   
 
   
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
     <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
         <link rel="up" title="Module code" href="../../../index.html"/> 
 
@@ -40,6 +45,7 @@
 
 <body class="wy-body-for-nav" role="document">
 
+   
   <div class="wy-grid-for-nav">
 
     
@@ -76,7 +82,10 @@
           
             
             
-                <ul>
+              
+            
+            
+              <ul>
 <li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
@@ -90,6 +99,8 @@
 <li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
 </ul>
@@ -104,8 +115,10 @@
 
       
       <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
-        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
-        <a href="../../../../index.html">Airflow</a>
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
       </nav>
 
 
@@ -118,19 +131,36 @@
 
 
 
+
+
+
+
+
+
+
+
+
+
 <div role="navigation" aria-label="breadcrumbs navigation">
+
   <ul class="wy-breadcrumbs">
-    <li><a href="../../../../index.html">Docs</a> &raquo;</li>
-      
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
           <li><a href="../../../index.html">Module code</a> &raquo;</li>
-      
-    <li>airflow.contrib.hooks.gcs_hook</li>
+        
+      <li>airflow.contrib.hooks.gcs_hook</li>
+    
+    
       <li class="wy-breadcrumbs-aside">
         
-          
+            
         
       </li>
+    
   </ul>
+
+  
   <hr/>
 </div>
           <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -151,17 +181,14 @@
 <span class="c1"># See the License for the specific language governing permissions and</span>
 <span class="c1"># limitations under the License.</span>
 <span class="c1">#</span>
+<span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="k">import</span> <span class="n">build</span>
+<span class="kn">from</span> <span class="nn">apiclient.http</span> <span class="k">import</span> <span class="n">MediaFileUpload</span>
+<span class="kn">from</span> <span class="nn">googleapiclient</span> <span class="k">import</span> <span class="n">errors</span>
 
-<span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="kn">import</span> <span class="n">GoogleCloudBaseHook</span>
-<span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="kn">import</span> <span class="n">build</span>
-<span class="kn">from</span> <span class="nn">apiclient.http</span> <span class="kn">import</span> <span class="n">MediaFileUpload</span>
-
-<span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;google_cloud_storage&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">)</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
 
 
-<div class="viewcode-block" id="GoogleCloudStorageHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStorageHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">):</span>
+<div class="viewcode-block" id="GoogleCloudStorageHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStorageHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Interact with Google Cloud Storage. This hook uses the Google Cloud Platform</span>
 <span class="sd">    connection.</span>
@@ -169,17 +196,65 @@
 
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                  <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_storage_default&#39;</span><span class="p">,</span>
-                 <span class="n">delegate_to</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">GoogleCloudStorageHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="p">)</span>
+                 <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">GoogleCloudStorageHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
+                                                     <span class="n">delegate_to</span><span class="p">)</span>
 
-    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<div class="viewcode-block" id="GoogleCloudStorageHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Returns a Google Cloud Storage service object.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
-        <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">&#39;storage&#39;</span><span class="p">,</span> <span class="s1">&#39;v1&#39;</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">&#39;storage&#39;</span><span class="p">,</span> <span class="s1">&#39;v1&#39;</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">)</span></div>
+
+
+    <span class="c1"># pylint:disable=redefined-builtin</span>
+<div class="viewcode-block" id="GoogleCloudStorageHook.copy"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.copy">[docs]</a>    <span class="k">def</span> <span class="nf">copy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">source_bucket</span><span class="p">,</span> <span class="n">source_object</span><span class="p">,</span> <span class="n">destination_bucket</span><s [...]
+             <span class="n">destination_object</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Copies an object from a bucket to another, with renaming if requested.</span>
+
+<span class="sd">        destination_bucket or destination_object can be omitted, in which case</span>
+<span class="sd">        source bucket/object is used, but not both.</span>
+
+<span class="sd">        :param bucket: The bucket of the object to copy from.</span>
+<span class="sd">        :type bucket: string</span>
+<span class="sd">        :param object: The object to copy.</span>
+<span class="sd">        :type object: string</span>
+<span class="sd">        :param destination_bucket: The destination of the object to copied to.</span>
+<span class="sd">            Can be omitted; then the same bucket is used.</span>
+<span class="sd">        :type destination_bucket: string</span>
+<span class="sd">        :param destination_object: The (renamed) path of the object if given.</span>
+<span class="sd">            Can be omitted; then the same name is used.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">destination_bucket</span> <span class="o">=</span> <span class="n">destination_bucket</span> <span class="ow">or</span> <span class="n">source_bucket</span>
+        <span class="n">destination_object</span> <span class="o">=</span> <span class="n">destination_object</span> <span class="ow">or</span> <span class="n">source_object</span>
+        <span class="k">if</span> <span class="p">(</span><span class="n">source_bucket</span> <span class="o">==</span> <span class="n">destination_bucket</span> <span class="ow">and</span>
+            <span class="n">source_object</span> <span class="o">==</span> <span class="n">destination_object</span><span class="p">):</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s1">&#39;Either source/destination bucket or source/destination object &#39;</span>
+                <span class="s1">&#39;must be different, not both the same: bucket=</span><span class="si">%s</span><span class="s1">, object=</span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span>
+                <span class="p">(</span><span class="n">source_bucket</span><span class="p">,</span> <span class="n">source_object</span><span class="p">))</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">source_bucket</span> <span class="ow">or</span> <span class="ow">not</span> <span class="n">source_object</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;source_bucket and source_object cannot be empty.&#39;</span><span class="p">)</span>
 
-    <span class="k">def</span> <span class="nf">download</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">filename</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
+        <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">service</span> \
+                <span class="o">.</span><span class="n">objects</span><span class="p">()</span> \
+                <span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">sourceBucket</span><span class="o">=</span><span class="n">source_bucket</span><span class="p">,</span> <span class="n">sourceObject</span><span class="o">=</span><span class="n">source_object</span><span class="p">,</span>
+                      <span class="n">destinationBucket</span><span class="o">=</span><span class="n">destination_bucket</span><span class="p">,</span>
+                      <span class="n">destinationObject</span><span class="o">=</span><span class="n">destination_object</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">)</span> \
+                <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="k">return</span> <span class="kc">True</span>
+        <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">ex</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">ex</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;404&#39;</span><span class="p">:</span>
+                <span class="k">return</span> <span class="kc">False</span>
+            <span class="k">raise</span></div>
+
+
+    <span class="c1"># pylint:disable=redefined-builtin</span>
+<div class="viewcode-block" id="GoogleCloudStorageHook.download"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.download">[docs]</a>    <span class="k">def</span> <span class="nf">download</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">filename</span><span class=" [...]
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Get a file from Google Cloud Storage.</span>
 
@@ -198,12 +273,14 @@
 
         <span class="c1"># Write the file to local file path, if requested.</span>
         <span class="k">if</span> <span class="n">filename</span><span class="p">:</span>
-            <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s1">&#39;w&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">file_fd</span><span class="p">:</span>
+            <span class="n">write_argument</span> <span class="o">=</span> <span class="s1">&#39;wb&#39;</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">downloaded_file_bytes</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">)</span> <span class="k">else</span> <span class="s1">&#39;w&#39;</span>
+            <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">write_argument</span><span class="p">)</span> <span class="k">as</span> <span class="n">file_fd</span><span class="p">:</span>
                 <span class="n">file_fd</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">downloaded_file_bytes</span><span class="p">)</span>
 
-        <span class="k">return</span> <span class="n">downloaded_file_bytes</span>
+        <span class="k">return</span> <span class="n">downloaded_file_bytes</span></div>
 
-    <span class="k">def</span> <span class="nf">upload</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="n">mime_type</span><span class="o">=</span><span class="s1">&#39;application/octet-stream&#39;</span><span class="p">):</span>
+    <span class="c1"># pylint:disable=redefined-builtin</span>
+<div class="viewcode-block" id="GoogleCloudStorageHook.upload"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.upload">[docs]</a>    <span class="k">def</span> <span class="nf">upload</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</ [...]
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Uploads a local file to Google Cloud Storage.</span>
 
@@ -222,9 +299,145 @@
             <span class="o">.</span><span class="n">objects</span><span class="p">()</span> \
             <span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">bucket</span><span class="o">=</span><span class="n">bucket</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="nb">object</span><span class="p">,</span> <span class="n">media_body</span><span class="o">=</span><span class="n">media</span><span class="p">)</span> \
             <span class="o">.</span><span class="n">execute</span><span class="p">()</span></div>
+
+    <span class="c1"># pylint:disable=redefined-builtin</span>
+<div class="viewcode-block" id="GoogleCloudStorageHook.exists"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.exists">[docs]</a>    <span class="k">def</span> <span class="nf">exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Checks for the existence of a file in Google Cloud Storage.</span>
+
+<span class="sd">        :param bucket: The Google cloud storage bucket where the object is.</span>
+<span class="sd">        :type bucket: string</span>
+<span class="sd">        :param object: The name of the object to check in the Google cloud</span>
+<span class="sd">            storage bucket.</span>
+<span class="sd">        :type object: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">service</span> \
+                <span class="o">.</span><span class="n">objects</span><span class="p">()</span> \
+                <span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">bucket</span><span class="o">=</span><span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="o">=</span><span class="nb">object</span><span class="p">)</span> \
+                <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="k">return</span> <span class="kc">True</span>
+        <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">ex</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">ex</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;404&#39;</span><span class="p">:</span>
+                <span class="k">return</span> <span class="kc">False</span>
+            <span class="k">raise</span></div>
+
+    <span class="c1"># pylint:disable=redefined-builtin</span>
+<div class="viewcode-block" id="GoogleCloudStorageHook.is_updated_after"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.is_updated_after">[docs]</a>    <span class="k">def</span> <span class="nf">is_updated_after</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">ts</ [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Checks if an object is updated in Google Cloud Storage.</span>
+
+<span class="sd">        :param bucket: The Google cloud storage bucket where the object is.</span>
+<span class="sd">        :type bucket: string</span>
+<span class="sd">        :param object: The name of the object to check in the Google cloud</span>
+<span class="sd">            storage bucket.</span>
+<span class="sd">        :type object: string</span>
+<span class="sd">        :param ts: The timestamp to check against.</span>
+<span class="sd">        :type ts: datetime</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">response</span> <span class="o">=</span> <span class="p">(</span><span class="n">service</span>
+                        <span class="o">.</span><span class="n">objects</span><span class="p">()</span>
+                        <span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">bucket</span><span class="o">=</span><span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="o">=</span><span class="nb">object</span><span class="p">)</span>
+                        <span class="o">.</span><span class="n">execute</span><span class="p">())</span>
+
+            <span class="k">if</span> <span class="s1">&#39;updated&#39;</span> <span class="ow">in</span> <span class="n">response</span><span class="p">:</span>
+                <span class="kn">import</span> <span class="nn">dateutil.parser</span>
+                <span class="kn">import</span> <span class="nn">dateutil.tz</span>
+
+                <span class="k">if</span> <span class="ow">not</span> <span class="n">ts</span><span class="o">.</span><span class="n">tzinfo</span><span class="p">:</span>
+                    <span class="n">ts</span> <span class="o">=</span> <span class="n">ts</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">tzinfo</span><span class="o">=</span><span class="n">dateutil</span><span class="o">.</span><span class="n">tz</span><span class="o">.</span><span class="n">tzutc</span><span class="p">())</span>
+
+                <span class="n">updated</span> <span class="o">=</span> <span class="n">dateutil</span><span class="o">.</span><span class="n">parser</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;updated&#39;</span><span class="p">])</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Verify object date: </span><span class="si">%s</span><span class="s2"> &gt; </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">updated</span><span class="p">,</span> <span class="n">ts</span><span class="p">)</span>
+
+                <span class="k">if</span> <span class="n">updated</span> <span class="o">&gt;</span> <span class="n">ts</span><span class="p">:</span>
+                    <span class="k">return</span> <span class="kc">True</span>
+
+        <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">ex</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">ex</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">&#39;404&#39;</span><span class="p">:</span>
+                <span class="k">raise</span>
+
+        <span class="k">return</span> <span class="kc">False</span></div>
+
+<div class="viewcode-block" id="GoogleCloudStorageHook.delete"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.delete">[docs]</a>    <span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">generation</span><span class="o">= [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Delete an object if versioning is not enabled for the bucket, or if generation</span>
+<span class="sd">        parameter is used.</span>
+
+<span class="sd">        :param bucket: name of the bucket, where the object resides</span>
+<span class="sd">        :type bucket: string</span>
+<span class="sd">        :param object: name of the object to delete</span>
+<span class="sd">        :type object: string</span>
+<span class="sd">        :param generation: if present, permanently delete the object of this generation</span>
+<span class="sd">        :type generation: string</span>
+<span class="sd">        :return: True if succeeded</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">service</span> \
+                <span class="o">.</span><span class="n">objects</span><span class="p">()</span> \
+                <span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">bucket</span><span class="o">=</span><span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="o">=</span><span class="nb">object</span><span class="p">,</span> <span class="n">generation</span><span class="o">=</span><span class="n">generation</span><span class="p">)</span> \
+                <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="k">return</span> <span class="kc">True</span>
+        <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">ex</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">ex</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;404&#39;</span><span class="p">:</span>
+                <span class="k">return</span> <span class="kc">False</span>
+            <span class="k">raise</span></div>
+
+<div class="viewcode-block" id="GoogleCloudStorageHook.list"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.list">[docs]</a>    <span class="k">def</span> <span class="nf">list</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">versions</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <sp [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        List all objects from the bucket with the give string prefix in name</span>
+
+<span class="sd">        :param bucket: bucket name</span>
+<span class="sd">        :type bucket: string</span>
+<span class="sd">        :param versions: if true, list all versions of the objects</span>
+<span class="sd">        :type versions: boolean</span>
+<span class="sd">        :param maxResults: max count of items to return in a single page of responses</span>
+<span class="sd">        :type maxResults: integer</span>
+<span class="sd">        :param prefix: prefix string which filters objects whose name begin with this prefix</span>
+<span class="sd">        :type prefix: string</span>
+<span class="sd">        :return: a stream of object names matching the filtering criteria</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+        <span class="n">ids</span> <span class="o">=</span> <span class="nb">list</span><span class="p">()</span>
+        <span class="n">pageToken</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="k">while</span><span class="p">(</span><span class="kc">True</span><span class="p">):</span>
+            <span class="n">response</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">objects</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span>
+                <span class="n">bucket</span><span class="o">=</span><span class="n">bucket</span><span class="p">,</span>
+                <span class="n">versions</span><span class="o">=</span><span class="n">versions</span><span class="p">,</span>
+                <span class="n">maxResults</span><span class="o">=</span><span class="n">maxResults</span><span class="p">,</span>
+                <span class="n">pageToken</span><span class="o">=</span><span class="n">pageToken</span><span class="p">,</span>
+                <span class="n">prefix</span><span class="o">=</span><span class="n">prefix</span>
+            <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+            <span class="k">if</span> <span class="s1">&#39;items&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">response</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;No items found for prefix: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">prefix</span><span class="p">)</span>
+                <span class="k">break</span>
+
+            <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;items&#39;</span><span class="p">]:</span>
+                <span class="k">if</span> <span class="n">item</span> <span class="ow">and</span> <span class="s1">&#39;name&#39;</span> <span class="ow">in</span> <span class="n">item</span><span class="p">:</span>
+                    <span class="n">ids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">])</span>
+
+            <span class="k">if</span> <span class="s1">&#39;nextPageToken&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">response</span><span class="p">:</span>
+                <span class="c1"># no further pages of results, so stop the loop</span>
+                <span class="k">break</span>
+
+            <span class="n">pageToken</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;nextPageToken&#39;</span><span class="p">]</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="n">pageToken</span><span class="p">:</span>
+                <span class="c1"># empty next page token</span>
+                <span class="k">break</span>
+        <span class="k">return</span> <span class="n">ids</span></div></div>
 </pre></div>
 
            </div>
+           <div class="articleComments">
+            
+           </div>
           </div>
           <footer>
   
@@ -257,7 +470,8 @@
             VERSION:'',
             COLLAPSE_INDEX:false,
             FILE_SUFFIX:'.html',
-            HAS_SOURCE:  true
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
         };
     </script>
       <script type="text/javascript" src="../../../../_static/jquery.js"></script>
diff --git a/_modules/airflow/contrib/hooks/wasb_hook.html b/_modules/airflow/contrib/hooks/wasb_hook.html
new file mode 100644
index 0000000..2ff6ba4
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/wasb_hook.html
@@ -0,0 +1,345 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.wasb_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.wasb_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.wasb_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+
+<span class="kn">from</span> <span class="nn">azure.storage.blob</span> <span class="k">import</span> <span class="n">BlockBlobService</span>
+
+
+<div class="viewcode-block" id="WasbHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.wasb_hook.WasbHook">[docs]</a><span class="k">class</span> <span class="nc">WasbHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Interacts with Azure Blob Storage through the wasb:// protocol.</span>
+<span class="sd">    </span>
+<span class="sd">    Additional options passed in the &#39;extra&#39; field of the connection will be</span>
+<span class="sd">    passed to the `BlockBlockService()` constructor. For example, authenticate</span>
+<span class="sd">    using a SAS token by adding {&quot;sas_token&quot;: &quot;YOUR_TOKEN&quot;}.</span>
+<span class="sd">    </span>
+<span class="sd">    :param wasb_conn_id: Reference to the wasb connection.</span>
+<span class="sd">    :type wasb_conn_id: str</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">wasb_conn_id</span><span class="o">=</span><span class="s1">&#39;wasb_default&#39;</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">wasb_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;Return the BlockBlobService object.&quot;&quot;&quot;</span>
+        <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+        <span class="n">service_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+        <span class="k">return</span> <span class="n">BlockBlobService</span><span class="p">(</span><span class="n">account_name</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
+                                <span class="n">account_key</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> <span class="o">**</span><span class="n">service_options</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">check_for_blob</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Check if a blob exists on Azure Blob Storage.</span>
+<span class="sd">        </span>
+<span class="sd">        :param container_name: Name of the container.</span>
+<span class="sd">        :type container_name: str</span>
+<span class="sd">        :param blob_name: Name of the blob.</span>
+<span class="sd">        :type blob_name: str</span>
+<span class="sd">        :param kwargs: Optional keyword arguments that</span>
+<span class="sd">            `BlockBlobService.exists()` takes.</span>
+<span class="sd">        :type kwargs: object</span>
+<span class="sd">        :return: True if the blob exists, False otherwise.</span>
+<span class="sd">        :rtype bool</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">check_for_prefix</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">container_name</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Check if a prefix exists on Azure Blob storage.</span>
+<span class="sd">        </span>
+<span class="sd">        :param container_name: Name of the container.</span>
+<span class="sd">        :type container_name: str</span>
+<span class="sd">        :param prefix: Prefix of the blob.</span>
+<span class="sd">        :type prefix: str</span>
+<span class="sd">        :param kwargs: Optional keyword arguments that</span>
+<span class="sd">            `BlockBlobService.list_blobs()` takes.</span>
+<span class="sd">        :type kwargs: object</span>
+<span class="sd">        :return: True if blobs matching the prefix exist, False otherwise.</span>
+<span class="sd">        :rtype bool</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">matches</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">list_blobs</span><span class="p">(</span><span class="n">container_name</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span>
+                                             <span class="n">num_results</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">matches</span><span class="p">))</span> <span class="o">&gt;</span> <span class="mi">0</span>
+
+    <span class="k">def</span> <span class="nf">load_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_path</span><span class="p">,</span> <span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Upload a file to Azure Blob Storage.</span>
+<span class="sd">        </span>
+<span class="sd">        :param file_path: Path to the file to load.</span>
+<span class="sd">        :type file_path: str</span>
+<span class="sd">        :param container_name: Name of the container.</span>
+<span class="sd">        :type container_name: str</span>
+<span class="sd">        :param blob_name: Name of the blob.</span>
+<span class="sd">        :type blob_name: str</span>
+<span class="sd">        :param kwargs: Optional keyword arguments that</span>
+<span class="sd">            `BlockBlobService.create_blob_from_path()` takes.</span>
+<span class="sd">        :type kwargs: object</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="c1"># Reorder the argument order from airflow.hooks.S3_hook.load_file.</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">create_blob_from_path</span><span class="p">(</span><span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span>
+                                              <span class="n">file_path</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">load_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">string_data</span><span class="p">,</span> <span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Upload a string to Azure Blob Storage.</span>
+<span class="sd">        </span>
+<span class="sd">        :param string_data: String to load.</span>
+<span class="sd">        :type string_data: str</span>
+<span class="sd">        :param container_name: Name of the container.</span>
+<span class="sd">        :type container_name: str</span>
+<span class="sd">        :param blob_name: Name of the blob.</span>
+<span class="sd">        :type blob_name: str</span>
+<span class="sd">        :param kwargs: Optional keyword arguments that</span>
+<span class="sd">            `BlockBlobService.create_blob_from_text()` takes.</span>
+<span class="sd">        :type kwargs: object</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="c1"># Reorder the argument order from airflow.hooks.S3_hook.load_string.</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">create_blob_from_text</span><span class="p">(</span><span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span>
+                                              <span class="n">string_data</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/presto_check_operator.html b/_modules/airflow/contrib/operators/bigquery_check_operator.html
similarity index 57%
copy from _modules/presto_check_operator.html
copy to _modules/airflow/contrib/operators/bigquery_check_operator.html
index 325c884..cbad19f 100644
--- a/_modules/presto_check_operator.html
+++ b/_modules/airflow/contrib/operators/bigquery_check_operator.html
@@ -8,11 +8,13 @@
   
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   
-  <title>presto_check_operator &mdash; Airflow Documentation</title>
+  <title>airflow.contrib.operators.bigquery_check_operator &mdash; Airflow Documentation</title>
   
 
   
   
+  
+  
 
   
 
@@ -24,20 +26,20 @@
 
   
   
-    <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
   
 
   
 
   
         <link rel="index" title="Index"
-              href="../genindex.html"/>
-        <link rel="search" title="Search" href="../search.html"/>
-    <link rel="top" title="Airflow Documentation" href="../index.html"/>
-        <link rel="up" title="Module code" href="index.html"/> 
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
 
   
-  <script src="../_static/js/modernizr.min.js"></script>
+  <script src="../../../../_static/js/modernizr.min.js"></script>
 
 </head>
 
@@ -53,7 +55,7 @@
           
 
           
-            <a href="../index.html" class="icon icon-home"> Airflow
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
           
 
           
@@ -66,7 +68,7 @@
 
           
 <div role="search">
-  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
     <input type="text" name="q" placeholder="Search docs" />
     <input type="hidden" name="check_keywords" value="yes" />
     <input type="hidden" name="area" value="default" />
@@ -80,24 +82,27 @@
           
             
             
-                <ul>
-<li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../configuration.html">Configuration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../ui.html">UI / Screenshots</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../concepts.html">Concepts</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../profiling.html">Data Profiling</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../cli.html">Command Line Interface</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling &amp; Triggers</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../api.html">Experimental Rest API</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li>
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
 </ul>
 
             
@@ -112,7 +117,7 @@
       <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
         
           <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
-          <a href="../index.html">Airflow</a>
+          <a href="../../../../index.html">Airflow</a>
         
       </nav>
 
@@ -140,11 +145,11 @@
 
   <ul class="wy-breadcrumbs">
     
-      <li><a href="../index.html">Docs</a> &raquo;</li>
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
         
-          <li><a href="index.html">Module code</a> &raquo;</li>
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
         
-      <li>presto_check_operator</li>
+      <li>airflow.contrib.operators.bigquery_check_operator</li>
     
     
       <li class="wy-breadcrumbs-aside">
@@ -161,7 +166,7 @@
           <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
            <div itemprop="articleBody">
             
-  <h1>Source code for presto_check_operator</h1><div class="highlight"><pre>
+  <h1>Source code for airflow.contrib.operators.bigquery_check_operator</h1><div class="highlight"><pre>
 <span></span><span class="c1"># -*- coding: utf-8 -*-</span>
 <span class="c1">#</span>
 <span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
@@ -176,14 +181,14 @@
 <span class="c1"># See the License for the specific language governing permissions and</span>
 <span class="c1"># limitations under the License.</span>
 
-<span class="kn">from</span> <span class="nn">airflow.hooks.presto_hook</span> <span class="k">import</span> <span class="n">PrestoHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="k">import</span> <span class="n">BigQueryHook</span>
 <span class="kn">from</span> <span class="nn">airflow.operators.check_operator</span> <span class="k">import</span> <span class="n">CheckOperator</span><span class="p">,</span> <span class="n">ValueCheckOperator</span><span class="p">,</span> <span class="n">IntervalCheckOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
 
 
-<div class="viewcode-block" id="PrestoCheckOperator"><a class="viewcode-back" href="../code.html#airflow.operators.PrestoCheckOperator">[docs]</a><span class="k">class</span> <span class="nc">PrestoCheckOperator</span><span class="p">(</span><span class="n">CheckOperator</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryCheckOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_check_operator.BigQueryCheckOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryCheckOperator</span><span class="p">(</span><span class="n">CheckOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Performs checks against Presto. The ``PrestoCheckOperator`` expects</span>
+<span class="sd">    Performs checks against BigQuery. The ``BigQueryCheckOperator`` expects</span>
 <span class="sd">    a sql query that will return a single row. Each value on that</span>
 <span class="sd">    first row is evaluated using python ``bool`` casting. If any of the</span>
 <span class="sd">    values return ``False`` the check is failed and errors out.</span>
@@ -211,78 +216,82 @@
 
 <span class="sd">    :param sql: the sql to be executed</span>
 <span class="sd">    :type sql: string</span>
-<span class="sd">    :param presto_conn_id: reference to the Presto database</span>
-<span class="sd">    :type presto_conn_id: string</span>
+<span class="sd">    :param bigquery_conn_id: reference to the BigQuery database</span>
+<span class="sd">    :type bigquery_conn_id: string</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
-            <span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span>
-            <span class="n">presto_conn_id</span><span class="o">=</span><span class="s1">&#39;presto_default&#39;</span><span class="p">,</span>
-            <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">PrestoCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">sql</span><span class="o">=</span><span class="n">sql</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><s [...]
-
-        <span class="bp">self</span><span class="o">.</span><span class="n">presto_conn_id</span> <span class="o">=</span> <span class="n">presto_conn_id</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">sql</span><span class="p">,</span>
+            <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">&#39;bigquery_default&#39;</span><span class="p">,</span>
+            <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">sql</span><span class="o">=</span><span class="n">sql</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span [...]
+        <span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
 
     <span class="k">def</span> <span class="nf">get_db_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-        <span class="k">return</span> <span class="n">PrestoHook</span><span class="p">(</span><span class="n">presto_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">presto_conn_id</span><span class="p">)</span></div>
+        <span class="k">return</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">)</span></div>
 
 
-<div class="viewcode-block" id="PrestoValueCheckOperator"><a class="viewcode-back" href="../code.html#airflow.operators.PrestoValueCheckOperator">[docs]</a><span class="k">class</span> <span class="nc">PrestoValueCheckOperator</span><span class="p">(</span><span class="n">ValueCheckOperator</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryValueCheckOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_check_operator.BigQueryValueCheckOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryValueCheckOperator</span><span class="p">(</span><span class="n">ValueCheckOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Performs a simple value check using sql code.</span>
 
 <span class="sd">    :param sql: the sql to be executed</span>
 <span class="sd">    :type sql: string</span>
-<span class="sd">    :param presto_conn_id: reference to the Presto database</span>
-<span class="sd">    :type presto_conn_id: string</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
             <span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">pass_value</span><span class="p">,</span> <span class="n">tolerance</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-            <span class="n">presto_conn_id</span><span class="o">=</span><span class="s1">&#39;presto_default&#39;</span><span class="p">,</span>
+            <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">&#39;bigquery_default&#39;</span><span class="p">,</span>
             <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">PrestoValueCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryValueCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
             <span class="n">sql</span><span class="o">=</span><span class="n">sql</span><span class="p">,</span> <span class="n">pass_value</span><span class="o">=</span><span class="n">pass_value</span><span class="p">,</span> <span class="n">tolerance</span><span class="o">=</span><span class="n">tolerance</span><span class="p">,</span>
             <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">presto_conn_id</span> <span class="o">=</span> <span class="n">presto_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
 
     <span class="k">def</span> <span class="nf">get_db_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-        <span class="k">return</span> <span class="n">PrestoHook</span><span class="p">(</span><span class="n">presto_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">presto_conn_id</span><span class="p">)</span></div>
+        <span class="k">return</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">)</span></div>
 
 
-<div class="viewcode-block" id="PrestoIntervalCheckOperator"><a class="viewcode-back" href="../code.html#airflow.operators.PrestoIntervalCheckOperator">[docs]</a><span class="k">class</span> <span class="nc">PrestoIntervalCheckOperator</span><span class="p">(</span><span class="n">IntervalCheckOperator</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryIntervalCheckOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_check_operator.BigQueryIntervalCheckOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryIntervalCheckOperator</span><span class="p">(</span><span class="n">IntervalCheckOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Checks that the values of metrics given as SQL expressions are within</span>
 <span class="sd">    a certain tolerance of the ones from days_back before.</span>
 
+<span class="sd">    This method constructs a query like so:</span>
+
+<span class="sd">    SELECT {metrics_threshold_dict_key} FROM {table}</span>
+<span class="sd">        WHERE {date_filter_column}=&lt;date&gt;</span>
+
 <span class="sd">    :param table: the table name</span>
 <span class="sd">    :type table: str</span>
 <span class="sd">    :param days_back: number of days between ds and the ds we want to check</span>
 <span class="sd">        against. Defaults to 7 days</span>
 <span class="sd">    :type days_back: int</span>
-<span class="sd">    :param metrics_threshold: a dictionary of ratios indexed by metrics</span>
+<span class="sd">    :param metrics_threshold: a dictionary of ratios indexed by metrics, for</span>
+<span class="sd">        example &#39;COUNT(*)&#39;: 1.5 would require a 50 percent or less difference</span>
+<span class="sd">        between the current day, and the prior days_back.</span>
 <span class="sd">    :type metrics_threshold: dict</span>
-<span class="sd">    :param presto_conn_id: reference to the Presto database</span>
-<span class="sd">    :type presto_conn_id: string</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
             <span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">metrics_thresholds</span><span class="p">,</span>
             <span class="n">date_filter_column</span><span class="o">=</span><span class="s1">&#39;ds&#39;</span><span class="p">,</span> <span class="n">days_back</span><span class="o">=-</span><span class="mi">7</span><span class="p">,</span>
-            <span class="n">presto_conn_id</span><span class="o">=</span><span class="s1">&#39;presto_default&#39;</span><span class="p">,</span>
+            <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">&#39;bigquery_default&#39;</span><span class="p">,</span>
             <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">PrestoIntervalCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryIntervalCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
             <span class="n">table</span><span class="o">=</span><span class="n">table</span><span class="p">,</span> <span class="n">metrics_thresholds</span><span class="o">=</span><span class="n">metrics_thresholds</span><span class="p">,</span>
             <span class="n">date_filter_column</span><span class="o">=</span><span class="n">date_filter_column</span><span class="p">,</span> <span class="n">days_back</span><span class="o">=</span><span class="n">days_back</span><span class="p">,</span>
             <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">presto_conn_id</span> <span class="o">=</span> <span class="n">presto_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
 
     <span class="k">def</span> <span class="nf">get_db_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-        <span class="k">return</span> <span class="n">PrestoHook</span><span class="p">(</span><span class="n">presto_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">presto_conn_id</span><span class="p">)</span></div>
+        <span class="k">return</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">)</span></div>
 </pre></div>
 
            </div>
@@ -317,7 +326,7 @@
 
     <script type="text/javascript">
         var DOCUMENTATION_OPTIONS = {
-            URL_ROOT:'../',
+            URL_ROOT:'../../../../',
             VERSION:'',
             COLLAPSE_INDEX:false,
             FILE_SUFFIX:'.html',
@@ -325,15 +334,15 @@
             SOURCELINK_SUFFIX: '.txt'
         };
     </script>
-      <script type="text/javascript" src="../_static/jquery.js"></script>
-      <script type="text/javascript" src="../_static/underscore.js"></script>
-      <script type="text/javascript" src="../_static/doctools.js"></script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
 
   
 
   
   
-    <script type="text/javascript" src="../_static/js/theme.js"></script>
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
   
 
   
diff --git a/_modules/airflow/contrib/operators/bigquery_operator.html b/_modules/airflow/contrib/operators/bigquery_operator.html
index c7fcfd9..5a74ba1 100644
--- a/_modules/airflow/contrib/operators/bigquery_operator.html
+++ b/_modules/airflow/contrib/operators/bigquery_operator.html
@@ -13,6 +13,8 @@
 
   
   
+  
+  
 
   
 
@@ -30,6 +32,9 @@
   
 
   
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
     <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
         <link rel="up" title="Module code" href="../../../index.html"/> 
 
@@ -40,6 +45,7 @@
 
 <body class="wy-body-for-nav" role="document">
 
+   
   <div class="wy-grid-for-nav">
 
     
@@ -76,7 +82,10 @@
           
             
             
-                <ul>
+              
+            
+            
+              <ul>
 <li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
@@ -90,6 +99,8 @@
 <li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
 </ul>
@@ -104,8 +115,10 @@
 
       
       <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
-        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
-        <a href="../../../../index.html">Airflow</a>
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
       </nav>
 
 
@@ -118,19 +131,36 @@
 
 
 
+
+
+
+
+
+
+
+
+
+
 <div role="navigation" aria-label="breadcrumbs navigation">
+
   <ul class="wy-breadcrumbs">
-    <li><a href="../../../../index.html">Docs</a> &raquo;</li>
-      
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
           <li><a href="../../../index.html">Module code</a> &raquo;</li>
-      
-    <li>airflow.contrib.operators.bigquery_operator</li>
+        
+      <li>airflow.contrib.operators.bigquery_operator</li>
+    
+    
       <li class="wy-breadcrumbs-aside">
         
-          
+            
         
       </li>
+    
   </ul>
+
+  
   <hr/>
 </div>
           <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -151,16 +181,47 @@
 <span class="c1"># See the License for the specific language governing permissions and</span>
 <span class="c1"># limitations under the License.</span>
 
-<span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
-<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="k">import</span> <span class="n">BigQueryHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
 
 
-<div class="viewcode-block" id="BigQueryOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.bigquery_operator.BigQueryOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_operator.BigQueryOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Executes BigQuery SQL queries in a specific BigQuery database</span>
+
+<span class="sd">    :param bql: the sql code to be executed</span>
+<span class="sd">    :type bql: Can receive a str representing a sql statement,</span>
+<span class="sd">        a list of str (sql statements), or reference to a template file.</span>
+<span class="sd">        Template reference are recognized by str ending in &#39;.sql&#39;</span>
+<span class="sd">    :param destination_dataset_table: A dotted</span>
+<span class="sd">        (&lt;project&gt;.|&lt;project&gt;:)&lt;dataset&gt;.&lt;table&gt; that, if set, will store the results</span>
+<span class="sd">        of the query.</span>
+<span class="sd">    :type destination_dataset_table: string</span>
+<span class="sd">    :param write_disposition: Specifies the action that occurs if the destination table</span>
+<span class="sd">        already exists. (default: &#39;WRITE_EMPTY&#39;)</span>
+<span class="sd">    :type write_disposition: string</span>
+<span class="sd">    :param create_disposition: Specifies whether the job is allowed to create new tables.</span>
+<span class="sd">        (default: &#39;CREATE_IF_NEEDED&#39;)</span>
+<span class="sd">    :type create_disposition: string</span>
+<span class="sd">    :param bigquery_conn_id: reference to a specific BigQuery hook.</span>
+<span class="sd">    :type bigquery_conn_id: string</span>
+<span class="sd">    :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd">        For this to work, the service account making the request must have domain-wide</span>
+<span class="sd">        delegation enabled.</span>
+<span class="sd">    :type delegate_to: string</span>
+<span class="sd">    :param udf_config: The User Defined Function configuration for the query.</span>
+<span class="sd">        See https://cloud.google.com/bigquery/user-defined-functions for details.</span>
+<span class="sd">    :type udf_config: list</span>
+<span class="sd">    :param use_legacy_sql: Whether to use legacy SQL (true) or standard SQL (false).</span>
+<span class="sd">    :type use_legacy_sql: boolean</span>
+<span class="sd">    :param maximum_billing_tier: Positive integer that serves as a multiplier of the basic price.</span>
+<span class="sd">        Defaults to None, in which case it uses the value set in the project.</span>
+<span class="sd">    :type maximum_billing_tier: integer</span>
+<span class="sd">    :param query_params: a dictionary containing query parameter types and values, passed to</span>
+<span class="sd">        BigQuery.</span>
+<span class="sd">    :type query_params: dict</span>
+
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;bql&#39;</span><span class="p">,</span> <span class="s1">&#39;destination_dataset_table&#39;</span><span class="p">)</span>
     <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.sql&#39;</span><span class="p">,)</span>
@@ -169,51 +230,47 @@
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                  <span class="n">bql</span><span class="p">,</span>
-                 <span class="n">destination_dataset_table</span> <span class="o">=</span> <span class="bp">False</span><span class="p">,</span>
-                 <span class="n">write_disposition</span> <span class="o">=</span> <span class="s1">&#39;WRITE_EMPTY&#39;</span><span class="p">,</span>
-                 <span class="n">allow_large_results</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+                 <span class="n">destination_dataset_table</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                 <span class="n">write_disposition</span><span class="o">=</span><span class="s1">&#39;WRITE_EMPTY&#39;</span><span class="p">,</span>
+                 <span class="n">allow_large_results</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
                  <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">&#39;bigquery_default&#39;</span><span class="p">,</span>
-                 <span class="n">delegate_to</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
-                 <span class="n">udf_config</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+                 <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">udf_config</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                 <span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+                 <span class="n">maximum_billing_tier</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">create_disposition</span><span class="o">=</span><span class="s1">&#39;CREATE_IF_NEEDED&#39;</span><span class="p">,</span>
+                 <span class="n">query_params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                  <span class="o">*</span><span class="n">args</span><span class="p">,</span>
                  <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
-        <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Create a new BigQueryOperator.</span>
-
-<span class="sd">        :param bql: the sql code to be executed</span>
-<span class="sd">        :type bql: Can receive a str representing a sql statement,</span>
-<span class="sd">            a list of str (sql statements), or reference to a template file.</span>
-<span class="sd">            Template reference are recognized by str ending in &#39;.sql&#39;</span>
-<span class="sd">        :param destination_dataset_table: A dotted dataset.table that, if set,</span>
-<span class="sd">            will store the results of the query.</span>
-<span class="sd">        :type destination_dataset_table: string</span>
-<span class="sd">        :param bigquery_conn_id: reference to a specific BigQuery hook.</span>
-<span class="sd">        :type bigquery_conn_id: string</span>
-<span class="sd">        :param delegate_to: The account to impersonate, if any.</span>
-<span class="sd">            For this to work, the service account making the request must have domain-wide delegation enabled.</span>
-<span class="sd">        :type delegate_to: string</span>
-<span class="sd">        :param udf_config: The User Defined Function configuration for the query.</span>
-<span class="sd">            See https://cloud.google.com/bigquery/user-defined-functions for details.</span>
-<span class="sd">        :type udf_config: list</span>
-<span class="sd">        &quot;&quot;&quot;</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">bql</span> <span class="o">=</span> <span class="n">bql</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">destination_dataset_table</span> <span class="o">=</span> <span class="n">destination_dataset_table</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span> <span class="o">=</span> <span class="n">write_disposition</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">create_disposition</span> <span class="o">=</span> <span class="n">create_disposition</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">allow_large_results</span> <span class="o">=</span> <span class="n">allow_large_results</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">udf_config</span> <span class="o">=</span> <span class="n">udf_config</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span> <span class="o">=</span> <span class="n">use_legacy_sql</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">maximum_billing_tier</span> <span class="o">=</span> <span class="n">maximum_billing_tier</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">query_params</span> <span class="o">=</span> <span class="n">query_params</span>
 
     <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Executing: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bql</span><span class="p">))</span>
-        <span class="n">hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Executing: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bql</span><span class="p">)</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
+                            <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
         <span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
-        <span class="n">cursor</span><span class="o">.</span><span class="n">run_query</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bql</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">destination_dataset_table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span><span class="p">,</span> <span class="bp">self</span><s [...]
+        <span class="n">cursor</span><span class="o">.</span><span class="n">run_query</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bql</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">destination_dataset_table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span><span class="p">,</span>
+                         <span class="bp">self</span><span class="o">.</span><span class="n">allow_large_results</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">udf_config</span><span class="p">,</span>
+                         <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">maximum_billing_tier</span><span class="p">,</span>
+                         <span class="bp">self</span><span class="o">.</span><span class="n">create_disposition</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">query_params</span><span class="p">)</span></div>
 </pre></div>
 
            </div>
+           <div class="articleComments">
+            
+           </div>
           </div>
           <footer>
   
@@ -246,7 +303,8 @@
             VERSION:'',
             COLLAPSE_INDEX:false,
             FILE_SUFFIX:'.html',
-            HAS_SOURCE:  true
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
         };
     </script>
       <script type="text/javascript" src="../../../../_static/jquery.js"></script>
diff --git a/_modules/airflow/contrib/operators/bigquery_operator.html b/_modules/airflow/contrib/operators/bigquery_to_bigquery.html
similarity index 57%
copy from _modules/airflow/contrib/operators/bigquery_operator.html
copy to _modules/airflow/contrib/operators/bigquery_to_bigquery.html
index c7fcfd9..ab12d51 100644
--- a/_modules/airflow/contrib/operators/bigquery_operator.html
+++ b/_modules/airflow/contrib/operators/bigquery_to_bigquery.html
@@ -8,11 +8,13 @@
   
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   
-  <title>airflow.contrib.operators.bigquery_operator &mdash; Airflow Documentation</title>
+  <title>airflow.contrib.operators.bigquery_to_bigquery &mdash; Airflow Documentation</title>
   
 
   
   
+  
+  
 
   
 
@@ -30,6 +32,9 @@
   
 
   
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
     <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
         <link rel="up" title="Module code" href="../../../index.html"/> 
 
@@ -40,6 +45,7 @@
 
 <body class="wy-body-for-nav" role="document">
 
+   
   <div class="wy-grid-for-nav">
 
     
@@ -76,7 +82,10 @@
           
             
             
-                <ul>
+              
+            
+            
+              <ul>
 <li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
@@ -90,6 +99,8 @@
 <li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
 </ul>
@@ -104,8 +115,10 @@
 
       
       <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
-        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
-        <a href="../../../../index.html">Airflow</a>
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
       </nav>
 
 
@@ -118,25 +131,42 @@
 
 
 
+
+
+
+
+
+
+
+
+
+
 <div role="navigation" aria-label="breadcrumbs navigation">
+
   <ul class="wy-breadcrumbs">
-    <li><a href="../../../../index.html">Docs</a> &raquo;</li>
-      
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
           <li><a href="../../../index.html">Module code</a> &raquo;</li>
-      
-    <li>airflow.contrib.operators.bigquery_operator</li>
+        
+      <li>airflow.contrib.operators.bigquery_to_bigquery</li>
+    
+    
       <li class="wy-breadcrumbs-aside">
         
-          
+            
         
       </li>
+    
   </ul>
+
+  
   <hr/>
 </div>
           <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
            <div itemprop="articleBody">
             
-  <h1>Source code for airflow.contrib.operators.bigquery_operator</h1><div class="highlight"><pre>
+  <h1>Source code for airflow.contrib.operators.bigquery_to_bigquery</h1><div class="highlight"><pre>
 <span></span><span class="c1"># -*- coding: utf-8 -*-</span>
 <span class="c1">#</span>
 <span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
@@ -151,69 +181,81 @@
 <span class="c1"># See the License for the specific language governing permissions and</span>
 <span class="c1"># limitations under the License.</span>
 
-<span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
-<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="k">import</span> <span class="n">BigQueryHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
 
 
-<div class="viewcode-block" id="BigQueryOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.bigquery_operator.BigQueryOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryToBigQueryOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_to_bigquery.BigQueryToBigQueryOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryToBigQueryOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Executes BigQuery SQL queries in a specific BigQuery database</span>
+<span class="sd">    Copies data from one BigQuery table to another. See here:</span>
+
+<span class="sd">    https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.copy</span>
+
+<span class="sd">    For more details about these parameters.</span>
+
+<span class="sd">    :param source_project_dataset_tables: One or more</span>
+<span class="sd">        dotted (project:|project.)&lt;dataset&gt;.&lt;table&gt; BigQuery tables to use as the</span>
+<span class="sd">        source data. If &lt;project&gt; is not included, project will be the project defined</span>
+<span class="sd">        in the connection json. Use a list if there are multiple source tables.</span>
+<span class="sd">    :type source_project_dataset_tables: list|string</span>
+<span class="sd">    :param destination_project_dataset_table: The destination BigQuery</span>
+<span class="sd">        table. Format is: (project:|project.)&lt;dataset&gt;.&lt;table&gt;</span>
+<span class="sd">    :type destination_project_dataset_table: string</span>
+<span class="sd">    :param write_disposition: The write disposition if the table already exists.</span>
+<span class="sd">    :type write_disposition: string</span>
+<span class="sd">    :param create_disposition: The create disposition if the table doesn&#39;t exist.</span>
+<span class="sd">    :type create_disposition: string</span>
+<span class="sd">    :param bigquery_conn_id: reference to a specific BigQuery hook.</span>
+<span class="sd">    :type bigquery_conn_id: string</span>
+<span class="sd">    :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd">        For this to work, the service account making the request must have domain-wide</span>
+<span class="sd">        delegation enabled.</span>
+<span class="sd">    :type delegate_to: string</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;bql&#39;</span><span class="p">,</span> <span class="s1">&#39;destination_dataset_table&#39;</span><span class="p">)</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;source_project_dataset_tables&#39;</span><span class="p">,</span>
+                       <span class="s1">&#39;destination_project_dataset_table&#39;</span><span class="p">)</span>
     <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.sql&#39;</span><span class="p">,)</span>
-    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#e4f0e8&#39;</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#e6f0e4&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
-                 <span class="n">bql</span><span class="p">,</span>
-                 <span class="n">destination_dataset_table</span> <span class="o">=</span> <span class="bp">False</span><span class="p">,</span>
-                 <span class="n">write_disposition</span> <span class="o">=</span> <span class="s1">&#39;WRITE_EMPTY&#39;</span><span class="p">,</span>
-                 <span class="n">allow_large_results</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+                 <span class="n">source_project_dataset_tables</span><span class="p">,</span>
+                 <span class="n">destination_project_dataset_table</span><span class="p">,</span>
+                 <span class="n">write_disposition</span><span class="o">=</span><span class="s1">&#39;WRITE_EMPTY&#39;</span><span class="p">,</span>
+                 <span class="n">create_disposition</span><span class="o">=</span><span class="s1">&#39;CREATE_IF_NEEDED&#39;</span><span class="p">,</span>
                  <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">&#39;bigquery_default&#39;</span><span class="p">,</span>
-                 <span class="n">delegate_to</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
-                 <span class="n">udf_config</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+                 <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                  <span class="o">*</span><span class="n">args</span><span class="p">,</span>
                  <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
-        <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Create a new BigQueryOperator.</span>
-
-<span class="sd">        :param bql: the sql code to be executed</span>
-<span class="sd">        :type bql: Can receive a str representing a sql statement,</span>
-<span class="sd">            a list of str (sql statements), or reference to a template file.</span>
-<span class="sd">            Template reference are recognized by str ending in &#39;.sql&#39;</span>
-<span class="sd">        :param destination_dataset_table: A dotted dataset.table that, if set,</span>
-<span class="sd">            will store the results of the query.</span>
-<span class="sd">        :type destination_dataset_table: string</span>
-<span class="sd">        :param bigquery_conn_id: reference to a specific BigQuery hook.</span>
-<span class="sd">        :type bigquery_conn_id: string</span>
-<span class="sd">        :param delegate_to: The account to impersonate, if any.</span>
-<span class="sd">            For this to work, the service account making the request must have domain-wide delegation enabled.</span>
-<span class="sd">        :type delegate_to: string</span>
-<span class="sd">        :param udf_config: The User Defined Function configuration for the query.</span>
-<span class="sd">            See https://cloud.google.com/bigquery/user-defined-functions for details.</span>
-<span class="sd">        :type udf_config: list</span>
-<span class="sd">        &quot;&quot;&quot;</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">bql</span> <span class="o">=</span> <span class="n">bql</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">destination_dataset_table</span> <span class="o">=</span> <span class="n">destination_dataset_table</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryToBigQueryOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">source_project_dataset_tables</span> <span class="o">=</span> <span class="n">source_project_dataset_tables</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">destination_project_dataset_table</span> <span class="o">=</span> <span class="n">destination_project_dataset_table</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span> <span class="o">=</span> <span class="n">write_disposition</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">allow_large_results</span> <span class="o">=</span> <span class="n">allow_large_results</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">create_disposition</span> <span class="o">=</span> <span class="n">create_disposition</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">udf_config</span> <span class="o">=</span> <span class="n">udf_config</span>
 
     <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Executing: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bql</span><span class="p">))</span>
-        <span class="n">hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+            <span class="s1">&#39;Executing copy of </span><span class="si">%s</span><span class="s1"> into: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">source_project_dataset_tables</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">destination_project_dataset_table</span>
+        <span class="p">)</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
+                            <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
         <span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
-        <span class="n">cursor</span><span class="o">.</span><span class="n">run_query</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bql</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">destination_dataset_table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span><span class="p">,</span> <span class="bp">self</span><s [...]
+        <span class="n">cursor</span><span class="o">.</span><span class="n">run_copy</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">source_project_dataset_tables</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">destination_project_dataset_table</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">create_disposition</span><span class="p">)</span></div>
 </pre></div>
 
            </div>
+           <div class="articleComments">
+            
+           </div>
           </div>
           <footer>
   
@@ -246,7 +288,8 @@
             VERSION:'',
             COLLAPSE_INDEX:false,
             FILE_SUFFIX:'.html',
-            HAS_SOURCE:  true
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
         };
     </script>
       <script type="text/javascript" src="../../../../_static/jquery.js"></script>
diff --git a/_modules/airflow/contrib/operators/bigquery_to_gcs.html b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
index 97cf815..31355ff 100644
--- a/_modules/airflow/contrib/operators/bigquery_to_gcs.html
+++ b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
@@ -13,6 +13,8 @@
 
   
   
+  
+  
 
   
 
@@ -30,6 +32,9 @@
   
 
   
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
     <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
         <link rel="up" title="Module code" href="../../../index.html"/> 
 
@@ -40,6 +45,7 @@
 
 <body class="wy-body-for-nav" role="document">
 
+   
   <div class="wy-grid-for-nav">
 
     
@@ -76,7 +82,10 @@
           
             
             
-                <ul>
+              
+            
+            
+              <ul>
 <li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
@@ -90,6 +99,8 @@
 <li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
 </ul>
@@ -104,8 +115,10 @@
 
       
       <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
-        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
-        <a href="../../../../index.html">Airflow</a>
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
       </nav>
 
 
@@ -118,19 +131,36 @@
 
 
 
+
+
+
+
+
+
+
+
+
+
 <div role="navigation" aria-label="breadcrumbs navigation">
+
   <ul class="wy-breadcrumbs">
-    <li><a href="../../../../index.html">Docs</a> &raquo;</li>
-      
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
           <li><a href="../../../index.html">Module code</a> &raquo;</li>
-      
-    <li>airflow.contrib.operators.bigquery_to_gcs</li>
+        
+      <li>airflow.contrib.operators.bigquery_to_gcs</li>
+    
+    
       <li class="wy-breadcrumbs-aside">
         
-          
+            
         
       </li>
+    
   </ul>
+
+  
   <hr/>
 </div>
           <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -151,65 +181,63 @@
 <span class="c1"># See the License for the specific language governing permissions and</span>
 <span class="c1"># limitations under the License.</span>
 
-<span class="kn">import</span> <span class="nn">logging</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="k">import</span> <span class="n">BigQueryHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
 
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
-<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
-
-<div class="viewcode-block" id="BigQueryToCloudStorageOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryToCloudStorageOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryToCloudStorageOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryToCloudStorageOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Transfers a BigQuery table to a Google Cloud Storage bucket.</span>
+
+<span class="sd">    See here:</span>
+
+<span class="sd">    https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
+
+<span class="sd">    For more details about these parameters.</span>
+
+<span class="sd">    :param source_project_dataset_table: The dotted</span>
+<span class="sd">        (&lt;project&gt;.|&lt;project&gt;:)&lt;dataset&gt;.&lt;table&gt; BigQuery table to use as the source</span>
+<span class="sd">        data. If &lt;project&gt; is not included, project will be the project defined in</span>
+<span class="sd">        the connection json.</span>
+<span class="sd">    :type source_project_dataset_table: string</span>
+<span class="sd">    :param destination_cloud_storage_uris: The destination Google Cloud</span>
+<span class="sd">        Storage URI (e.g. gs://some-bucket/some-file.txt). Follows</span>
+<span class="sd">        convention defined here:</span>
+<span class="sd">        https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple</span>
+<span class="sd">    :type destination_cloud_storage_uris: list</span>
+<span class="sd">    :param compression: Type of compression to use.</span>
+<span class="sd">    :type compression: string</span>
+<span class="sd">    :param export_format: File format to export.</span>
+<span class="sd">    :type field_delimiter: string</span>
+<span class="sd">    :param field_delimiter: The delimiter to use when extracting to a CSV.</span>
+<span class="sd">    :type field_delimiter: string</span>
+<span class="sd">    :param print_header: Whether to print a header for a CSV file extract.</span>
+<span class="sd">    :type print_header: boolean</span>
+<span class="sd">    :param bigquery_conn_id: reference to a specific BigQuery hook.</span>
+<span class="sd">    :type bigquery_conn_id: string</span>
+<span class="sd">    :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd">        For this to work, the service account making the request must have domain-wide</span>
+<span class="sd">        delegation enabled.</span>
+<span class="sd">    :type delegate_to: string</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;source_project_dataset_table&#39;</span><span class="p">,</span><span class="s1">&#39;destination_cloud_storage_uris&#39;</span><span class="p">,)</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;source_project_dataset_table&#39;</span><span class="p">,</span> <span class="s1">&#39;destination_cloud_storage_uris&#39;</span><span class="p">)</span>
     <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.sql&#39;</span><span class="p">,)</span>
     <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#e4e6f0&#39;</span>
 
     <span class="nd">@apply_defaults</span>
-    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
-        <span class="bp">self</span><span class="p">,</span>
-        <span class="n">source_project_dataset_table</span><span class="p">,</span>
-        <span class="n">destination_cloud_storage_uris</span><span class="p">,</span>
-        <span class="n">compression</span><span class="o">=</span><span class="s1">&#39;NONE&#39;</span><span class="p">,</span>
-        <span class="n">export_format</span><span class="o">=</span><span class="s1">&#39;CSV&#39;</span><span class="p">,</span>
-        <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">&#39;,&#39;</span><span class="p">,</span>
-        <span class="n">print_header</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
-        <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">&#39;bigquery_default&#39;</span><span class="p">,</span>
-        <span class="n">delegate_to</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
-        <span class="o">*</span><span class="n">args</span><span class="p">,</span>
-        <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
-        <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Create a new BigQueryToCloudStorage to move data from BigQuery to</span>
-<span class="sd">        Google Cloud Storage.  See here:</span>
-
-<span class="sd">        https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
-
-<span class="sd">        For more details about these parameters.</span>
-
-<span class="sd">        :param source_project_dataset_table: The dotted (&lt;project&gt;.)&lt;dataset&gt;.&lt;table&gt; BigQuery table to use as the</span>
-<span class="sd">            source data. If &lt;project&gt; is not included, project will be the project defined in the connection json.</span>
-<span class="sd">        :type source_project_dataset_table: string</span>
-<span class="sd">        :param destination_cloud_storage_uris: The destination Google Cloud</span>
-<span class="sd">            Storage URI (e.g. gs://some-bucket/some-file.txt). Follows</span>
-<span class="sd">            convention defined here:</span>
-<span class="sd">            https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple</span>
-<span class="sd">        :type destination_cloud_storage_uris: list</span>
-<span class="sd">        :param compression: Type of compression to use.</span>
-<span class="sd">        :type compression: string</span>
-<span class="sd">        :param export_format: File format to export.</span>
-<span class="sd">        :type field_delimiter: string</span>
-<span class="sd">        :param field_delimiter: The delimiter to use when extracting to a CSV.</span>
-<span class="sd">        :type field_delimiter: string</span>
-<span class="sd">        :param print_header: Whether to print a header for a CSV file extract.</span>
-<span class="sd">        :type print_header: boolean</span>
-<span class="sd">        :param bigquery_conn_id: reference to a specific BigQuery hook.</span>
-<span class="sd">        :type bigquery_conn_id: string</span>
-<span class="sd">        :param delegate_to: The account to impersonate, if any.</span>
-<span class="sd">            For this to work, the service account making the request must have domain-wide delegation enabled.</span>
-<span class="sd">        :type delegate_to: string</span>
-<span class="sd">        &quot;&quot;&quot;</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryToCloudStorageOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">source_project_dataset_table</span><span class="p">,</span>
+                 <span class="n">destination_cloud_storage_uris</span><span class="p">,</span>
+                 <span class="n">compression</span><span class="o">=</span><span class="s1">&#39;NONE&#39;</span><span class="p">,</span>
+                 <span class="n">export_format</span><span class="o">=</span><span class="s1">&#39;CSV&#39;</span><span class="p">,</span>
+                 <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">&#39;,&#39;</span><span class="p">,</span>
+                 <span class="n">print_header</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+                 <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">&#39;bigquery_default&#39;</span><span class="p">,</span>
+                 <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+                 <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryToCloudStorageOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">source_project_dataset_table</span> <span class="o">=</span> <span class="n">source_project_dataset_table</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">destination_cloud_storage_uris</span> <span class="o">=</span> <span class="n">destination_cloud_storage_uris</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">compression</span> <span class="o">=</span> <span class="n">compression</span>
@@ -220,8 +248,11 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
 
     <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Executing extract of </span><span class="si">%s</span><span class="s1"> into: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">source_project_dataset_table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class= [...]
-        <span class="n">hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Executing extract of </span><span class="si">%s</span><span class="s1"> into: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+                      <span class="bp">self</span><span class="o">.</span><span class="n">source_project_dataset_table</span><span class="p">,</span>
+                      <span class="bp">self</span><span class="o">.</span><span class="n">destination_cloud_storage_uris</span><span class="p">)</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
+                            <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
         <span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
         <span class="n">cursor</span><span class="o">.</span><span class="n">run_extract</span><span class="p">(</span>
@@ -234,6 +265,9 @@
 </pre></div>
 
            </div>
+           <div class="articleComments">
+            
+           </div>
           </div>
           <footer>
   
@@ -266,7 +300,8 @@
             VERSION:'',
             COLLAPSE_INDEX:false,
             FILE_SUFFIX:'.html',
-            HAS_SOURCE:  true
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
         };
     </script>
       <script type="text/javascript" src="../../../../_static/jquery.js"></script>
diff --git a/_modules/airflow/contrib/operators/databricks_operator.html b/_modules/airflow/contrib/operators/databricks_operator.html
new file mode 100644
index 0000000..cb38349
--- /dev/null
+++ b/_modules/airflow/contrib/operators/databricks_operator.html
@@ -0,0 +1,487 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.operators.databricks_operator &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.operators.databricks_operator</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.operators.databricks_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">import</span> <span class="nn">six</span>
+<span class="kn">import</span> <span class="nn">time</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.databricks_hook</span> <span class="k">import</span> <span class="n">DatabricksHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+
+
+<div class="viewcode-block" id="DatabricksSubmitRunOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator">[docs]</a><span class="k">class</span> <span class="nc">DatabricksSubmitRunOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Submits an Spark job run to Databricks using the</span>
+<span class="sd">    `api/2.0/jobs/runs/submit</span>
+<span class="sd">    &lt;https://docs.databricks.com/api/latest/jobs.html#runs-submit&gt;`_</span>
+<span class="sd">    API endpoint.</span>
+
+<span class="sd">    There are two ways to instantiate this operator.</span>
+
+<span class="sd">    In the first way, you can take the JSON payload that you typically use</span>
+<span class="sd">    to call the ``api/2.0/jobs/runs/submit`` endpoint and pass it directly</span>
+<span class="sd">    to our ``DatabricksSubmitRunOperator`` through the ``json`` parameter.</span>
+<span class="sd">    For example ::</span>
+<span class="sd">        json = {</span>
+<span class="sd">          &#39;new_cluster&#39;: {</span>
+<span class="sd">            &#39;spark_version&#39;: &#39;2.1.0-db3-scala2.11&#39;,</span>
+<span class="sd">            &#39;num_workers&#39;: 2</span>
+<span class="sd">          },</span>
+<span class="sd">          &#39;notebook_task&#39;: {</span>
+<span class="sd">            &#39;notebook_path&#39;: &#39;/Users/airflow@example.com/PrepareData&#39;,</span>
+<span class="sd">          },</span>
+<span class="sd">        }</span>
+<span class="sd">        notebook_run = DatabricksSubmitRunOperator(task_id=&#39;notebook_run&#39;, json=json)</span>
+
+<span class="sd">    Another way to accomplish the same thing is to use the named parameters</span>
+<span class="sd">    of the ``DatabricksSubmitRunOperator`` directly. Note that there is exactly</span>
+<span class="sd">    one named parameter for each top level parameter in the ``runs/submit``</span>
+<span class="sd">    endpoint. In this method, your code would look like this: ::</span>
+<span class="sd">        new_cluster = {</span>
+<span class="sd">          &#39;spark_version&#39;: &#39;2.1.0-db3-scala2.11&#39;,</span>
+<span class="sd">          &#39;num_workers&#39;: 2</span>
+<span class="sd">        }</span>
+<span class="sd">        notebook_task = {</span>
+<span class="sd">          &#39;notebook_path&#39;: &#39;/Users/airflow@example.com/PrepareData&#39;,</span>
+<span class="sd">        }</span>
+<span class="sd">        notebook_run = DatabricksSubmitRunOperator(</span>
+<span class="sd">            task_id=&#39;notebook_run&#39;,</span>
+<span class="sd">            new_cluster=new_cluster,</span>
+<span class="sd">            notebook_task=notebook_task)</span>
+
+<span class="sd">    In the case where both the json parameter **AND** the named parameters</span>
+<span class="sd">    are provided, they will be merged together. If there are conflicts during the merge,</span>
+<span class="sd">    the named parameters will take precedence and override the top level ``json`` keys.</span>
+
+<span class="sd">    Currently the named parameters that ``DatabricksSubmitRunOperator`` supports are</span>
+<span class="sd">        - ``spark_jar_task``</span>
+<span class="sd">        - ``notebook_task``</span>
+<span class="sd">        - ``new_cluster``</span>
+<span class="sd">        - ``existing_cluster_id``</span>
+<span class="sd">        - ``libraries``</span>
+<span class="sd">        - ``run_name``</span>
+<span class="sd">        - ``timeout_seconds``</span>
+
+<span class="sd">    :param json: A JSON object containing API parameters which will be passed</span>
+<span class="sd">        directly to the ``api/2.0/jobs/runs/submit`` endpoint. The other named parameters</span>
+<span class="sd">        (i.e. ``spark_jar_task``, ``notebook_task``..) to this operator will</span>
+<span class="sd">        be merged with this json dictionary if they are provided.</span>
+<span class="sd">        If there are conflicts during the merge, the named parameters will</span>
+<span class="sd">        take precedence and override the top level json keys. This field will be</span>
+<span class="sd">        templated.</span>
+
+<span class="sd">        .. seealso::</span>
+<span class="sd">            For more information about templating see :ref:`jinja-templating`.</span>
+<span class="sd">            https://docs.databricks.com/api/latest/jobs.html#runs-submit</span>
+<span class="sd">    :type json: dict</span>
+<span class="sd">    :param spark_jar_task: The main class and parameters for the JAR task. Note that</span>
+<span class="sd">        the actual JAR is specified in the ``libraries``.</span>
+<span class="sd">        *EITHER* ``spark_jar_task`` *OR* ``notebook_task`` should be specified.</span>
+<span class="sd">        This field will be templated.</span>
+
+<span class="sd">        .. seealso::</span>
+<span class="sd">            https://docs.databricks.com/api/latest/jobs.html#jobssparkjartask</span>
+<span class="sd">    :type spark_jar_task: dict</span>
+<span class="sd">    :param notebook_task: The notebook path and parameters for the notebook task.</span>
+<span class="sd">        *EITHER* ``spark_jar_task`` *OR* ``notebook_task`` should be specified.</span>
+<span class="sd">        This field will be templated.</span>
+
+<span class="sd">        .. seealso::</span>
+<span class="sd">            https://docs.databricks.com/api/latest/jobs.html#jobsnotebooktask</span>
+<span class="sd">    :type notebook_task: dict</span>
+<span class="sd">    :param new_cluster: Specs for a new cluster on which this task will be run.</span>
+<span class="sd">        *EITHER* ``new_cluster`` *OR* ``existing_cluster_id`` should be specified.</span>
+<span class="sd">        This field will be templated.</span>
+
+<span class="sd">        .. seealso::</span>
+<span class="sd">            https://docs.databricks.com/api/latest/jobs.html#jobsclusterspecnewcluster</span>
+<span class="sd">    :type new_cluster: dict</span>
+<span class="sd">    :param existing_cluster_id: ID for existing cluster on which to run this task.</span>
+<span class="sd">        *EITHER* ``new_cluster`` *OR* ``existing_cluster_id`` should be specified.</span>
+<span class="sd">        This field will be templated.</span>
+<span class="sd">    :type existing_cluster_id: string</span>
+<span class="sd">    :param libraries: Libraries which this run will use.</span>
+<span class="sd">        This field will be templated.</span>
+
+<span class="sd">        .. seealso::</span>
+<span class="sd">            https://docs.databricks.com/api/latest/libraries.html#managedlibrarieslibrary</span>
+<span class="sd">    :type libraries: list of dicts</span>
+<span class="sd">    :param run_name: The run name used for this task.</span>
+<span class="sd">        By default this will be set to the Airflow ``task_id``. This ``task_id`` is a</span>
+<span class="sd">        required parameter of the superclass ``BaseOperator``.</span>
+<span class="sd">        This field will be templated.</span>
+<span class="sd">    :type run_name: string</span>
+<span class="sd">    :param timeout_seconds: The timeout for this run. By default a value of 0 is used</span>
+<span class="sd">        which means to have no timeout.</span>
+<span class="sd">        This field will be templated.</span>
+<span class="sd">    :type timeout_seconds: int32</span>
+<span class="sd">    :param databricks_conn_id: The name of the Airflow connection to use.</span>
+<span class="sd">        By default and in the common case this will be ``databricks_default``. To use</span>
+<span class="sd">        token based authentication, provide the key ``token`` in the extra field for the</span>
+<span class="sd">        connection.</span>
+<span class="sd">    :type databricks_conn_id: string</span>
+<span class="sd">    :param polling_period_seconds: Controls the rate which we poll for the result of</span>
+<span class="sd">        this run. By default the operator will poll every 30 seconds.</span>
+<span class="sd">    :type polling_period_seconds: int</span>
+<span class="sd">    :param databricks_retry_limit: Amount of times retry if the Databricks backend is</span>
+<span class="sd">        unreachable. Its value must be greater than or equal to 1.</span>
+<span class="sd">    :type databricks_retry_limit: int</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="c1"># Used in airflow.models.BaseOperator</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;json&#39;</span><span class="p">,)</span>
+    <span class="c1"># Databricks brand color (blue) under white text</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#1CB1C2&#39;</span>
+    <span class="n">ui_fgcolor</span> <span class="o">=</span> <span class="s1">&#39;#fff&#39;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">json</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">spark_jar_task</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">notebook_task</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">new_cluster</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">existing_cluster_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">libraries</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">run_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">timeout_seconds</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">databricks_conn_id</span><span class="o">=</span><span class="s1">&#39;databricks_default&#39;</span><span class="p">,</span>
+            <span class="n">polling_period_seconds</span><span class="o">=</span><span class="mi">30</span><span class="p">,</span>
+            <span class="n">databricks_retry_limit</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Creates a new ``DatabricksSubmitRunOperator``.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">DatabricksSubmitRunOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">json</span> <span class="o">=</span> <span class="n">json</span> <span class="ow">or</span> <span class="p">{}</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn_id</span> <span class="o">=</span> <span class="n">databricks_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">polling_period_seconds</span> <span class="o">=</span> <span class="n">polling_period_seconds</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">databricks_retry_limit</span> <span class="o">=</span> <span class="n">databricks_retry_limit</span>
+        <span class="k">if</span> <span class="n">spark_jar_task</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">json</span><span class="p">[</span><span class="s1">&#39;spark_jar_task&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">spark_jar_task</span>
+        <span class="k">if</span> <span class="n">notebook_task</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">json</span><span class="p">[</span><span class="s1">&#39;notebook_task&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">notebook_task</span>
+        <span class="k">if</span> <span class="n">new_cluster</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">json</span><span class="p">[</span><span class="s1">&#39;new_cluster&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_cluster</span>
+        <span class="k">if</span> <span class="n">existing_cluster_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">json</span><span class="p">[</span><span class="s1">&#39;existing_cluster_id&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">existing_cluster_id</span>
+        <span class="k">if</span> <span class="n">libraries</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">json</span><span class="p">[</span><span class="s1">&#39;libraries&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">libraries</span>
+        <span class="k">if</span> <span class="n">run_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">json</span><span class="p">[</span><span class="s1">&#39;run_name&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">run_name</span>
+        <span class="k">if</span> <span class="n">timeout_seconds</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">json</span><span class="p">[</span><span class="s1">&#39;timeout_seconds&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">timeout_seconds</span>
+        <span class="k">if</span> <span class="s1">&#39;run_name&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">json</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">json</span><span class="p">[</span><span class="s1">&#39;run_name&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">run_name</span> <span class="ow">or</span> <span class="n">kwargs</span><span class="p">[</span><span class="s1">&#39;task_id&#39;</span><span class="p">]</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">json</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_deep_string_coerce</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">json</span><span class="p">)</span>
+        <span class="c1"># This variable will be used in case our task gets killed.</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">run_id</span> <span class="o">=</span> <span class="kc">None</span>
+
+    <span class="k">def</span> <span class="nf">_deep_string_coerce</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">content</span><span class="p">,</span> <span class="n">json_path</span><span class="o">=</span><span class="s1">&#39;json&#39;</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Coerces content or all values of content if it is a dict to a string. The</span>
+<span class="sd">        function will throw if content contains non-string or non-numeric types.</span>
+
+<span class="sd">        The reason why we have this function is because the ``self.json`` field must be a dict</span>
+<span class="sd">        with only string values. This is because ``render_template`` will fail for numerical values.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">c</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_deep_string_coerce</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">):</span>
+            <span class="k">return</span> <span class="n">content</span>
+        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">integer_types</span><span class="o">+</span><span class="p">(</span><span class="nb">float</span><span class="p">,)):</span>
+            <span class="c1"># Databricks can tolerate either numeric or string types in the API backend.</span>
+            <span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
+            <span class="k">return</span> <span class="p">[</span><span class="n">c</span><span class="p">(</span><span class="n">e</span><span class="p">,</span> <span class="s1">&#39;</span><span class="si">{0}</span><span class="s1">[</span><span class="si">{1}</span><span class="s1">]&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json_path</span><span class="p">,</span> <span class="n">i</span><span class="p">))</span> <span c [...]
+        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
+            <span class="k">return</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">c</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="s1">&#39;</span><span class="si">{0}</span><span class="s1">[</span><span class="si">{1}</span><span class="s1">]&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json_path</span><span class="p">,</span> <span cl [...]
+                    <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="n">content</span><span class="o">.</span><span class="n">items</span><span class="p">())}</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">param_type</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
+            <span class="n">msg</span> <span class="o">=</span> <span class="s1">&#39;Type </span><span class="si">{0}</span><span class="s1"> used for parameter </span><span class="si">{1}</span><span class="s1"> is not a number or a string&#39;</span> \
+                    <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">param_type</span><span class="p">,</span> <span class="n">json_path</span><span class="p">)</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_log_run_page_url</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">url</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;View run status, Spark UI, and logs at </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="n">DatabricksHook</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn_id</span><span class="p">,</span>
+            <span class="n">retry_limit</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">databricks_retry_limit</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">run_id</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">submit_run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">json</span><span class="p">)</span>
+        <span class="n">run_page_url</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_run_page_url</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">run_id</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Run submitted with run_id: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_id</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_log_run_page_url</span><span class="p">(</span><span class="n">run_page_url</span><span class="p">)</span>
+        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+            <span class="n">run_state</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_run_state</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">run_id</span><span class="p">)</span>
+            <span class="k">if</span> <span class="n">run_state</span><span class="o">.</span><span class="n">is_terminal</span><span class="p">:</span>
+                <span class="k">if</span> <span class="n">run_state</span><span class="o">.</span><span class="n">is_successful</span><span class="p">:</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">%s</span><span class="s1"> completed successfully.&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">_log_run_page_url</span><span class="p">(</span><span class="n">run_page_url</span><span class="p">)</span>
+                    <span class="k">return</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="n">error_message</span> <span class="o">=</span> <span class="s1">&#39;</span><span class="si">{t}</span><span class="s1"> failed with terminal state: </span><span class="si">{s}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                        <span class="n">t</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span>
+                        <span class="n">s</span><span class="o">=</span><span class="n">run_state</span><span class="p">)</span>
+                    <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="n">error_message</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">%s</span><span class="s1"> in run state: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="n">run_state</span><span clas [...]
+                <span class="bp">self</span><span class="o">.</span><span class="n">_log_run_page_url</span><span class="p">(</span><span class="n">run_page_url</span><span class="p">)</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Sleeping for </span><span class="si">%s</span><span class="s1"> seconds.&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">polling_period_seconds</span><span class="p">)</span>
+                <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">polling_period_seconds</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span>
+        <span class="n">hook</span><span class="o">.</span><span class="n">cancel_run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">run_id</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+            <span class="s1">&#39;Task: </span><span class="si">%s</span><span class="s1"> with run_id: </span><span class="si">%s</span><span class="s1"> was requested to be cancelled.&#39;</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_id</span>
+        <span class="p">)</span></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/operators/dataflow_operator.html b/_modules/airflow/contrib/operators/dataflow_operator.html
new file mode 100644
index 0000000..6807e24
--- /dev/null
+++ b/_modules/airflow/contrib/operators/dataflow_operator.html
@@ -0,0 +1,470 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.operators.dataflow_operator &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.operators.dataflow_operator</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.operators.dataflow_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">copy</span>
+<span class="kn">import</span> <span class="nn">re</span>
+<span class="kn">import</span> <span class="nn">uuid</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="k">import</span> <span class="n">GoogleCloudStorageHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_dataflow_hook</span> <span class="k">import</span> <span class="n">DataFlowHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="DataFlowJavaOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataflow_operator.DataFlowJavaOperator">[docs]</a><span class="k">class</span> <span class="nc">DataFlowJavaOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Start a Java Cloud DataFlow batch job. The parameters of the operation</span>
+<span class="sd">    will be passed to the job.</span>
+
+<span class="sd">    It&#39;s a good practice to define dataflow_* parameters in the default_args of the dag</span>
+<span class="sd">    like the project, zone and staging location.</span>
+
+<span class="sd">    ```</span>
+<span class="sd">    default_args = {</span>
+<span class="sd">        &#39;dataflow_default_options&#39;: {</span>
+<span class="sd">            &#39;project&#39;: &#39;my-gcp-project&#39;,</span>
+<span class="sd">            &#39;zone&#39;: &#39;europe-west1-d&#39;,</span>
+<span class="sd">            &#39;stagingLocation&#39;: &#39;gs://my-staging-bucket/staging/&#39;</span>
+<span class="sd">        }</span>
+<span class="sd">    }</span>
+<span class="sd">    ```</span>
+
+<span class="sd">    You need to pass the path to your dataflow as a file reference with the ``jar``</span>
+<span class="sd">    parameter, the jar needs to be a self executing jar. Use ``options`` to pass on</span>
+<span class="sd">    options to your job.</span>
+
+<span class="sd">    ```</span>
+<span class="sd">    t1 = DataFlowOperation(</span>
+<span class="sd">        task_id=&#39;datapflow_example&#39;,</span>
+<span class="sd">        jar=&#39;{{var.value.gcp_dataflow_base}}pipeline/build/libs/pipeline-example-1.0.jar&#39;,</span>
+<span class="sd">        options={</span>
+<span class="sd">            &#39;autoscalingAlgorithm&#39;: &#39;BASIC&#39;,</span>
+<span class="sd">            &#39;maxNumWorkers&#39;: &#39;50&#39;,</span>
+<span class="sd">            &#39;start&#39;: &#39;{{ds}}&#39;,</span>
+<span class="sd">            &#39;partitionType&#39;: &#39;DAY&#39;</span>
+<span class="sd">        },</span>
+<span class="sd">        dag=my-dag)</span>
+<span class="sd">    ```</span>
+
+<span class="sd">    Both ``jar`` and ``options`` are templated so you can use variables in them.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;options&#39;</span><span class="p">,</span> <span class="s1">&#39;jar&#39;</span><span class="p">]</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#0273d4&#39;</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">jar</span><span class="p">,</span>
+            <span class="n">dataflow_default_options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_default&#39;</span><span class="p">,</span>
+            <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Create a new DataFlowJavaOperator. Note that both</span>
+<span class="sd">        dataflow_default_options and options will be merged to specify pipeline</span>
+<span class="sd">        execution parameter, and dataflow_default_options is expected to save</span>
+<span class="sd">        high-level options, for instances, project and zone information, which</span>
+<span class="sd">        apply to all dataflow operators in the DAG.</span>
+
+<span class="sd">        For more detail on job submission have a look at the reference:</span>
+
+<span class="sd">        https://cloud.google.com/dataflow/pipelines/specifying-exec-params</span>
+
+<span class="sd">        :param jar: The reference to a self executing DataFlow jar.</span>
+<span class="sd">        :type jar: string</span>
+<span class="sd">        :param dataflow_default_options: Map of default job options.</span>
+<span class="sd">        :type dataflow_default_options: dict</span>
+<span class="sd">        :param options: Map of job specific options.</span>
+<span class="sd">        :type options: dict</span>
+<span class="sd">        :param gcp_conn_id: The connection ID to use connecting to Google Cloud</span>
+<span class="sd">        Platform.</span>
+<span class="sd">        :type gcp_conn_id: string</span>
+<span class="sd">        :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd">            For this to work, the service account making the request must have</span>
+<span class="sd">            domain-wide delegation enabled.</span>
+<span class="sd">        :type delegate_to: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">DataFlowJavaOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+        <span class="n">dataflow_default_options</span> <span class="o">=</span> <span class="n">dataflow_default_options</span> <span class="ow">or</span> <span class="p">{}</span>
+        <span class="n">options</span> <span class="o">=</span> <span class="n">options</span> <span class="ow">or</span> <span class="p">{}</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">jar</span> <span class="o">=</span> <span class="n">jar</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dataflow_default_options</span> <span class="o">=</span> <span class="n">dataflow_default_options</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">options</span> <span class="o">=</span> <span class="n">options</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="n">bucket_helper</span> <span class="o">=</span> <span class="n">GoogleCloudBucketHelper</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">jar</span> <span class="o">=</span> <span class="n">bucket_helper</span><span class="o">.</span><span class="n">google_cloud_to_local</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">jar</span><span class="p">)</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">DataFlowHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+                            <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+
+        <span class="n">dataflow_options</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataflow_default_options</span><span class="p">)</span>
+        <span class="n">dataflow_options</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">options</span><span class="p">)</span>
+
+        <span class="n">hook</span><span class="o">.</span><span class="n">start_java_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="n">dataflow_options</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">jar</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="DataFlowPythonOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataflow_operator.DataFlowPythonOperator">[docs]</a><span class="k">class</span> <span class="nc">DataFlowPythonOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;options&#39;</span><span class="p">,</span> <span class="s1">&#39;dataflow_default_options&#39;</span><span class="p">]</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">py_file</span><span class="p">,</span>
+            <span class="n">py_options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">dataflow_default_options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_default&#39;</span><span class="p">,</span>
+            <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Create a new DataFlowPythonOperator. Note that both</span>
+<span class="sd">        dataflow_default_options and options will be merged to specify pipeline</span>
+<span class="sd">        execution parameter, and dataflow_default_options is expected to save</span>
+<span class="sd">        high-level options, for instances, project and zone information, which</span>
+<span class="sd">        apply to all dataflow operators in the DAG.</span>
+
+<span class="sd">        For more detail on job submission have a look at the reference:</span>
+
+<span class="sd">        https://cloud.google.com/dataflow/pipelines/specifying-exec-params</span>
+
+<span class="sd">        :param py_file: Reference to the python dataflow pipleline file, e.g.,</span>
+<span class="sd">            /some/local/file/path/to/your/python/pipeline/file.py.</span>
+<span class="sd">        :type py_file: string</span>
+<span class="sd">        :param py_options: Additional python options.</span>
+<span class="sd">        :type pyt_options: list of strings, e.g., [&quot;-m&quot;, &quot;-v&quot;].</span>
+<span class="sd">        :param dataflow_default_options: Map of default job options.</span>
+<span class="sd">        :type dataflow_default_options: dict</span>
+<span class="sd">        :param options: Map of job specific options.</span>
+<span class="sd">        :type options: dict</span>
+<span class="sd">        :param gcp_conn_id: The connection ID to use connecting to Google Cloud</span>
+<span class="sd">            Platform.</span>
+<span class="sd">        :type gcp_conn_id: string</span>
+<span class="sd">        :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd">            For this to work, the service account making the request must have</span>
+<span class="sd">            domain-wide  delegation enabled.</span>
+<span class="sd">        :type delegate_to: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">DataFlowPythonOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">py_file</span> <span class="o">=</span> <span class="n">py_file</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">py_options</span> <span class="o">=</span> <span class="n">py_options</span> <span class="ow">or</span> <span class="p">[]</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dataflow_default_options</span> <span class="o">=</span> <span class="n">dataflow_default_options</span> <span class="ow">or</span> <span class="p">{}</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">options</span> <span class="o">=</span> <span class="n">options</span> <span class="ow">or</span> <span class="p">{}</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;Execute the python dataflow job.&quot;&quot;&quot;</span>
+        <span class="n">bucket_helper</span> <span class="o">=</span> <span class="n">GoogleCloudBucketHelper</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">py_file</span> <span class="o">=</span> <span class="n">bucket_helper</span><span class="o">.</span><span class="n">google_cloud_to_local</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">py_file</span><span class="p">)</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">DataFlowHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+                            <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+        <span class="n">dataflow_options</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataflow_default_options</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+        <span class="n">dataflow_options</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">options</span><span class="p">)</span>
+        <span class="c1"># Convert argument names from lowerCamelCase to snake case.</span>
+        <span class="n">camel_to_snake</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">name</span><span class="p">:</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span>
+            <span class="sa">r</span><span class="s1">&#39;[A-Z]&#39;</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="s1">&#39;_&#39;</span> <span class="o">+</span> <span class="n">x</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">(),</span> <span class="n">name</s [...]
+        <span class="n">formatted_options</span> <span class="o">=</span> <span class="p">{</span><span class="n">camel_to_snake</span><span class="p">(</span><span class="n">key</span><span class="p">):</span> <span class="n">dataflow_options</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
+                             <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">dataflow_options</span><span class="p">}</span>
+        <span class="n">hook</span><span class="o">.</span><span class="n">start_python_dataflow</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="n">formatted_options</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">py_file</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">py_options</span><span class="p">)</span></div>
+
+
+<span class="k">class</span> <span class="nc">GoogleCloudBucketHelper</span><span class="p">():</span>
+    <span class="sd">&quot;&quot;&quot;GoogleCloudStorageHook helper class to download GCS object.&quot;&quot;&quot;</span>
+    <span class="n">GCS_PREFIX_LENGTH</span> <span class="o">=</span> <span class="mi">5</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_default&#39;</span><span class="p">,</span>
+                 <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_gcs_hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">google_cloud_to_local</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_name</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Checks whether the file specified by file_name is stored in Google Cloud</span>
+<span class="sd">        Storage (GCS), if so, downloads the file and saves it locally. The full</span>
+<span class="sd">        path of the saved file will be returned. Otherwise the local file_name</span>
+<span class="sd">        will be returned immediately.</span>
+
+<span class="sd">        :param file_name: The full path of input file.</span>
+<span class="sd">        :type file_name: string</span>
+<span class="sd">        :return: The full path of local file.</span>
+<span class="sd">        :type: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">file_name</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;gs://&#39;</span><span class="p">):</span>
+            <span class="k">return</span> <span class="n">file_name</span>
+
+        <span class="c1"># Extracts bucket_id and object_id by first removing &#39;gs://&#39; prefix and</span>
+        <span class="c1"># then split the remaining by path delimiter &#39;/&#39;.</span>
+        <span class="n">path_components</span> <span class="o">=</span> <span class="n">file_name</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">GCS_PREFIX_LENGTH</span><span class="p">:]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">path_components</span> <span class="o">&lt;</span> <span class="mi">2</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
+                <span class="s1">&#39;Invalid Google Cloud Storage (GCS) object path: </span><span class="si">{}</span><span class="s1">.&#39;</span>
+                <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">file_name</span><span class="p">))</span>
+
+        <span class="n">bucket_id</span> <span class="o">=</span> <span class="n">path_components</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+        <span class="n">object_id</span> <span class="o">=</span> <span class="s1">&#39;/&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">path_components</span><span class="p">[</span><span class="mi">1</span><span class="p">:])</span>
+        <span class="n">local_file</span> <span class="o">=</span> <span class="s1">&#39;/tmp/dataflow</span><span class="si">{}</span><span class="s1">-</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid1</span><span class="p">())[:</span><span class="mi">8</span><span class="p">] [...]
+                                                 <span class="n">path_components</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
+        <span class="n">file_size</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_gcs_hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="n">bucket_id</span><span class="p">,</span> <span class="n">object_id</span><span class="p">,</span> <span class="n">local_file</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">file_size</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">local_file</span>
+        <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
+            <span class="s1">&#39;Failed to download Google Cloud Storage GCS object: </span><span class="si">{}</span><span class="s1">&#39;</span>
+            <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">file_name</span><span class="p">))</span>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/operators/dataproc_operator.html b/_modules/airflow/contrib/operators/dataproc_operator.html
new file mode 100644
index 0000000..d49719e
--- /dev/null
+++ b/_modules/airflow/contrib/operators/dataproc_operator.html
@@ -0,0 +1,1139 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.operators.dataproc_operator &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.operators.dataproc_operator</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.operators.dataproc_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">import</span> <span class="nn">time</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_dataproc_hook</span> <span class="k">import</span> <span class="n">DataProcHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.version</span> <span class="k">import</span> <span class="n">version</span>
+<span class="kn">from</span> <span class="nn">googleapiclient.errors</span> <span class="k">import</span> <span class="n">HttpError</span>
+
+
+<span class="k">class</span> <span class="nc">DataprocClusterCreateOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Create a new cluster on Google Cloud Dataproc. The operator will wait until the</span>
+<span class="sd">    creation is successful or an error occurs in the creation process.</span>
+
+<span class="sd">    The parameters allow to configure the cluster. Please refer to</span>
+
+<span class="sd">    https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.clusters</span>
+
+<span class="sd">    for a detailed explanation on the different parameters. Most of the configuration</span>
+<span class="sd">    parameters detailed in the link are available as a parameter to this operator.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;cluster_name&#39;</span><span class="p">,]</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">cluster_name</span><span class="p">,</span>
+                 <span class="n">project_id</span><span class="p">,</span>
+                 <span class="n">num_workers</span><span class="p">,</span>
+                 <span class="n">zone</span><span class="p">,</span>
+                 <span class="n">storage_bucket</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">init_actions_uris</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">metadata</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">image_version</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">properties</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">master_machine_type</span><span class="o">=</span><span class="s1">&#39;n1-standard-4&#39;</span><span class="p">,</span>
+                 <span class="n">master_disk_size</span><span class="o">=</span><span class="mi">500</span><span class="p">,</span>
+                 <span class="n">worker_machine_type</span><span class="o">=</span><span class="s1">&#39;n1-standard-4&#39;</span><span class="p">,</span>
+                 <span class="n">worker_disk_size</span><span class="o">=</span><span class="mi">500</span><span class="p">,</span>
+                 <span class="n">num_preemptible_workers</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+                 <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">region</span><span class="o">=</span><span class="s1">&#39;global&#39;</span><span class="p">,</span>
+                 <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_default&#39;</span><span class="p">,</span>
+                 <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">service_account</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">service_account_scopes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+                 <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Create a new DataprocClusterCreateOperator.</span>
+
+<span class="sd">        For more info on the creation of a cluster through the API, have a look at:</span>
+
+<span class="sd">        https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.clusters</span>
+
+<span class="sd">        :param cluster_name: The name of the DataProc cluster to create.</span>
+<span class="sd">        :type cluster_name: string</span>
+<span class="sd">        :param project_id: The ID of the google cloud project in which</span>
+<span class="sd">            to create the cluster</span>
+<span class="sd">        :type project_id: string</span>
+<span class="sd">        :param num_workers: The # of workers to spin up</span>
+<span class="sd">        :type num_workers: int</span>
+<span class="sd">        :param storage_bucket: The storage bucket to use, setting to None lets dataproc</span>
+<span class="sd">            generate a custom one for you</span>
+<span class="sd">        :type storage_bucket: string</span>
+<span class="sd">        :param init_actions_uris: List of GCS uri&#39;s containing</span>
+<span class="sd">            dataproc initialization scripts</span>
+<span class="sd">        :type init_actions_uris: list[string]</span>
+<span class="sd">        :param metadata: dict of key-value google compute engine metadata entries</span>
+<span class="sd">            to add to all instances</span>
+<span class="sd">        :type metadata: dict</span>
+<span class="sd">        :param image_version: the version of software inside the Dataproc cluster</span>
+<span class="sd">        :type image_version: string</span>
+<span class="sd">        :param properties: dict of properties to set on</span>
+<span class="sd">            config files (e.g. spark-defaults.conf), see</span>
+<span class="sd">            https://cloud.google.com/dataproc/docs/reference/rest/v1/ \</span>
+<span class="sd">            projects.regions.clusters#SoftwareConfig</span>
+<span class="sd">        :type properties: dict</span>
+<span class="sd">        :param master_machine_type: Compute engine machine type to use for the master node</span>
+<span class="sd">        :type master_machine_type: string</span>
+<span class="sd">        :param master_disk_size: Disk size for the master node</span>
+<span class="sd">        :type int</span>
+<span class="sd">        :param worker_machine_type:Compute engine machine type to use for the worker nodes</span>
+<span class="sd">        :type worker_machine_type: string</span>
+<span class="sd">        :param worker_disk_size: Disk size for the worker nodes</span>
+<span class="sd">        :type worker_disk_size: int</span>
+<span class="sd">        :param num_preemptible_workers: The # of preemptible worker nodes to spin up</span>
+<span class="sd">        :type num_preemptible_workers: int</span>
+<span class="sd">        :param labels: dict of labels to add to the cluster</span>
+<span class="sd">        :type labels: dict</span>
+<span class="sd">        :param zone: The zone where the cluster will be located</span>
+<span class="sd">        :type zone: string</span>
+<span class="sd">        :param region: leave as &#39;global&#39;, might become relevant in the future</span>
+<span class="sd">        :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
+<span class="sd">        :type gcp_conn_id: string</span>
+<span class="sd">        :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd">            For this to work, the service account making the request must have domain-wide</span>
+<span class="sd">            delegation enabled.</span>
+<span class="sd">        :type delegate_to: string</span>
+<span class="sd">        :param service_account: The service account of the dataproc instances.</span>
+<span class="sd">        :type service_account: string</span>
+<span class="sd">        :param service_account_scopes: The URIs of service account scopes to be included.</span>
+<span class="sd">        :type service_account_scopes: list[string]</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">DataprocClusterCreateOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span> <span class="o">=</span> <span class="n">cluster_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span> <span class="o">=</span> <span class="n">num_workers</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">num_preemptible_workers</span> <span class="o">=</span> <span class="n">num_preemptible_workers</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">storage_bucket</span> <span class="o">=</span> <span class="n">storage_bucket</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">init_actions_uris</span> <span class="o">=</span> <span class="n">init_actions_uris</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span> <span class="o">=</span> <span class="n">metadata</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">image_version</span> <span class="o">=</span> <span class="n">image_version</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">properties</span> <span class="o">=</span> <span class="n">properties</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">master_machine_type</span> <span class="o">=</span> <span class="n">master_machine_type</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">master_disk_size</span> <span class="o">=</span> <span class="n">master_disk_size</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">worker_machine_type</span> <span class="o">=</span> <span class="n">worker_machine_type</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">worker_disk_size</span> <span class="o">=</span> <span class="n">worker_disk_size</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">labels</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">zone</span> <span class="o">=</span> <span class="n">zone</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">service_account</span> <span class="o">=</span> <span class="n">service_account</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">service_account_scopes</span> <span class="o">=</span> <span class="n">service_account_scopes</span>
+
+    <span class="k">def</span> <span class="nf">_get_cluster_list_for_project</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">service</span><span class="p">):</span>
+        <span class="n">result</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">regions</span><span class="p">()</span><span class="o">.</span><span class="n">clusters</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span>
+            <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+            <span class="n">region</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">region</span>
+        <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;clusters&#39;</span><span class="p">,</span> <span class="p">[])</span>
+
+    <span class="k">def</span> <span class="nf">_get_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">service</span><span class="p">):</span>
+        <span class="n">cluster_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_cluster_list_for_project</span><span class="p">(</span><span class="n">service</span><span class="p">)</span>
+        <span class="n">cluster</span> <span class="o">=</span> <span class="p">[</span><span class="n">c</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">cluster_list</span> <span class="k">if</span> <span class="n">c</span><span class="p">[</span><span class="s1">&#39;clusterName&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span c [...]
+        <span class="k">if</span> <span class="n">cluster</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">cluster</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+        <span class="k">return</span> <span class="kc">None</span>
+
+    <span class="k">def</span> <span class="nf">_get_cluster_state</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">service</span><span class="p">):</span>
+        <span class="n">cluster</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_cluster</span><span class="p">(</span><span class="n">service</span><span class="p">)</span>
+        <span class="k">if</span> <span class="s1">&#39;status&#39;</span> <span class="ow">in</span> <span class="n">cluster</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">cluster</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">][</span><span class="s1">&#39;state&#39;</span><span class="p">]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">None</span>
+
+    <span class="k">def</span> <span class="nf">_cluster_ready</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">service</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">state</span> <span class="o">==</span> <span class="s1">&#39;RUNNING&#39;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">True</span>
+        <span class="k">if</span> <span class="n">state</span> <span class="o">==</span> <span class="s1">&#39;ERROR&#39;</span><span class="p">:</span>
+            <span class="n">cluster</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_cluster</span><span class="p">(</span><span class="n">service</span><span class="p">)</span>
+            <span class="k">try</span><span class="p">:</span>
+                <span class="n">error_details</span> <span class="o">=</span> <span class="n">cluster</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">][</span><span class="s1">&#39;details&#39;</span><span class="p">]</span>
+            <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
+                <span class="n">error_details</span> <span class="o">=</span> <span class="s1">&#39;Unknown error in cluster creation, &#39;</span> \
+                                <span class="s1">&#39;check Google Cloud console for details.&#39;</span>
+            <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="n">error_details</span><span class="p">)</span>
+        <span class="k">return</span> <span class="kc">False</span>
+
+    <span class="k">def</span> <span class="nf">_wait_for_done</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">service</span><span class="p">):</span>
+        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+            <span class="n">state</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_cluster_state</span><span class="p">(</span><span class="n">service</span><span class="p">)</span>
+            <span class="k">if</span> <span class="n">state</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;No state for cluster &#39;</span><span class="si">%s</span><span class="s2">&#39;&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">)</span>
+                <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">15</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;State for cluster &#39;</span><span class="si">%s</span><span class="s2">&#39; is </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">,</span> <span class="n">st [...]
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_cluster_ready</span><span class="p">(</span><span class="n">state</span><span class="p">,</span> <span class="n">service</span><span class="p">):</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                        <span class="s2">&quot;Cluster &#39;</span><span class="si">%s</span><span class="s2">&#39; successfully created&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span>
+                    <span class="p">)</span>
+                    <span class="k">return</span>
+                <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">15</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_build_cluster_data</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">zone_uri</span> <span class="o">=</span> \
+            <span class="s1">&#39;https://www.googleapis.com/compute/v1/projects/</span><span class="si">{}</span><span class="s1">/zones/</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">zone</span>
+            <span class="p">)</span>
+        <span class="n">master_type_uri</span> <span class="o">=</span> \
+            <span class="s2">&quot;https://www.googleapis.com/compute/v1/projects/</span><span class="si">{}</span><span class="s2">/zones/</span><span class="si">{}</span><span class="s2">/machineTypes/</span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">zone</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">master_machine_type</span>
+            <span class="p">)</span>
+        <span class="n">worker_type_uri</span> <span class="o">=</span> \
+            <span class="s2">&quot;https://www.googleapis.com/compute/v1/projects/</span><span class="si">{}</span><span class="s2">/zones/</span><span class="si">{}</span><span class="s2">/machineTypes/</span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">zone</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_machine_type</span>
+            <span class="p">)</span>
+        <span class="n">cluster_data</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+            <span class="s1">&#39;clusterName&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">,</span>
+            <span class="s1">&#39;config&#39;</span><span class="p">:</span> <span class="p">{</span>
+                <span class="s1">&#39;gceClusterConfig&#39;</span><span class="p">:</span> <span class="p">{</span>
+                    <span class="s1">&#39;zoneUri&#39;</span><span class="p">:</span> <span class="n">zone_uri</span>
+                <span class="p">},</span>
+                <span class="s1">&#39;masterConfig&#39;</span><span class="p">:</span> <span class="p">{</span>
+                    <span class="s1">&#39;numInstances&#39;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
+                    <span class="s1">&#39;machineTypeUri&#39;</span><span class="p">:</span> <span class="n">master_type_uri</span><span class="p">,</span>
+                    <span class="s1">&#39;diskConfig&#39;</span><span class="p">:</span> <span class="p">{</span>
+                        <span class="s1">&#39;bootDiskSizeGb&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">master_disk_size</span>
+                    <span class="p">}</span>
+                <span class="p">},</span>
+                <span class="s1">&#39;workerConfig&#39;</span><span class="p">:</span> <span class="p">{</span>
+                    <span class="s1">&#39;numInstances&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="p">,</span>
+                    <span class="s1">&#39;machineTypeUri&#39;</span><span class="p">:</span> <span class="n">worker_type_uri</span><span class="p">,</span>
+                    <span class="s1">&#39;diskConfig&#39;</span><span class="p">:</span> <span class="p">{</span>
+                        <span class="s1">&#39;bootDiskSizeGb&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_disk_size</span>
+                    <span class="p">}</span>
+                <span class="p">},</span>
+                <span class="s1">&#39;secondaryWorkerConfig&#39;</span><span class="p">:</span> <span class="p">{},</span>
+                <span class="s1">&#39;softwareConfig&#39;</span><span class="p">:</span> <span class="p">{}</span>
+            <span class="p">}</span>
+        <span class="p">}</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_preemptible_workers</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="n">cluster_data</span><span class="p">[</span><span class="s1">&#39;config&#39;</span><span class="p">][</span><span class="s1">&#39;secondaryWorkerConfig&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
+                <span class="s1">&#39;numInstances&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_preemptible_workers</span><span class="p">,</span>
+                <span class="s1">&#39;machineTypeUri&#39;</span><span class="p">:</span> <span class="n">worker_type_uri</span><span class="p">,</span>
+                <span class="s1">&#39;diskConfig&#39;</span><span class="p">:</span> <span class="p">{</span>
+                    <span class="s1">&#39;bootDiskSizeGb&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_disk_size</span>
+                <span class="p">},</span>
+                <span class="s1">&#39;isPreemptible&#39;</span><span class="p">:</span> <span class="kc">True</span>
+            <span class="p">}</span>
+
+        <span class="n">cluster_data</span><span class="p">[</span><span class="s1">&#39;labels&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="k">else</span> <span class="p">{}</span>
+        <span class="c1"># Dataproc labels must conform to the following regex:</span>
+        <span class="c1"># [a-z]([-a-z0-9]*[a-z0-9])? (current airflow version string follows</span>
+        <span class="c1"># semantic versioning spec: x.y.z).</span>
+        <span class="n">cluster_data</span><span class="p">[</span><span class="s1">&#39;labels&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s1">&#39;airflow-version&#39;</span><span class="p">:</span>
+                                       <span class="s1">&#39;v&#39;</span> <span class="o">+</span> <span class="n">version</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="s1">&#39;-&#39;</span><span class="p">)})</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">storage_bucket</span><span class="p">:</span>
+            <span class="n">cluster_data</span><span class="p">[</span><span class="s1">&#39;config&#39;</span><span class="p">][</span><span class="s1">&#39;configBucket&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">storage_bucket</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="p">:</span>
+            <span class="n">cluster_data</span><span class="p">[</span><span class="s1">&#39;config&#39;</span><span class="p">][</span><span class="s1">&#39;gceClusterConfig&#39;</span><span class="p">][</span><span class="s1">&#39;metadata&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_version</span><span class="p">:</span>
+            <span class="n">cluster_data</span><span class="p">[</span><span class="s1">&#39;config&#39;</span><span class="p">][</span><span class="s1">&#39;softwareConfig&#39;</span><span class="p">][</span><span class="s1">&#39;imageVersion&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_version</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="p">:</span>
+            <span class="n">cluster_data</span><span class="p">[</span><span class="s1">&#39;config&#39;</span><span class="p">][</span><span class="s1">&#39;softwareConfig&#39;</span><span class="p">][</span><span class="s1">&#39;properties&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">properties</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">init_actions_uris</span><span class="p">:</span>
+            <span class="n">init_actions_dict</span> <span class="o">=</span> <span class="p">[</span>
+                <span class="p">{</span><span class="s1">&#39;executableFile&#39;</span><span class="p">:</span> <span class="n">uri</span><span class="p">}</span> <span class="k">for</span> <span class="n">uri</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">init_actions_uris</span>
+            <span class="p">]</span>
+            <span class="n">cluster_data</span><span class="p">[</span><span class="s1">&#39;config&#39;</span><span class="p">][</span><span class="s1">&#39;initializationActions&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">init_actions_dict</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">service_account</span><span class="p">:</span>
+            <span class="n">cluster_data</span><span class="p">[</span><span class="s1">&#39;config&#39;</span><span class="p">][</span><span class="s1">&#39;gceClusterConfig&#39;</span><span class="p">][</span><span class="s1">&#39;serviceAccount&#39;</span><span class="p">]</span> <span class="o">=</span>\
+                    <span class="bp">self</span><span class="o">.</span><span class="n">service_account</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">service_account_scopes</span><span class="p">:</span>
+            <span class="n">cluster_data</span><span class="p">[</span><span class="s1">&#39;config&#39;</span><span class="p">][</span><span class="s1">&#39;gceClusterConfig&#39;</span><span class="p">][</span><span class="s1">&#39;serviceAccountScopes&#39;</span><span class="p">]</span> <span class="o">=</span>\
+                    <span class="bp">self</span><span class="o">.</span><span class="n">service_account_scopes</span>
+        <span class="k">return</span> <span class="n">cluster_data</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Creating cluster: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">)</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span>
+            <span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+            <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span>
+        <span class="p">)</span>
+        <span class="n">service</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_cluster</span><span class="p">(</span><span class="n">service</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                <span class="s1">&#39;Cluster </span><span class="si">%s</span><span class="s1"> already exists... Checking status...&#39;</span><span class="p">,</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span>
+            <span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">)</span>
+            <span class="k">return</span> <span class="kc">True</span>
+
+        <span class="n">cluster_data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_cluster_data</span><span class="p">()</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">service</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">regions</span><span class="p">()</span><span class="o">.</span><span class="n">clusters</span><span class="p">()</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
+                <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                <span class="n">region</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">,</span>
+                <span class="n">body</span><span class="o">=</span><span class="n">cluster_data</span>
+            <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="c1"># probably two cluster start commands at the same time</span>
+            <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_cluster</span><span class="p">(</span><span class="n">service</span><span class="p">):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                    <span class="s1">&#39;Cluster </span><span class="si">{}</span><span class="s1"> already exists... Checking status...&#39;</span><span class="p">,</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span>
+                 <span class="p">)</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">)</span>
+                <span class="k">return</span> <span class="kc">True</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="n">e</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">DataprocClusterDeleteOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Delete a cluster on Google Cloud Dataproc. The operator will wait until the</span>
+<span class="sd">    cluster is destroyed.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;cluster_name&#39;</span><span class="p">]</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">cluster_name</span><span class="p">,</span>
+                 <span class="n">project_id</span><span class="p">,</span>
+                 <span class="n">region</span><span class="o">=</span><span class="s1">&#39;global&#39;</span><span class="p">,</span>
+                 <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_default&#39;</span><span class="p">,</span>
+                 <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+                 <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Delete a cluster on Google Cloud Dataproc.</span>
+
+<span class="sd">        :param cluster_name: The name of the cluster to create.</span>
+<span class="sd">        :type cluster_name: string</span>
+<span class="sd">        :param project_id: The ID of the google cloud project in which</span>
+<span class="sd">            the cluster runs</span>
+<span class="sd">        :type project_id: string</span>
+<span class="sd">        :param region: leave as &#39;global&#39;, might become relevant in the future</span>
+<span class="sd">        :type region: string</span>
+<span class="sd">        :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
+<span class="sd">        :type gcp_conn_id: string</span>
+<span class="sd">        :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd">            For this to work, the service account making the request must have domain-wide</span>
+<span class="sd">            delegation enabled.</span>
+<span class="sd">        :type delegate_to: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">DataprocClusterDeleteOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span> <span class="o">=</span> <span class="n">cluster_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span>
+
+    <span class="k">def</span> <span class="nf">_wait_for_done</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">service</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">):</span>
+        <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">15</span><span class="p">)</span>
+        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+            <span class="n">response</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">regions</span><span class="p">()</span><span class="o">.</span><span class="n">operations</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+                <span class="n">name</span><span class="o">=</span><span class="n">operation_name</span>
+            <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+            <span class="k">if</span> <span class="s1">&#39;done&#39;</span> <span class="ow">in</span> <span class="n">response</span> <span class="ow">and</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;done&#39;</span><span class="p">]:</span>
+                <span class="k">if</span> <span class="s1">&#39;error&#39;</span> <span class="ow">in</span> <span class="n">response</span><span class="p">:</span>
+                    <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;error&#39;</span><span class="p">]))</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="k">return</span>
+            <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">15</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Deleting cluster: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">)</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span>
+            <span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+            <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span>
+        <span class="p">)</span>
+        <span class="n">service</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+        <span class="n">response</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">regions</span><span class="p">()</span><span class="o">.</span><span class="n">clusters</span><span class="p">()</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span>
+            <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+            <span class="n">region</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">,</span>
+            <span class="n">clusterName</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span>
+        <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="n">operation_name</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Cluster delete operation name: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="DataProcPigOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcPigOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcPigOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Start a Pig query Job on a Cloud DataProc cluster. The parameters of the operation</span>
+<span class="sd">    will be passed to the cluster.</span>
+
+<span class="sd">    It&#39;s a good practice to define dataproc_* parameters in the default_args of the dag</span>
+<span class="sd">    like the cluster name and UDFs.</span>
+
+<span class="sd">    ```</span>
+<span class="sd">    default_args = {</span>
+<span class="sd">        &#39;cluster_name&#39;: &#39;cluster-1&#39;,</span>
+<span class="sd">        &#39;dataproc_pig_jars&#39;: [</span>
+<span class="sd">            &#39;gs://example/udf/jar/datafu/1.2.0/datafu.jar&#39;,</span>
+<span class="sd">            &#39;gs://example/udf/jar/gpig/1.2/gpig.jar&#39;</span>
+<span class="sd">        ]</span>
+<span class="sd">    }</span>
+<span class="sd">    ```</span>
+
+<span class="sd">    You can pass a pig script as string or file reference. Use variables to pass on</span>
+<span class="sd">    variables for the pig script to be resolved on the cluster or use the parameters to</span>
+<span class="sd">    be resolved in the script as template parameters.</span>
+
+<span class="sd">    ```</span>
+<span class="sd">    t1 = DataProcPigOperator(</span>
+<span class="sd">        task_id=&#39;dataproc_pig&#39;,</span>
+<span class="sd">        query=&#39;a_pig_script.pig&#39;,</span>
+<span class="sd">        variables={&#39;out&#39;: &#39;gs://example/output/{{ds}}&#39;},</span>
+<span class="sd">    dag=dag)</span>
+<span class="sd">    ```</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">,</span> <span class="s1">&#39;variables&#39;</span><span class="p">,</span> <span class="s1">&#39;job_name&#39;</span><span class="p">,</span> <span class="s1">&#39;cluster_name&#39;</span><span class="p">]</span>
+    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.pg&#39;</span><span class="p">,</span> <span class="s1">&#39;.pig&#39;</span><span class="p">,)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#0273d4&#39;</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">query</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">query_uri</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">variables</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">job_name</span><span class="o">=</span><span class="s1">&#39;{{task.task_id}}_{{ds_nodash}}&#39;</span><span class="p">,</span>
+            <span class="n">cluster_name</span><span class="o">=</span><span class="s1">&#39;cluster-1&#39;</span><span class="p">,</span>
+            <span class="n">dataproc_pig_properties</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">dataproc_pig_jars</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_default&#39;</span><span class="p">,</span>
+            <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Create a new DataProcPigOperator.</span>
+
+<span class="sd">        For more detail on about job submission have a look at the reference:</span>
+
+<span class="sd">        https://cloud.google.com/dataproc/reference/rest/v1/projects.regions.jobs</span>
+
+<span class="sd">        :param query: The query or reference to the query file (pg or pig extension).</span>
+<span class="sd">        :type query: string</span>
+<span class="sd">        :param query_uri: The uri of a pig script on Cloud Storage.</span>
+<span class="sd">        :type query_uri: string</span>
+<span class="sd">        :param variables: Map of named parameters for the query.</span>
+<span class="sd">        :type variables: dict</span>
+<span class="sd">        :param job_name: The job name used in the DataProc cluster. This name by default</span>
+<span class="sd">            is the task_id appended with the execution data, but can be templated. The</span>
+<span class="sd">            name will always be appended with a random number to avoid name clashes.</span>
+<span class="sd">        :type job_name: string</span>
+<span class="sd">        :param cluster_name: The name of the DataProc cluster.</span>
+<span class="sd">        :type cluster_name: string</span>
+<span class="sd">        :param dataproc_pig_properties: Map for the Pig properties. Ideal to put in</span>
+<span class="sd">            default arguments</span>
+<span class="sd">        :type dataproc_pig_properties: dict</span>
+<span class="sd">        :param dataproc_pig_jars: URIs to jars provisioned in Cloud Storage (example: for</span>
+<span class="sd">            UDFs and libs) and are ideal to put in default arguments.</span>
+<span class="sd">        :type dataproc_pig_jars: list</span>
+<span class="sd">        :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
+<span class="sd">        :type gcp_conn_id: string</span>
+<span class="sd">        :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd">            For this to work, the service account making the request must have domain-wide</span>
+<span class="sd">            delegation enabled.</span>
+<span class="sd">        :type delegate_to: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">DataProcPigOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">query</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">query_uri</span> <span class="o">=</span> <span class="n">query_uri</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">variables</span> <span class="o">=</span> <span class="n">variables</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">job_name</span> <span class="o">=</span> <span class="n">job_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span> <span class="o">=</span> <span class="n">cluster_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_properties</span> <span class="o">=</span> <span class="n">dataproc_pig_properties</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_pig_jars</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+                            <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+        <span class="n">job</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">create_job_template</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">,</span> <span class="s2">&quot;pigJob&quot;</span><span class="p">,</span>
+                                       <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_properties</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">query</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">job</span><span class="o">.</span><span class="n">add_query_uri</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">query_uri</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">job</span><span class="o">.</span><span class="n">add_query</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">query</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_variables</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">variables</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_jar_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span>
+
+        <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">())</span></div>
+
+
+<div class="viewcode-block" id="DataProcHiveOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcHiveOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcHiveOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Start a Hive query Job on a Cloud DataProc cluster.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">,</span> <span class="s1">&#39;variables&#39;</span><span class="p">,</span> <span class="s1">&#39;job_name&#39;</span><span class="p">,</span> <span class="s1">&#39;cluster_name&#39;</span><span class="p">]</span>
+    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.q&#39;</span><span class="p">,)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#0273d4&#39;</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">query</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">query_uri</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">variables</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">job_name</span><span class="o">=</span><span class="s1">&#39;{{task.task_id}}_{{ds_nodash}}&#39;</span><span class="p">,</span>
+            <span class="n">cluster_name</span><span class="o">=</span><span class="s1">&#39;cluster-1&#39;</span><span class="p">,</span>
+            <span class="n">dataproc_hive_properties</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">dataproc_hive_jars</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_default&#39;</span><span class="p">,</span>
+            <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Create a new DataProcHiveOperator.</span>
+
+<span class="sd">        :param query: The query or reference to the query file (q extension).</span>
+<span class="sd">        :type query: string</span>
+<span class="sd">        :param query_uri: The uri of a hive script on Cloud Storage.</span>
+<span class="sd">        :type query_uri: string</span>
+<span class="sd">        :param variables: Map of named parameters for the query.</span>
+<span class="sd">        :type variables: dict</span>
+<span class="sd">        :param job_name: The job name used in the DataProc cluster. This name by default</span>
+<span class="sd">            is the task_id appended with the execution data, but can be templated. The</span>
+<span class="sd">            name will always be appended with a random number to avoid name clashes.</span>
+<span class="sd">        :type job_name: string</span>
+<span class="sd">        :param cluster_name: The name of the DataProc cluster.</span>
+<span class="sd">        :type cluster_name: string</span>
+<span class="sd">        :param dataproc_hive_properties: Map for the Pig properties. Ideal to put in</span>
+<span class="sd">            default arguments</span>
+<span class="sd">        :type dataproc_hive_properties: dict</span>
+<span class="sd">        :param dataproc_hive_jars: URIs to jars provisioned in Cloud Storage (example: for</span>
+<span class="sd">            UDFs and libs) and are ideal to put in default arguments.</span>
+<span class="sd">        :type dataproc_hive_jars: list</span>
+<span class="sd">        :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
+<span class="sd">        :type gcp_conn_id: string</span>
+<span class="sd">        :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd">            For this to work, the service account making the request must have domain-wide</span>
+<span class="sd">            delegation enabled.</span>
+<span class="sd">        :type delegate_to: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">DataProcHiveOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">query</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">query_uri</span> <span class="o">=</span> <span class="n">query_uri</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">variables</span> <span class="o">=</span> <span class="n">variables</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">job_name</span> <span class="o">=</span> <span class="n">job_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span> <span class="o">=</span> <span class="n">cluster_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_properties</span> <span class="o">=</span> <span class="n">dataproc_hive_properties</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_hive_jars</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+                            <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+
+        <span class="n">job</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">create_job_template</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">,</span> <span class="s2">&quot;hiveJob&quot;</span><span class="p">,</span>
+                                       <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_properties</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">query</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">job</span><span class="o">.</span><span class="n">add_query_uri</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">query_uri</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">job</span><span class="o">.</span><span class="n">add_query</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">query</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_variables</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">variables</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_jar_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span>
+
+        <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">())</span></div>
+
+
+<div class="viewcode-block" id="DataProcSparkSqlOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcSparkSqlOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcSparkSqlOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Start a Spark SQL query Job on a Cloud DataProc cluster.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">,</span> <span class="s1">&#39;variables&#39;</span><span class="p">,</span> <span class="s1">&#39;job_name&#39;</span><span class="p">,</span> <span class="s1">&#39;cluster_name&#39;</span><span class="p">]</span>
+    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.q&#39;</span><span class="p">,)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#0273d4&#39;</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">query</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">query_uri</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">variables</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">job_name</span><span class="o">=</span><span class="s1">&#39;{{task.task_id}}_{{ds_nodash}}&#39;</span><span class="p">,</span>
+            <span class="n">cluster_name</span><span class="o">=</span><span class="s1">&#39;cluster-1&#39;</span><span class="p">,</span>
+            <span class="n">dataproc_spark_properties</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">dataproc_spark_jars</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_default&#39;</span><span class="p">,</span>
+            <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Create a new DataProcSparkSqlOperator.</span>
+
+<span class="sd">        :param query: The query or reference to the query file (q extension).</span>
+<span class="sd">        :type query: string</span>
+<span class="sd">        :param query_uri: The uri of a spark sql script on Cloud Storage.</span>
+<span class="sd">        :type query_uri: string</span>
+<span class="sd">        :param variables: Map of named parameters for the query.</span>
+<span class="sd">        :type variables: dict</span>
+<span class="sd">        :param job_name: The job name used in the DataProc cluster. This name by default</span>
+<span class="sd">            is the task_id appended with the execution data, but can be templated. The</span>
+<span class="sd">            name will always be appended with a random number to avoid name clashes.</span>
+<span class="sd">        :type job_name: string</span>
+<span class="sd">        :param cluster_name: The name of the DataProc cluster.</span>
+<span class="sd">        :type cluster_name: string</span>
+<span class="sd">        :param dataproc_spark_properties: Map for the Pig properties. Ideal to put in</span>
+<span class="sd">            default arguments</span>
+<span class="sd">        :type dataproc_spark_properties: dict</span>
+<span class="sd">        :param dataproc_spark_jars: URIs to jars provisioned in Cloud Storage (example:</span>
+<span class="sd">            for UDFs and libs) and are ideal to put in default arguments.</span>
+<span class="sd">        :type dataproc_spark_jars: list</span>
+<span class="sd">        :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
+<span class="sd">        :type gcp_conn_id: string</span>
+<span class="sd">        :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd">            For this to work, the service account making the request must have domain-wide</span>
+<span class="sd">            delegation enabled.</span>
+<span class="sd">        :type delegate_to: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">DataProcSparkSqlOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">query</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">query_uri</span> <span class="o">=</span> <span class="n">query_uri</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">variables</span> <span class="o">=</span> <span class="n">variables</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">job_name</span> <span class="o">=</span> <span class="n">job_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span> <span class="o">=</span> <span class="n">cluster_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_properties</span> <span class="o">=</span> <span class="n">dataproc_spark_properties</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_spark_jars</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+                            <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+
+        <span class="n">job</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">create_job_template</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">,</span> <span class="s2">&quot;sparkSqlJob&quot;</span><span class="p">,</span>
+                                       <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_properties</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">query</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">job</span><span class="o">.</span><span class="n">add_query_uri</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">query_uri</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">job</span><span class="o">.</span><span class="n">add_query</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">query</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_variables</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">variables</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_jar_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span>
+
+        <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">())</span></div>
+
+
+<div class="viewcode-block" id="DataProcSparkOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcSparkOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcSparkOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Start a Spark Job on a Cloud DataProc cluster.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;arguments&#39;</span><span class="p">,</span> <span class="s1">&#39;job_name&#39;</span><span class="p">,</span> <span class="s1">&#39;cluster_name&#39;</span><span class="p">]</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#0273d4&#39;</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">main_jar</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">main_class</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">arguments</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">archives</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">files</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">job_name</span><span class="o">=</span><span class="s1">&#39;{{task.task_id}}_{{ds_nodash}}&#39;</span><span class="p">,</span>
+            <span class="n">cluster_name</span><span class="o">=</span><span class="s1">&#39;cluster-1&#39;</span><span class="p">,</span>
+            <span class="n">dataproc_spark_properties</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">dataproc_spark_jars</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_default&#39;</span><span class="p">,</span>
+            <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Create a new DataProcSparkOperator.</span>
+
+<span class="sd">        :param main_jar: URI of the job jar provisioned on Cloud Storage. (use this or</span>
+<span class="sd">            the main_class, not both together).</span>
+<span class="sd">        :type main_jar: string</span>
+<span class="sd">        :param main_class: Name of the job class. (use this or the main_jar, not both</span>
+<span class="sd">            together).</span>
+<span class="sd">        :type main_class: string</span>
+<span class="sd">        :param arguments: Arguments for the job.</span>
+<span class="sd">        :type arguments: list</span>
+<span class="sd">        :param archives: List of archived files that will be unpacked in the work</span>
+<span class="sd">            directory. Should be stored in Cloud Storage.</span>
+<span class="sd">        :type archives: list</span>
+<span class="sd">        :param files: List of files to be copied to the working directory</span>
+<span class="sd">        :type files: list</span>
+<span class="sd">        :param job_name: The job name used in the DataProc cluster. This name by default</span>
+<span class="sd">            is the task_id appended with the execution data, but can be templated. The</span>
+<span class="sd">            name will always be appended with a random number to avoid name clashes.</span>
+<span class="sd">        :type job_name: string</span>
+<span class="sd">        :param cluster_name: The name of the DataProc cluster.</span>
+<span class="sd">        :type cluster_name: string</span>
+<span class="sd">        :param dataproc_spark_properties: Map for the Pig properties. Ideal to put in</span>
+<span class="sd">            default arguments</span>
+<span class="sd">        :type dataproc_spark_properties: dict</span>
+<span class="sd">        :param dataproc_spark_jars: URIs to jars provisioned in Cloud Storage (example:</span>
+<span class="sd">            for UDFs and libs) and are ideal to put in default arguments.</span>
+<span class="sd">        :type dataproc_spark_jars: list</span>
+<span class="sd">        :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
+<span class="sd">        :type gcp_conn_id: string</span>
+<span class="sd">        :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd">            For this to work, the service account making the request must have domain-wide</span>
+<span class="sd">            delegation enabled.</span>
+<span class="sd">        :type delegate_to: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">DataProcSparkOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">main_jar</span> <span class="o">=</span> <span class="n">main_jar</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">main_class</span> <span class="o">=</span> <span class="n">main_class</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">arguments</span> <span class="o">=</span> <span class="n">arguments</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">archives</span> <span class="o">=</span> <span class="n">archives</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">files</span> <span class="o">=</span> <span class="n">files</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">job_name</span> <span class="o">=</span> <span class="n">job_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span> <span class="o">=</span> <span class="n">cluster_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_properties</span> <span class="o">=</span> <span class="n">dataproc_spark_properties</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_spark_jars</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+                            <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+        <span class="n">job</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">create_job_template</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">,</span> <span class="s2">&quot;sparkJob&quot;</span><span class="p">,</span>
+                                       <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_properties</span><span class="p">)</span>
+
+        <span class="n">job</span><span class="o">.</span><span class="n">set_main</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">main_jar</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">main_class</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_args</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">arguments</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_jar_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_archive_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">archives</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span>
+
+        <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">())</span></div>
+
+
+<div class="viewcode-block" id="DataProcHadoopOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcHadoopOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcHadoopOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Start a Hadoop Job on a Cloud DataProc cluster.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;arguments&#39;</span><span class="p">,</span> <span class="s1">&#39;job_name&#39;</span><span class="p">,</span> <span class="s1">&#39;cluster_name&#39;</span><span class="p">]</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#0273d4&#39;</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">main_jar</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">main_class</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">arguments</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">archives</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">files</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">job_name</span><span class="o">=</span><span class="s1">&#39;{{task.task_id}}_{{ds_nodash}}&#39;</span><span class="p">,</span>
+            <span class="n">cluster_name</span><span class="o">=</span><span class="s1">&#39;cluster-1&#39;</span><span class="p">,</span>
+            <span class="n">dataproc_hadoop_properties</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">dataproc_hadoop_jars</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_default&#39;</span><span class="p">,</span>
+            <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Create a new DataProcHadoopOperator.</span>
+
+<span class="sd">        :param main_jar: URI of the job jar provisioned on Cloud Storage. (use this or</span>
+<span class="sd">            the main_class, not both together).</span>
+<span class="sd">        :type main_jar: string</span>
+<span class="sd">        :param main_class: Name of the job class. (use this or the main_jar, not both</span>
+<span class="sd">            together).</span>
+<span class="sd">        :type main_class: string</span>
+<span class="sd">        :param arguments: Arguments for the job.</span>
+<span class="sd">        :type arguments: list</span>
+<span class="sd">        :param archives: List of archived files that will be unpacked in the work</span>
+<span class="sd">            directory. Should be stored in Cloud Storage.</span>
+<span class="sd">        :type archives: list</span>
+<span class="sd">        :param files: List of files to be copied to the working directory</span>
+<span class="sd">        :type files: list</span>
+<span class="sd">        :param job_name: The job name used in the DataProc cluster. This name by default</span>
+<span class="sd">            is the task_id appended with the execution data, but can be templated. The</span>
+<span class="sd">            name will always be appended with a random number to avoid name clashes.</span>
+<span class="sd">        :type job_name: string</span>
+<span class="sd">        :param cluster_name: The name of the DataProc cluster.</span>
+<span class="sd">        :type cluster_name: string</span>
+<span class="sd">        :param dataproc_hadoop_properties: Map for the Pig properties. Ideal to put in</span>
+<span class="sd">            default arguments</span>
+<span class="sd">        :type dataproc_hadoop_properties: dict</span>
+<span class="sd">        :param dataproc_hadoop_jars: URIs to jars provisioned in Cloud Storage (example:</span>
+<span class="sd">            for UDFs and libs) and are ideal to put in default arguments.</span>
+<span class="sd">        :type dataproc_hadoop_jars: list</span>
+<span class="sd">        :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
+<span class="sd">        :type gcp_conn_id: string</span>
+<span class="sd">        :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd">            For this to work, the service account making the request must have domain-wide</span>
+<span class="sd">            delegation enabled.</span>
+<span class="sd">        :type delegate_to: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">DataProcHadoopOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">main_jar</span> <span class="o">=</span> <span class="n">main_jar</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">main_class</span> <span class="o">=</span> <span class="n">main_class</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">arguments</span> <span class="o">=</span> <span class="n">arguments</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">archives</span> <span class="o">=</span> <span class="n">archives</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">files</span> <span class="o">=</span> <span class="n">files</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">job_name</span> <span class="o">=</span> <span class="n">job_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span> <span class="o">=</span> <span class="n">cluster_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_properties</span> <span class="o">=</span> <span class="n">dataproc_hadoop_properties</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_hadoop_jars</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+                            <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+        <span class="n">job</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">create_job_template</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">,</span> <span class="s2">&quot;hadoopJob&quot;</span><span class="p">,</span>
+                                       <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_properties</span><span class="p">)</span>
+
+        <span class="n">job</span><span class="o">.</span><span class="n">set_main</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">main_jar</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">main_class</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_args</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">arguments</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_jar_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_archive_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">archives</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span>
+
+        <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">())</span></div>
+
+
+<div class="viewcode-block" id="DataProcPySparkOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcPySparkOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcPySparkOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Start a PySpark Job on a Cloud DataProc cluster.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;arguments&#39;</span><span class="p">,</span> <span class="s1">&#39;job_name&#39;</span><span class="p">,</span> <span class="s1">&#39;cluster_name&#39;</span><span class="p">]</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#0273d4&#39;</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">main</span><span class="p">,</span>
+            <span class="n">arguments</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">archives</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">pyfiles</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">files</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">job_name</span><span class="o">=</span><span class="s1">&#39;{{task.task_id}}_{{ds_nodash}}&#39;</span><span class="p">,</span>
+            <span class="n">cluster_name</span><span class="o">=</span><span class="s1">&#39;cluster-1&#39;</span><span class="p">,</span>
+            <span class="n">dataproc_pyspark_properties</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">dataproc_pyspark_jars</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_default&#39;</span><span class="p">,</span>
+            <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Create a new DataProcPySparkOperator.</span>
+
+<span class="sd">        :param main: [Required] The Hadoop Compatible Filesystem (HCFS) URI of the main</span>
+<span class="sd">            Python file to use as the driver. Must be a .py file.</span>
+<span class="sd">        :type main: string</span>
+<span class="sd">        :param arguments: Arguments for the job.</span>
+<span class="sd">        :type arguments: list</span>
+<span class="sd">        :param archives: List of archived files that will be unpacked in the work</span>
+<span class="sd">            directory. Should be stored in Cloud Storage.</span>
+<span class="sd">        :type archives: list</span>
+<span class="sd">        :param files: List of files to be copied to the working directory</span>
+<span class="sd">        :type files: list</span>
+<span class="sd">        :param pyfiles: List of Python files to pass to the PySpark framework.</span>
+<span class="sd">            Supported file types: .py, .egg, and .zip</span>
+<span class="sd">        :type pyfiles: list</span>
+<span class="sd">        :param job_name: The job name used in the DataProc cluster. This name by default</span>
+<span class="sd">            is the task_id appended with the execution data, but can be templated. The</span>
+<span class="sd">            name will always be appended with a random number to avoid name clashes.</span>
+<span class="sd">        :type job_name: string</span>
+<span class="sd">        :param cluster_name: The name of the DataProc cluster.</span>
+<span class="sd">        :type cluster_name: string</span>
+<span class="sd">        :param dataproc_pyspark_properties: Map for the Pig properties. Ideal to put in</span>
+<span class="sd">            default arguments</span>
+<span class="sd">        :type dataproc_pyspark_properties: dict</span>
+<span class="sd">        :param dataproc_pyspark_jars: URIs to jars provisioned in Cloud Storage (example:</span>
+<span class="sd">            for UDFs and libs) and are ideal to put in default arguments.</span>
+<span class="sd">        :type dataproc_pyspark_jars: list</span>
+<span class="sd">        :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
+<span class="sd">        :type gcp_conn_id: string</span>
+<span class="sd">        :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd">            For this to work, the service account making the request must have</span>
+<span class="sd">            domain-wide delegation enabled.</span>
+<span class="sd">        :type delegate_to: string</span>
+<span class="sd">         &quot;&quot;&quot;</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">DataProcPySparkOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">main</span> <span class="o">=</span> <span class="n">main</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">arguments</span> <span class="o">=</span> <span class="n">arguments</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">archives</span> <span class="o">=</span> <span class="n">archives</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">files</span> <span class="o">=</span> <span class="n">files</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">pyfiles</span> <span class="o">=</span> <span class="n">pyfiles</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">job_name</span> <span class="o">=</span> <span class="n">job_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span> <span class="o">=</span> <span class="n">cluster_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_properties</span> <span class="o">=</span> <span class="n">dataproc_pyspark_properties</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_pyspark_jars</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+                            <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+        <span class="n">job</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">create_job_template</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">,</span> <span class="s2">&quot;pysparkJob&quot;</span><span class="p">,</span>
+                                       <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_properties</span><span class="p">)</span>
+
+        <span class="n">job</span><span class="o">.</span><span class="n">set_python_main</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">main</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_args</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">arguments</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_jar_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_archive_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">archives</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">add_python_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">pyfiles</span><span class="p">)</span>
+        <span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span>
+
+        <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">())</span></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/operators/ecs_operator.html b/_modules/airflow/contrib/operators/ecs_operator.html
new file mode 100644
index 0000000..5c181c5
--- /dev/null
+++ b/_modules/airflow/contrib/operators/ecs_operator.html
@@ -0,0 +1,360 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.operators.ecs_operator &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.operators.ecs_operator</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.operators.ecs_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="kn">import</span> <span class="nn">sys</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
+
+
+<div class="viewcode-block" id="ECSOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.ecs_operator.ECSOperator">[docs]</a><span class="k">class</span> <span class="nc">ECSOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Execute a task on AWS EC2 Container Service</span>
+
+<span class="sd">    :param task_definition: the task definition name on EC2 Container Service</span>
+<span class="sd">    :type task_definition: str</span>
+<span class="sd">    :param cluster: the cluster name on EC2 Container Service</span>
+<span class="sd">    :type cluster: str</span>
+<span class="sd">    :param: overrides: the same parameter that boto3 will receive:</span>
+<span class="sd">            http://boto3.readthedocs.org/en/latest/reference/services/ecs.html#ECS.Client.run_task</span>
+<span class="sd">    :type: overrides: dict</span>
+<span class="sd">    :param aws_conn_id: connection id of AWS credentials / region name. If None,</span>
+<span class="sd">            credential boto3 strategy will be used (http://boto3.readthedocs.io/en/latest/guide/configuration.html).</span>
+<span class="sd">    :type aws_conn_id: str</span>
+<span class="sd">    :param region_name: region name to use in AWS Hook. Override the region_name in connection (if provided)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#f0ede4&#39;</span>
+    <span class="n">client</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="n">arn</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;overrides&#39;</span><span class="p">,)</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_definition</span><span class="p">,</span> <span class="n">cluster</span><span class="p">,</span> <span class="n">overrides</span><span class="p">,</span>
+                 <span class="n">aws_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">ECSOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span> <span class="o">=</span> <span class="n">region_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_definition</span> <span class="o">=</span> <span class="n">task_definition</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cluster</span> <span class="o">=</span> <span class="n">cluster</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">overrides</span> <span class="o">=</span> <span class="n">overrides</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+            <span class="s1">&#39;Running ECS Task - Task definition: </span><span class="si">%s</span><span class="s1"> - on cluster </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">task_definition</span><span class="p">,</span><span class="bp">self</span><span class="o">.</span><span class="n">cluster</span>
+        <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;ECSOperator overrides: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">overrides</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span>
+            <span class="s1">&#39;ecs&#39;</span><span class="p">,</span>
+            <span class="n">region_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">region_name</span>
+        <span class="p">)</span>
+
+        <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">run_task</span><span class="p">(</span>
+            <span class="n">cluster</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cluster</span><span class="p">,</span>
+            <span class="n">taskDefinition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">task_definition</span><span class="p">,</span>
+            <span class="n">overrides</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">overrides</span><span class="p">,</span>
+            <span class="n">startedBy</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">owner</span>
+        <span class="p">)</span>
+
+        <span class="n">failures</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;failures&#39;</span><span class="p">]</span>
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">failures</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;ECS Task started: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">response</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">arn</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;tasks&#39;</span><span class="p">][</span><span class="mi">0</span><span class="p">][</span><span class="s1">&#39;taskArn&#39;</span><span class="p">]</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_task_ended</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">_check_success_task</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;ECS Task has been successfully executed: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">response</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_wait_for_task_ended</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">waiter</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">get_waiter</span><span class="p">(</span><span class="s1">&#39;tasks_stopped&#39;</span><span class="p">)</span>
+        <span class="n">waiter</span><span class="o">.</span><span class="n">config</span><span class="o">.</span><span class="n">max_attempts</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">maxsize</span>  <span class="c1"># timeout is managed by airflow</span>
+        <span class="n">waiter</span><span class="o">.</span><span class="n">wait</span><span class="p">(</span>
+            <span class="n">cluster</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cluster</span><span class="p">,</span>
+            <span class="n">tasks</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">arn</span><span class="p">]</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_check_success_task</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">describe_tasks</span><span class="p">(</span>
+            <span class="n">cluster</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cluster</span><span class="p">,</span>
+            <span class="n">tasks</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">arn</span><span class="p">]</span>
+        <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;ECS Task stopped, check status: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">response</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;failures&#39;</span><span class="p">,</span> <span class="p">[]))</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;tasks&#39;</span><span class="p">]:</span>
+            <span class="n">containers</span> <span class="o">=</span> <span class="n">task</span><span class="p">[</span><span class="s1">&#39;containers&#39;</span><span class="p">]</span>
+            <span class="k">for</span> <span class="n">container</span> <span class="ow">in</span> <span class="n">containers</span><span class="p">:</span>
+                <span class="k">if</span> <span class="n">container</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;lastStatus&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s1">&#39;STOPPED&#39;</span> <span class="ow">and</span> <span class="n">container</span><span class="p">[</span><span class="s1">&#39;exitCode&#39;</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span [...]
+                    <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">&#39;This task is not in success state </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">task</span><span class="p">))</span>
+                <span class="k">elif</span> <span class="n">container</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;lastStatus&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s1">&#39;PENDING&#39;</span><span class="p">:</span>
+                    <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">&#39;This task is still pending </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">task</span><span class="p">))</span>
+                <span class="k">elif</span> <span class="s1">&#39;error&#39;</span> <span class="ow">in</span> <span class="n">container</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;reason&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">():</span>
+                    <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">&#39;This containers encounter an error during launching : </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span>
+                                           <span class="nb">format</span><span class="p">(</span><span class="n">container</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;reason&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()))</span>
+
+    <span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="n">AwsHook</span><span class="p">(</span>
+            <span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">stop_task</span><span class="p">(</span>
+            <span class="n">cluster</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cluster</span><span class="p">,</span>
+            <span class="n">task</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">arn</span><span class="p">,</span>
+            <span class="n">reason</span><span class="o">=</span><span class="s1">&#39;Task killed by the user&#39;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">response</span><span class="p">)</span></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/operators/emr_add_steps_operator.html b/_modules/airflow/contrib/operators/emr_add_steps_operator.html
new file mode 100644
index 0000000..daac24b
--- /dev/null
+++ b/_modules/airflow/contrib/operators/emr_add_steps_operator.html
@@ -0,0 +1,291 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.operators.emr_add_steps_operator &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.operators.emr_add_steps_operator</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.operators.emr_add_steps_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.emr_hook</span> <span class="k">import</span> <span class="n">EmrHook</span>
+
+
+<div class="viewcode-block" id="EmrAddStepsOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.emr_add_steps_operator.EmrAddStepsOperator">[docs]</a><span class="k">class</span> <span class="nc">EmrAddStepsOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    An operator that adds steps to an existing EMR job_flow.</span>
+
+<span class="sd">    :param job_flow_id: id of the JobFlow to add steps to</span>
+<span class="sd">    :type job_flow_name: str</span>
+<span class="sd">    :param aws_conn_id: aws connection to uses</span>
+<span class="sd">    :type aws_conn_id: str</span>
+<span class="sd">    :param steps: boto3 style steps to be added to the jobflow</span>
+<span class="sd">    :type steps: list</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;job_flow_id&#39;</span><span class="p">]</span>
+    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">()</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#f9c915&#39;</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">job_flow_id</span><span class="p">,</span>
+            <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">&#39;s3_default&#39;</span><span class="p">,</span>
+            <span class="n">steps</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">EmrAddStepsOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="n">steps</span> <span class="o">=</span> <span class="n">steps</span> <span class="ow">or</span> <span class="p">[]</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">job_flow_id</span> <span class="o">=</span> <span class="n">job_flow_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">steps</span> <span class="o">=</span> <span class="n">steps</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="n">emr</span> <span class="o">=</span> <span class="n">EmrHook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Adding steps to </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_flow_id</span><span class="p">)</span>
+        <span class="n">response</span> <span class="o">=</span> <span class="n">emr</span><span class="o">.</span><span class="n">add_job_flow_steps</span><span class="p">(</span><span class="n">JobFlowId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">job_flow_id</span><span class="p">,</span> <span class="n">Steps</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">steps</span><span class=" [...]
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;ResponseMetadata&#39;</span><span class="p">][</span><span class="s1">&#39;HTTPStatusCode&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">&#39;Adding steps failed: </span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">response</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Steps </span><span class="si">%s</span><span class="s1"> added to JobFlow&#39;</span><span class="p">,</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;StepIds&#39;</span><span class="p">])</span>
+            <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;StepIds&#39;</span><span class="p">]</span></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/operators/emr_create_job_flow_operator.html b/_modules/airflow/contrib/operators/emr_create_job_flow_operator.html
new file mode 100644
index 0000000..1e477c5
--- /dev/null
+++ b/_modules/airflow/contrib/operators/emr_create_job_flow_operator.html
@@ -0,0 +1,296 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.operators.emr_create_job_flow_operator &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.operators.emr_create_job_flow_operator</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.operators.emr_create_job_flow_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.emr_hook</span> <span class="k">import</span> <span class="n">EmrHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+
+
+<div class="viewcode-block" id="EmrCreateJobFlowOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.emr_create_job_flow_operator.EmrCreateJobFlowOperator">[docs]</a><span class="k">class</span> <span class="nc">EmrCreateJobFlowOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Creates an EMR JobFlow, reading the config from the EMR connection.</span>
+<span class="sd">    A dictionary of JobFlow overrides can be passed that override the config from the connection.</span>
+
+<span class="sd">    :param aws_conn_id: aws connection to uses</span>
+<span class="sd">    :type aws_conn_id: str</span>
+<span class="sd">    :param emr_conn_id: emr connection to use</span>
+<span class="sd">    :type emr_conn_id: str</span>
+<span class="sd">    :param job_flow_overrides: boto3 style arguments to override emr_connection extra</span>
+<span class="sd">    :type steps: dict</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[]</span>
+    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">()</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#f9c915&#39;</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">&#39;s3_default&#39;</span><span class="p">,</span>
+            <span class="n">emr_conn_id</span><span class="o">=</span><span class="s1">&#39;emr_default&#39;</span><span class="p">,</span>
+            <span class="n">job_flow_overrides</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">EmrCreateJobFlowOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">emr_conn_id</span> <span class="o">=</span> <span class="n">emr_conn_id</span>
+        <span class="k">if</span> <span class="n">job_flow_overrides</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">job_flow_overrides</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">job_flow_overrides</span> <span class="o">=</span> <span class="n">job_flow_overrides</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="n">emr</span> <span class="o">=</span> <span class="n">EmrHook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">,</span> <span class="n">emr_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">emr_conn_id</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+            <span class="s1">&#39;Creating JobFlow using aws-conn-id: </span><span class="si">%s</span><span class="s1">, emr-conn-id: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">emr_conn_id</span>
+        <span class="p">)</span>
+        <span class="n">response</span> <span class="o">=</span> <span class="n">emr</span><span class="o">.</span><span class="n">create_job_flow</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_flow_overrides</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;ResponseMetadata&#39;</span><span class="p">][</span><span class="s1">&#39;HTTPStatusCode&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">&#39;JobFlow creation failed: </span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">response</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;JobFlow with id </span><span class="si">%s</span><span class="s1"> created&#39;</span><span class="p">,</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;JobFlowId&#39;</span><span class="p">])</span>
+            <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;JobFlowId&#39;</span><span class="p">]</span></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/operators/emr_terminate_job_flow_operator.html b/_modules/airflow/contrib/operators/emr_terminate_job_flow_operator.html
new file mode 100644
index 0000000..7f2f1ca
--- /dev/null
+++ b/_modules/airflow/contrib/operators/emr_terminate_job_flow_operator.html
@@ -0,0 +1,285 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.operators.emr_terminate_job_flow_operator &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.operators.emr_terminate_job_flow_operator</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.operators.emr_terminate_job_flow_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.emr_hook</span> <span class="k">import</span> <span class="n">EmrHook</span>
+
+
+<div class="viewcode-block" id="EmrTerminateJobFlowOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.emr_terminate_job_flow_operator.EmrTerminateJobFlowOperator">[docs]</a><span class="k">class</span> <span class="nc">EmrTerminateJobFlowOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Operator to terminate EMR JobFlows.</span>
+
+<span class="sd">    :param job_flow_id: id of the JobFlow to terminate</span>
+<span class="sd">    :type job_flow_name: str</span>
+<span class="sd">    :param aws_conn_id: aws connection to uses</span>
+<span class="sd">    :type aws_conn_id: str</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;job_flow_id&#39;</span><span class="p">]</span>
+    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">()</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#f9c915&#39;</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">job_flow_id</span><span class="p">,</span>
+            <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">&#39;s3_default&#39;</span><span class="p">,</span>
+            <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">EmrTerminateJobFlowOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">job_flow_id</span> <span class="o">=</span> <span class="n">job_flow_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="n">emr</span> <span class="o">=</span> <span class="n">EmrHook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Terminating JobFlow </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_flow_id</span><span class="p">)</span>
+        <span class="n">response</span> <span class="o">=</span> <span class="n">emr</span><span class="o">.</span><span class="n">terminate_job_flows</span><span class="p">(</span><span class="n">JobFlowIds</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">job_flow_id</span><span class="p">])</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;ResponseMetadata&#39;</span><span class="p">][</span><span class="s1">&#39;HTTPStatusCode&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">&#39;JobFlow termination failed: </span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">response</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;JobFlow with id </span><span class="si">%s</span><span class="s1"> terminated&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_flow_id</span><span class="p">)</span></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/operators/bigquery_operator.html b/_modules/airflow/contrib/operators/file_to_wasb.html
similarity index 50%
copy from _modules/airflow/contrib/operators/bigquery_operator.html
copy to _modules/airflow/contrib/operators/file_to_wasb.html
index c7fcfd9..0c67052 100644
--- a/_modules/airflow/contrib/operators/bigquery_operator.html
+++ b/_modules/airflow/contrib/operators/file_to_wasb.html
@@ -8,11 +8,13 @@
   
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   
-  <title>airflow.contrib.operators.bigquery_operator &mdash; Airflow Documentation</title>
+  <title>airflow.contrib.operators.file_to_wasb &mdash; Airflow Documentation</title>
   
 
   
   
+  
+  
 
   
 
@@ -30,6 +32,9 @@
   
 
   
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
     <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
         <link rel="up" title="Module code" href="../../../index.html"/> 
 
@@ -40,6 +45,7 @@
 
 <body class="wy-body-for-nav" role="document">
 
+   
   <div class="wy-grid-for-nav">
 
     
@@ -76,7 +82,10 @@
           
             
             
-                <ul>
+              
+            
+            
+              <ul>
 <li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
@@ -90,6 +99,8 @@
 <li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
 </ul>
@@ -104,8 +115,10 @@
 
       
       <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
-        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
-        <a href="../../../../index.html">Airflow</a>
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
       </nav>
 
 
@@ -118,25 +131,42 @@
 
 
 
+
+
+
+
+
+
+
+
+
+
 <div role="navigation" aria-label="breadcrumbs navigation">
+
   <ul class="wy-breadcrumbs">
-    <li><a href="../../../../index.html">Docs</a> &raquo;</li>
-      
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
           <li><a href="../../../index.html">Module code</a> &raquo;</li>
-      
-    <li>airflow.contrib.operators.bigquery_operator</li>
+        
+      <li>airflow.contrib.operators.file_to_wasb</li>
+    
+    
       <li class="wy-breadcrumbs-aside">
         
-          
+            
         
       </li>
+    
   </ul>
+
+  
   <hr/>
 </div>
           <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
            <div itemprop="articleBody">
             
-  <h1>Source code for airflow.contrib.operators.bigquery_operator</h1><div class="highlight"><pre>
+  <h1>Source code for airflow.contrib.operators.file_to_wasb</h1><div class="highlight"><pre>
 <span></span><span class="c1"># -*- coding: utf-8 -*-</span>
 <span class="c1">#</span>
 <span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
@@ -150,70 +180,56 @@
 <span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
 <span class="c1"># See the License for the specific language governing permissions and</span>
 <span class="c1"># limitations under the License.</span>
-
-<span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
-<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
+<span class="c1">#</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.wasb_hook</span> <span class="k">import</span> <span class="n">WasbHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
 
 
-<div class="viewcode-block" id="BigQueryOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.bigquery_operator.BigQueryOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+<div class="viewcode-block" id="FileToWasbOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.file_to_wasb.FileToWasbOperator">[docs]</a><span class="k">class</span> <span class="nc">FileToWasbOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Executes BigQuery SQL queries in a specific BigQuery database</span>
+<span class="sd">    Uploads a file to Azure Blob Storage.</span>
+
+<span class="sd">    :param file_path: Path to the file to load.</span>
+<span class="sd">    :type file_path: str</span>
+<span class="sd">    :param container_name: Name of the container.</span>
+<span class="sd">    :type container_name: str</span>
+<span class="sd">    :param blob_name: Name of the blob.</span>
+<span class="sd">    :type blob_name: str</span>
+<span class="sd">    :param wasb_conn_id: Reference to the wasb connection.</span>
+<span class="sd">    :type wasb_conn_id: str</span>
+<span class="sd">    :param load_options: Optional keyword arguments that</span>
+<span class="sd">        `WasbHook.load_file()` takes.</span>
+<span class="sd">    :type load_options: dict</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;bql&#39;</span><span class="p">,</span> <span class="s1">&#39;destination_dataset_table&#39;</span><span class="p">)</span>
-    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.sql&#39;</span><span class="p">,)</span>
-    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#e4f0e8&#39;</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;file_path&#39;</span><span class="p">,</span> <span class="s1">&#39;container_name&#39;</span><span class="p">,</span> <span class="s1">&#39;blob_name&#39;</span><span class="p">)</span>
 
     <span class="nd">@apply_defaults</span>
-    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
-                 <span class="n">bql</span><span class="p">,</span>
-                 <span class="n">destination_dataset_table</span> <span class="o">=</span> <span class="bp">False</span><span class="p">,</span>
-                 <span class="n">write_disposition</span> <span class="o">=</span> <span class="s1">&#39;WRITE_EMPTY&#39;</span><span class="p">,</span>
-                 <span class="n">allow_large_results</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
-                 <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">&#39;bigquery_default&#39;</span><span class="p">,</span>
-                 <span class="n">delegate_to</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
-                 <span class="n">udf_config</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
-                 <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_path</span><span class="p">,</span> <span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span>
+                 <span class="n">wasb_conn_id</span><span class="o">=</span><span class="s1">&#39;wasb_default&#39;</span><span class="p">,</span> <span class="n">load_options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span>
                  <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
-        <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Create a new BigQueryOperator.</span>
-
-<span class="sd">        :param bql: the sql code to be executed</span>
-<span class="sd">        :type bql: Can receive a str representing a sql statement,</span>
-<span class="sd">            a list of str (sql statements), or reference to a template file.</span>
-<span class="sd">            Template reference are recognized by str ending in &#39;.sql&#39;</span>
-<span class="sd">        :param destination_dataset_table: A dotted dataset.table that, if set,</span>
-<span class="sd">            will store the results of the query.</span>
-<span class="sd">        :type destination_dataset_table: string</span>
-<span class="sd">        :param bigquery_conn_id: reference to a specific BigQuery hook.</span>
-<span class="sd">        :type bigquery_conn_id: string</span>
-<span class="sd">        :param delegate_to: The account to impersonate, if any.</span>
-<span class="sd">            For this to work, the service account making the request must have domain-wide delegation enabled.</span>
-<span class="sd">        :type delegate_to: string</span>
-<span class="sd">        :param udf_config: The User Defined Function configuration for the query.</span>
-<span class="sd">            See https://cloud.google.com/bigquery/user-defined-functions for details.</span>
-<span class="sd">        :type udf_config: list</span>
-<span class="sd">        &quot;&quot;&quot;</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">bql</span> <span class="o">=</span> <span class="n">bql</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">destination_dataset_table</span> <span class="o">=</span> <span class="n">destination_dataset_table</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span> <span class="o">=</span> <span class="n">write_disposition</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">allow_large_results</span> <span class="o">=</span> <span class="n">allow_large_results</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">udf_config</span> <span class="o">=</span> <span class="n">udf_config</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">FileToWasbOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">load_options</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">load_options</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">file_path</span> <span class="o">=</span> <span class="n">file_path</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">container_name</span> <span class="o">=</span> <span class="n">container_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">blob_name</span> <span class="o">=</span> <span class="n">blob_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">wasb_conn_id</span> <span class="o">=</span> <span class="n">wasb_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">load_options</span> <span class="o">=</span> <span class="n">load_options</span>
 
     <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Executing: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bql</span><span class="p">))</span>
-        <span class="n">hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
-        <span class="n">conn</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
-        <span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
-        <span class="n">cursor</span><span class="o">.</span><span class="n">run_query</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bql</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">destination_dataset_table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span><span class="p">,</span> <span class="bp">self</span><s [...]
+        <span class="sd">&quot;&quot;&quot;Upload a file to Azure Blob Storage.&quot;&quot;&quot;</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">WasbHook</span><span class="p">(</span><span class="n">wasb_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">wasb_conn_id</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+            <span class="s1">&#39;Uploading </span><span class="si">{self.file_path}</span><span class="s1"> to wasb://</span><span class="si">{self.container_name}</span><span class="s1"> as </span><span class="si">{self.blob_name}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
+        <span class="p">)</span>
+        <span class="n">hook</span><span class="o">.</span><span class="n">load_file</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">file_path</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">container_name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">blob_name</span><span class="p">,</span> <span class="o">**</span><span class="bp">sel [...]
 </pre></div>
 
            </div>
+           <div class="articleComments">
+            
+           </div>
           </div>
           <footer>
   
@@ -246,7 +262,8 @@
             VERSION:'',
             COLLAPSE_INDEX:false,
             FILE_SUFFIX:'.html',
-            HAS_SOURCE:  true
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
         };
     </script>
       <script type="text/javascript" src="../../../../_static/jquery.js"></script>
diff --git a/_modules/airflow/contrib/operators/gcs_download_operator.html b/_modules/airflow/contrib/operators/gcs_download_operator.html
index 39186e2..de261e7 100644
--- a/_modules/airflow/contrib/operators/gcs_download_operator.html
+++ b/_modules/airflow/contrib/operators/gcs_download_operator.html
@@ -13,6 +13,8 @@
 
   
   
+  
+  
 
   
 
@@ -30,6 +32,9 @@
   
 
   
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
     <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
         <link rel="up" title="Module code" href="../../../index.html"/> 
 
@@ -40,6 +45,7 @@
 
 <body class="wy-body-for-nav" role="document">
 
+   
   <div class="wy-grid-for-nav">
 
     
@@ -76,7 +82,10 @@
           
             
             
-                <ul>
+              
+            
+            
+              <ul>
 <li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
@@ -90,6 +99,8 @@
 <li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
 </ul>
@@ -104,8 +115,10 @@
 
       
       <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
-        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
-        <a href="../../../../index.html">Airflow</a>
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
       </nav>
 
 
@@ -118,19 +131,36 @@
 
 
 
+
+
+
+
+
+
+
+
+
+
 <div role="navigation" aria-label="breadcrumbs navigation">
+
   <ul class="wy-breadcrumbs">
-    <li><a href="../../../../index.html">Docs</a> &raquo;</li>
-      
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
           <li><a href="../../../index.html">Module code</a> &raquo;</li>
-      
-    <li>airflow.contrib.operators.gcs_download_operator</li>
+        
+      <li>airflow.contrib.operators.gcs_download_operator</li>
+    
+    
       <li class="wy-breadcrumbs-aside">
         
-          
+            
         
       </li>
+    
   </ul>
+
+  
   <hr/>
 </div>
           <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -151,58 +181,52 @@
 <span class="c1"># See the License for the specific language governing permissions and</span>
 <span class="c1"># limitations under the License.</span>
 
-<span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">sys</span>
 
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="kn">import</span> <span class="n">GoogleCloudStorageHook</span>
-<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="k">import</span> <span class="n">GoogleCloudStorageHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
 
 
-<div class="viewcode-block" id="GoogleCloudStorageDownloadOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStorageDownloadOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+<div class="viewcode-block" id="GoogleCloudStorageDownloadOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStorageDownloadOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Downloads a file from Google Cloud Storage.</span>
+
+<span class="sd">    :param bucket: The Google cloud storage bucket where the object is.</span>
+<span class="sd">    :type bucket: string</span>
+<span class="sd">    :param object: The name of the object to download in the Google cloud</span>
+<span class="sd">        storage bucket.</span>
+<span class="sd">    :type object: string</span>
+<span class="sd">    :param filename: The file path on the local file system (where the</span>
+<span class="sd">        operator is being executed) that the file should be downloaded to.</span>
+<span class="sd">        If false, the downloaded data will not be stored on the local file</span>
+<span class="sd">        system.</span>
+<span class="sd">    :type filename: string</span>
+<span class="sd">    :param store_to_xcom_key: If this param is set, the operator will push</span>
+<span class="sd">        the contents of the downloaded file to XCom with the key set in this</span>
+<span class="sd">        parameter. If false, the downloaded data will not be pushed to XCom.</span>
+<span class="sd">    :type store_to_xcom_key: string</span>
+<span class="sd">    :param google_cloud_storage_conn_id: The connection ID to use when</span>
+<span class="sd">        connecting to Google cloud storage.</span>
+<span class="sd">    :type google_cloud_storage_conn_id: string</span>
+<span class="sd">    :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd">        For this to work, the service account making the request must have domain-wide delegation enabled.</span>
+<span class="sd">    :type delegate_to: string</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;bucket&#39;</span><span class="p">,</span><span class="s1">&#39;object&#39;</span><span class="p">,</span><span class="s1">&#39;filename&#39;</span><span class="p">,</span><span class="s1">&#39;store_to_xcom_key&#39;</span><span class="p">,)</span>
-    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.sql&#39;</span><span class="p">,)</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;bucket&#39;</span><span class="p">,</span> <span class="s1">&#39;object&#39;</span><span class="p">,</span> <span class="s1">&#39;filename&#39;</span><span class="p">,</span> <span class="s1">&#39;store_to_xcom_key&#39;</span><span class="p">,)</span>
     <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#f0eee4&#39;</span>
 
     <span class="nd">@apply_defaults</span>
-    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
-        <span class="bp">self</span><span class="p">,</span>
-        <span class="n">bucket</span><span class="p">,</span>
-        <span class="nb">object</span><span class="p">,</span>
-        <span class="n">filename</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
-        <span class="n">store_to_xcom_key</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
-        <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_storage_default&#39;</span><span class="p">,</span>
-        <span class="n">delegate_to</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
-        <span class="o">*</span><span class="n">args</span><span class="p">,</span>
-        <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
-        <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Create a new GoogleCloudStorageDownloadOperator.</span>
-
-<span class="sd">        :param bucket: The Google cloud storage bucket where the object is.</span>
-<span class="sd">        :type bucket: string</span>
-<span class="sd">        :param object: The name of the object to download in the Google cloud</span>
-<span class="sd">            storage bucket.</span>
-<span class="sd">        :type object: string</span>
-<span class="sd">        :param filename: The file path on the local file system (where the</span>
-<span class="sd">            operator is being executed) that the file should be downloaded to.</span>
-<span class="sd">            If false, the downloaded data will not be stored on the local file</span>
-<span class="sd">            system.</span>
-<span class="sd">        :type filename: string</span>
-<span class="sd">        :param store_to_xcom_key: If this param is set, the operator will push</span>
-<span class="sd">            the contents of the downloaded file to XCom with the key set in this</span>
-<span class="sd">            parameter. If false, the downloaded data will not be pushed to XCom.</span>
-<span class="sd">        :type store_to_xcom_key: string</span>
-<span class="sd">        :param google_cloud_storage_conn_id: The connection ID to use when</span>
-<span class="sd">            connecting to Google cloud storage.</span>
-<span class="sd">        :type google_cloud_storage_conn_id: string</span>
-<span class="sd">        :param delegate_to: The account to impersonate, if any.</span>
-<span class="sd">            For this to work, the service account making the request must have domain-wide delegation enabled.</span>
-<span class="sd">        :type delegate_to: string</span>
-<span class="sd">        &quot;&quot;&quot;</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">GoogleCloudStorageDownloadOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">bucket</span><span class="p">,</span>
+                 <span class="nb">object</span><span class="p">,</span>
+                 <span class="n">filename</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                 <span class="n">store_to_xcom_key</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                 <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_storage_default&#39;</span><span class="p">,</span>
+                 <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+                 <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">GoogleCloudStorageDownloadOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">object</span> <span class="o">=</span> <span class="nb">object</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">filename</span> <span class="o">=</span> <span class="n">filename</span>
@@ -211,7 +235,7 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
 
     <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Executing download: </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class= [...]
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Executing download: </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">, [...]
         <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span><span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
                                       <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
         <span class="n">file_bytes</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
@@ -220,10 +244,13 @@
                 <span class="n">context</span><span class="p">[</span><span class="s1">&#39;ti&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">xcom_push</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">store_to_xcom_key</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">file_bytes</span><span class="p">)</span>
             <span class="k">else</span><span class="p">:</span>
                 <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s1">&#39;The size of the downloaded file is too large to push to XCom!&#39;</span><span class="p">)</span>
-        <span class="k">print</span><span class="p">(</span><span class="n">file_bytes</span><span class="p">)</span></div>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">file_bytes</span><span class="p">)</span></div>
 </pre></div>
 
            </div>
+           <div class="articleComments">
+            
+           </div>
           </div>
           <footer>
   
@@ -256,7 +283,8 @@
             VERSION:'',
             COLLAPSE_INDEX:false,
             FILE_SUFFIX:'.html',
-            HAS_SOURCE:  true
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
         };
     </script>
       <script type="text/javascript" src="../../../../_static/jquery.js"></script>
diff --git a/_modules/airflow/contrib/operators/gcs_to_bq.html b/_modules/airflow/contrib/operators/gcs_to_bq.html
new file mode 100644
index 0000000..1d61268
--- /dev/null
+++ b/_modules/airflow/contrib/operators/gcs_to_bq.html
@@ -0,0 +1,429 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.operators.gcs_to_bq &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.