airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kaxiln...@apache.org
Subject [41/48] incubator-airflow-site git commit: 1.10.0 with Updated Api Reference
Date Wed, 29 Aug 2018 20:25:09 GMT
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/_modules/airflow/contrib/operators/bigquery_table_delete_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_table_delete_operator.html b/_modules/airflow/contrib/operators/bigquery_table_delete_operator.html
index 94bc9dd..8b3a424 100644
--- a/_modules/airflow/contrib/operators/bigquery_table_delete_operator.html
+++ b/_modules/airflow/contrib/operators/bigquery_table_delete_operator.html
@@ -24,26 +24,17 @@
 
   
 
-  
-  
-    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-  
-
-  
-
-  
-        <link rel="index" title="Index"
-              href="../../../../genindex.html"/>
-        <link rel="search" title="Search" href="../../../../search.html"/>
-    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
-        <link rel="up" title="Module code" href="../../../index.html"/> 
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
 
   
   <script src="../../../../_static/js/modernizr.min.js"></script>
 
 </head>
 
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
 
    
   <div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
     <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
 
       
-      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+      <nav class="wy-nav-top" aria-label="top navigation">
         
           <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
           <a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
       </nav>
 
 
-      
       <div class="wy-nav-content">
+        
         <div class="rst-content">
+        
           
 
 
@@ -156,8 +148,6 @@
     
       <li class="wy-breadcrumbs-aside">
         
-            
-        
       </li>
     
   </ul>
@@ -178,9 +168,9 @@
 <span class="c1"># to you under the Apache License, Version 2.0 (the</span>
 <span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
 <span class="c1"># with the License.  You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
 <span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
 <span class="c1"># Unless required by applicable law or agreed to in writing,</span>
 <span class="c1"># software distributed under the License is distributed on an</span>
 <span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -199,7 +189,7 @@
 
 <span class="sd">    :param deletion_dataset_table: A dotted</span>
 <span class="sd">        (&lt;project&gt;.|&lt;project&gt;:)&lt;dataset&gt;.&lt;table&gt; that indicates which table</span>
-<span class="sd">        will be deleted.</span>
+<span class="sd">        will be deleted. (templated)</span>
 <span class="sd">    :type deletion_dataset_table: string</span>
 <span class="sd">    :param bigquery_conn_id: reference to a specific BigQuery hook.</span>
 <span class="sd">    :type bigquery_conn_id: string</span>
@@ -238,9 +228,7 @@
 </pre></div>
 
            </div>
-           <div class="articleComments">
-            
-           </div>
+           
           </div>
           <footer>
   
@@ -252,7 +240,7 @@
 
     </p>
   </div>
-  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
 
 </footer>
 
@@ -271,6 +259,7 @@
         var DOCUMENTATION_OPTIONS = {
             URL_ROOT:'../../../../',
             VERSION:'',
+            LANGUAGE:'None',
             COLLAPSE_INDEX:false,
             FILE_SUFFIX:'.html',
             HAS_SOURCE:  true,
@@ -283,19 +272,13 @@
 
   
 
-  
-  
-    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-  
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
 
-  
-  
   <script type="text/javascript">
       jQuery(function () {
-          SphinxRtdTheme.StickyNav.enable();
+          SphinxRtdTheme.Navigation.enable(true);
       });
-  </script>
-   
+  </script> 
 
 </body>
 </html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/_modules/airflow/contrib/operators/bigquery_to_bigquery.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_to_bigquery.html b/_modules/airflow/contrib/operators/bigquery_to_bigquery.html
index 0abb905..da4edcd 100644
--- a/_modules/airflow/contrib/operators/bigquery_to_bigquery.html
+++ b/_modules/airflow/contrib/operators/bigquery_to_bigquery.html
@@ -24,26 +24,17 @@
 
   
 
-  
-  
-    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-  
-
-  
-
-  
-        <link rel="index" title="Index"
-              href="../../../../genindex.html"/>
-        <link rel="search" title="Search" href="../../../../search.html"/>
-    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
-        <link rel="up" title="Module code" href="../../../index.html"/> 
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
 
   
   <script src="../../../../_static/js/modernizr.min.js"></script>
 
 </head>
 
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
 
    
   <div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
     <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
 
       
-      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+      <nav class="wy-nav-top" aria-label="top navigation">
         
           <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
           <a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
       </nav>
 
 
-      
       <div class="wy-nav-content">
+        
         <div class="rst-content">
+        
           
 
 
@@ -156,8 +148,6 @@
     
       <li class="wy-breadcrumbs-aside">
         
-            
-        
       </li>
     
   </ul>
@@ -178,9 +168,9 @@
 <span class="c1"># to you under the Apache License, Version 2.0 (the</span>
 <span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
 <span class="c1"># with the License.  You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
 <span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
 <span class="c1"># Unless required by applicable law or agreed to in writing,</span>
 <span class="c1"># software distributed under the License is distributed on an</span>
 <span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -203,11 +193,12 @@
 
 <span class="sd">    :param source_project_dataset_tables: One or more</span>
 <span class="sd">        dotted (project:|project.)&lt;dataset&gt;.&lt;table&gt; BigQuery tables to use as the</span>
-<span class="sd">        source data. If &lt;project&gt; is not included, project will be the project defined</span>
-<span class="sd">        in the connection json. Use a list if there are multiple source tables.</span>
+<span class="sd">        source data. If &lt;project&gt; is not included, project will be the</span>
+<span class="sd">        project defined in the connection json. Use a list if there are multiple</span>
+<span class="sd">        source tables. (templated)</span>
 <span class="sd">    :type source_project_dataset_tables: list|string</span>
 <span class="sd">    :param destination_project_dataset_table: The destination BigQuery</span>
-<span class="sd">        table. Format is: (project:|project.)&lt;dataset&gt;.&lt;table&gt;</span>
+<span class="sd">        table. Format is: (project:|project.)&lt;dataset&gt;.&lt;table&gt; (templated)</span>
 <span class="sd">    :type destination_project_dataset_table: string</span>
 <span class="sd">    :param write_disposition: The write disposition if the table already exists.</span>
 <span class="sd">    :type write_disposition: string</span>
@@ -260,9 +251,7 @@
 </pre></div>
 
            </div>
-           <div class="articleComments">
-            
-           </div>
+           
           </div>
           <footer>
   
@@ -274,7 +263,7 @@
 
     </p>
   </div>
-  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
 
 </footer>
 
@@ -293,6 +282,7 @@
         var DOCUMENTATION_OPTIONS = {
             URL_ROOT:'../../../../',
             VERSION:'',
+            LANGUAGE:'None',
             COLLAPSE_INDEX:false,
             FILE_SUFFIX:'.html',
             HAS_SOURCE:  true,
@@ -305,19 +295,13 @@
 
   
 
-  
-  
-    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-  
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
 
-  
-  
   <script type="text/javascript">
       jQuery(function () {
-          SphinxRtdTheme.StickyNav.enable();
+          SphinxRtdTheme.Navigation.enable(true);
       });
-  </script>
-   
+  </script> 
 
 </body>
 </html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/_modules/airflow/contrib/operators/bigquery_to_gcs.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_to_gcs.html b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
index 6ebc523..9052db6 100644
--- a/_modules/airflow/contrib/operators/bigquery_to_gcs.html
+++ b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
@@ -24,26 +24,17 @@
 
   
 
-  
-  
-    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-  
-
-  
-
-  
-        <link rel="index" title="Index"
-              href="../../../../genindex.html"/>
-        <link rel="search" title="Search" href="../../../../search.html"/>
-    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
-        <link rel="up" title="Module code" href="../../../index.html"/> 
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
 
   
   <script src="../../../../_static/js/modernizr.min.js"></script>
 
 </head>
 
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
 
    
   <div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
     <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
 
       
-      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+      <nav class="wy-nav-top" aria-label="top navigation">
         
           <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
           <a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
       </nav>
 
 
-      
       <div class="wy-nav-content">
+        
         <div class="rst-content">
+        
           
 
 
@@ -156,8 +148,6 @@
     
       <li class="wy-breadcrumbs-aside">
         
-            
-        
       </li>
     
   </ul>
@@ -178,9 +168,9 @@
 <span class="c1"># to you under the Apache License, Version 2.0 (the</span>
 <span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
 <span class="c1"># with the License.  You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
 <span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
 <span class="c1"># Unless required by applicable law or agreed to in writing,</span>
 <span class="c1"># software distributed under the License is distributed on an</span>
 <span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -203,11 +193,11 @@
 
 <span class="sd">    :param source_project_dataset_table: The dotted</span>
 <span class="sd">        (&lt;project&gt;.|&lt;project&gt;:)&lt;dataset&gt;.&lt;table&gt; BigQuery table to use as the source</span>
-<span class="sd">        data. If &lt;project&gt; is not included, project will be the project defined in</span>
-<span class="sd">        the connection json.</span>
+<span class="sd">        data. If &lt;project&gt; is not included, project will be the project</span>
+<span class="sd">        defined in the connection json. (templated)</span>
 <span class="sd">    :type source_project_dataset_table: string</span>
 <span class="sd">    :param destination_cloud_storage_uris: The destination Google Cloud</span>
-<span class="sd">        Storage URI (e.g. gs://some-bucket/some-file.txt). Follows</span>
+<span class="sd">        Storage URI (e.g. gs://some-bucket/some-file.txt). (templated) Follows</span>
 <span class="sd">        convention defined here:</span>
 <span class="sd">        https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple</span>
 <span class="sd">    :type destination_cloud_storage_uris: list</span>
@@ -270,9 +260,7 @@
 </pre></div>
 
            </div>
-           <div class="articleComments">
-            
-           </div>
+           
           </div>
           <footer>
   
@@ -284,7 +272,7 @@
 
     </p>
   </div>
-  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
 
 </footer>
 
@@ -303,6 +291,7 @@
         var DOCUMENTATION_OPTIONS = {
             URL_ROOT:'../../../../',
             VERSION:'',
+            LANGUAGE:'None',
             COLLAPSE_INDEX:false,
             FILE_SUFFIX:'.html',
             HAS_SOURCE:  true,
@@ -315,19 +304,13 @@
 
   
 
-  
-  
-    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-  
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
 
-  
-  
   <script type="text/javascript">
       jQuery(function () {
-          SphinxRtdTheme.StickyNav.enable();
+          SphinxRtdTheme.Navigation.enable(true);
       });
-  </script>
-   
+  </script> 
 
 </body>
 </html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/_modules/airflow/contrib/operators/cassandra_to_gcs.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/cassandra_to_gcs.html b/_modules/airflow/contrib/operators/cassandra_to_gcs.html
new file mode 100644
index 0000000..a53bc3d
--- /dev/null
+++ b/_modules/airflow/contrib/operators/cassandra_to_gcs.html
@@ -0,0 +1,573 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.operators.cassandra_to_gcs &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      <div class="wy-nav-content">
+        
+        <div class="rst-content">
+        
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.operators.cassandra_to_gcs</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.operators.cassandra_to_gcs</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">unicode_literals</span>
+
+<span class="kn">import</span> <span class="nn">json</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
+<span class="kn">from</span> <span class="nn">base64</span> <span class="k">import</span> <span class="n">b64encode</span>
+<span class="kn">from</span> <span class="nn">cassandra.util</span> <span class="k">import</span> <span class="n">Date</span><span class="p">,</span> <span class="n">Time</span><span class="p">,</span> <span class="n">SortedSet</span><span class="p">,</span> <span class="n">OrderedMapSerializedKey</span>
+<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span>
+<span class="kn">from</span> <span class="nn">decimal</span> <span class="k">import</span> <span class="n">Decimal</span>
+<span class="kn">from</span> <span class="nn">six</span> <span class="k">import</span> <span class="n">text_type</span><span class="p">,</span> <span class="n">binary_type</span><span class="p">,</span> <span class="n">PY3</span>
+<span class="kn">from</span> <span class="nn">tempfile</span> <span class="k">import</span> <span class="n">NamedTemporaryFile</span>
+<span class="kn">from</span> <span class="nn">uuid</span> <span class="k">import</span> <span class="n">UUID</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="k">import</span> <span class="n">GoogleCloudStorageHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.cassandra_hook</span> <span class="k">import</span> <span class="n">CassandraHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="CassandraToGoogleCloudStorageOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.cassandra_to_gcs.CassandraToGoogleCloudStorageOperator">[docs]</a><span class="k">class</span> <span class="nc">CassandraToGoogleCloudStorageOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Copy data from Cassandra to Google cloud storage in JSON format</span>
+
+<span class="sd">    Note: Arrays of arrays are not supported.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;cql&#39;</span><span class="p">,</span> <span class="s1">&#39;bucket&#39;</span><span class="p">,</span> <span class="s1">&#39;filename&#39;</span><span class="p">,</span> <span class="s1">&#39;schema_filename&#39;</span><span class="p">,)</span>
+    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.cql&#39;</span><span class="p">,)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#a0e08c&#39;</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">cql</span><span class="p">,</span>
+                 <span class="n">bucket</span><span class="p">,</span>
+                 <span class="n">filename</span><span class="p">,</span>
+                 <span class="n">schema_filename</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">approx_max_file_size_bytes</span><span class="o">=</span><span class="mi">1900000000</span><span class="p">,</span>
+                 <span class="n">cassandra_conn_id</span><span class="o">=</span><span class="s1">&#39;cassandra_default&#39;</span><span class="p">,</span>
+                 <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_default&#39;</span><span class="p">,</span>
+                 <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+                 <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        :param cql: The CQL to execute on the Cassandra table.</span>
+<span class="sd">        :type cql: string</span>
+<span class="sd">        :param bucket: The bucket to upload to.</span>
+<span class="sd">        :type bucket: string</span>
+<span class="sd">        :param filename: The filename to use as the object name when uploading</span>
+<span class="sd">            to Google cloud storage. A {} should be specified in the filename</span>
+<span class="sd">            to allow the operator to inject file numbers in cases where the</span>
+<span class="sd">            file is split due to size.</span>
+<span class="sd">        :type filename: string</span>
+<span class="sd">        :param schema_filename: If set, the filename to use as the object name</span>
+<span class="sd">            when uploading a .json file containing the BigQuery schema fields</span>
+<span class="sd">            for the table that was dumped from MySQL.</span>
+<span class="sd">        :type schema_filename: string</span>
+<span class="sd">        :param approx_max_file_size_bytes: This operator supports the ability</span>
+<span class="sd">            to split large table dumps into multiple files (see notes in the</span>
+<span class="sd">            filenamed param docs above). Google cloud storage allows for files</span>
+<span class="sd">            to be a maximum of 4GB. This param allows developers to specify the</span>
+<span class="sd">            file size of the splits.</span>
+<span class="sd">        :type approx_max_file_size_bytes: long</span>
+<span class="sd">        :param cassandra_conn_id: Reference to a specific Cassandra hook.</span>
+<span class="sd">        :type cassandra_conn_id: string</span>
+<span class="sd">        :param google_cloud_storage_conn_id: Reference to a specific Google</span>
+<span class="sd">            cloud storage hook.</span>
+<span class="sd">        :type google_cloud_storage_conn_id: string</span>
+<span class="sd">        :param delegate_to: The account to impersonate, if any. For this to</span>
+<span class="sd">            work, the service account making the request must have domain-wide</span>
+<span class="sd">            delegation enabled.</span>
+<span class="sd">        :type delegate_to: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">CassandraToGoogleCloudStorageOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cql</span> <span class="o">=</span> <span class="n">cql</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">filename</span> <span class="o">=</span> <span class="n">filename</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">schema_filename</span> <span class="o">=</span> <span class="n">schema_filename</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">approx_max_file_size_bytes</span> <span class="o">=</span> <span class="n">approx_max_file_size_bytes</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cassandra_conn_id</span> <span class="o">=</span> <span class="n">cassandra_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="kc">None</span>
+
+    <span class="c1"># Default Cassandra to BigQuery type mapping</span>
+    <span class="n">CQL_TYPE_MAP</span> <span class="o">=</span> <span class="p">{</span>
+        <span class="s1">&#39;BytesType&#39;</span><span class="p">:</span> <span class="s1">&#39;BYTES&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;DecimalType&#39;</span><span class="p">:</span> <span class="s1">&#39;FLOAT&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;UUIDType&#39;</span><span class="p">:</span> <span class="s1">&#39;BYTES&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;BooleanType&#39;</span><span class="p">:</span> <span class="s1">&#39;BOOL&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;ByteType&#39;</span><span class="p">:</span> <span class="s1">&#39;INTEGER&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;AsciiType&#39;</span><span class="p">:</span> <span class="s1">&#39;STRING&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;FloatType&#39;</span><span class="p">:</span> <span class="s1">&#39;FLOAT&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;DoubleType&#39;</span><span class="p">:</span> <span class="s1">&#39;FLOAT&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;LongType&#39;</span><span class="p">:</span> <span class="s1">&#39;INTEGER&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;Int32Type&#39;</span><span class="p">:</span> <span class="s1">&#39;INTEGER&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;IntegerType&#39;</span><span class="p">:</span> <span class="s1">&#39;INTEGER&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;InetAddressType&#39;</span><span class="p">:</span> <span class="s1">&#39;STRING&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;CounterColumnType&#39;</span><span class="p">:</span> <span class="s1">&#39;INTEGER&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;DateType&#39;</span><span class="p">:</span> <span class="s1">&#39;TIMESTAMP&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;SimpleDateType&#39;</span><span class="p">:</span> <span class="s1">&#39;DATE&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;TimestampType&#39;</span><span class="p">:</span> <span class="s1">&#39;TIMESTAMP&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;TimeUUIDType&#39;</span><span class="p">:</span> <span class="s1">&#39;BYTES&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;ShortType&#39;</span><span class="p">:</span> <span class="s1">&#39;INTEGER&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;TimeType&#39;</span><span class="p">:</span> <span class="s1">&#39;TIME&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;DurationType&#39;</span><span class="p">:</span> <span class="s1">&#39;INTEGER&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;UTF8Type&#39;</span><span class="p">:</span> <span class="s1">&#39;STRING&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;VarcharType&#39;</span><span class="p">:</span> <span class="s1">&#39;STRING&#39;</span><span class="p">,</span>
+    <span class="p">}</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="n">cursor</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_query_cassandra</span><span class="p">()</span>
+        <span class="n">files_to_upload</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_write_local_data_files</span><span class="p">(</span><span class="n">cursor</span><span class="p">)</span>
+
+        <span class="c1"># If a schema is set, create a BQ schema JSON file.</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema_filename</span><span class="p">:</span>
+            <span class="n">files_to_upload</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_write_local_schema_file</span><span class="p">(</span><span class="n">cursor</span><span class="p">))</span>
+
+        <span class="c1"># Flush all files before uploading</span>
+        <span class="k">for</span> <span class="n">file_handle</span> <span class="ow">in</span> <span class="n">files_to_upload</span><span class="o">.</span><span class="n">values</span><span class="p">():</span>
+            <span class="n">file_handle</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">_upload_to_gcs</span><span class="p">(</span><span class="n">files_to_upload</span><span class="p">)</span>
+
+        <span class="c1"># Close all temp file handles.</span>
+        <span class="k">for</span> <span class="n">file_handle</span> <span class="ow">in</span> <span class="n">files_to_upload</span><span class="o">.</span><span class="n">values</span><span class="p">():</span>
+            <span class="n">file_handle</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+
+        <span class="c1"># Close all sessions and connection associated with this Cassandra cluster</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">shutdown_cluster</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">_query_cassandra</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Queries cassandra and returns a cursor to the results.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">CassandraHook</span><span class="p">(</span><span class="n">cassandra_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cassandra_conn_id</span><span class="p">)</span>
+        <span class="n">session</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="n">cursor</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cql</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">cursor</span>
+
+    <span class="k">def</span> <span class="nf">_write_local_data_files</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cursor</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Takes a cursor, and writes results to a local file.</span>
+
+<span class="sd">        :return: A dictionary where keys are filenames to be used as object</span>
+<span class="sd">            names in GCS, and values are file handles to local files that</span>
+<span class="sd">            contain the data for the GCS objects.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">file_no</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="n">tmp_file_handle</span> <span class="o">=</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="n">delete</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+        <span class="n">tmp_file_handles</span> <span class="o">=</span> <span class="p">{</span><span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">file_no</span><span class="p">):</span> <span class="n">tmp_file_handle</span><span class="p">}</span>
+        <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">cursor</span><span class="p">:</span>
+            <span class="n">row_dict</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">generate_data_dict</span><span class="p">(</span><span class="n">row</span><span class="o">.</span><span class="n">_fields</span><span class="p">,</span> <span class="n">row</span><span class="p">)</span>
+            <span class="n">s</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">row_dict</span><span class="p">)</span>
+            <span class="k">if</span> <span class="n">PY3</span><span class="p">:</span>
+                <span class="n">s</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
+            <span class="n">tmp_file_handle</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
+
+            <span class="c1"># Append newline to make dumps BigQuery compatible.</span>
+            <span class="n">tmp_file_handle</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="sa">b</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
+
+            <span class="k">if</span> <span class="n">tmp_file_handle</span><span class="o">.</span><span class="n">tell</span><span class="p">()</span> <span class="o">&gt;=</span> <span class="bp">self</span><span class="o">.</span><span class="n">approx_max_file_size_bytes</span><span class="p">:</span>
+                <span class="n">file_no</span> <span class="o">+=</span> <span class="mi">1</span>
+                <span class="n">tmp_file_handle</span> <span class="o">=</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="n">delete</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+                <span class="n">tmp_file_handles</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">file_no</span><span class="p">)]</span> <span class="o">=</span> <span class="n">tmp_file_handle</span>
+
+        <span class="k">return</span> <span class="n">tmp_file_handles</span>
+
+    <span class="k">def</span> <span class="nf">_write_local_schema_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cursor</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Takes a cursor, and writes the BigQuery schema for the results to a</span>
+<span class="sd">        local file system.</span>
+
+<span class="sd">        :return: A dictionary where key is a filename to be used as an object</span>
+<span class="sd">            name in GCS, and values are file handles to local files that</span>
+<span class="sd">            contains the BigQuery schema fields in .json format.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">schema</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="n">tmp_schema_file_handle</span> <span class="o">=</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="n">delete</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="nb">type</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">cursor</span><span class="o">.</span><span class="n">column_names</span><span class="p">,</span> <span class="n">cursor</span><span class="o">.</span><span class="n">column_types</span><span class="p">):</span>
+            <span class="n">schema</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">generate_schema_dict</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="nb">type</span><span class="p">))</span>
+        <span class="n">json_serialized_schema</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">schema</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">PY3</span><span class="p">:</span>
+            <span class="n">json_serialized_schema</span> <span class="o">=</span> <span class="n">json_serialized_schema</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
+
+        <span class="n">tmp_schema_file_handle</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">json_serialized_schema</span><span class="p">)</span>
+        <span class="k">return</span> <span class="p">{</span><span class="bp">self</span><span class="o">.</span><span class="n">schema_filename</span><span class="p">:</span> <span class="n">tmp_schema_file_handle</span><span class="p">}</span>
+
+    <span class="k">def</span> <span class="nf">_upload_to_gcs</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">files_to_upload</span><span class="p">):</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span>
+            <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
+            <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+        <span class="k">for</span> <span class="nb">object</span><span class="p">,</span> <span class="n">tmp_file_handle</span> <span class="ow">in</span> <span class="n">files_to_upload</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+            <span class="n">hook</span><span class="o">.</span><span class="n">upload</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">tmp_file_handle</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="s1">&#39;application/json&#39;</span><span class="p">)</span>
+
+    <span class="nd">@classmethod</span>
+    <span class="k">def</span> <span class="nf">generate_data_dict</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">names</span><span class="p">,</span> <span class="n">values</span><span class="p">):</span>
+        <span class="n">row_dict</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">values</span><span class="p">):</span>
+            <span class="n">row_dict</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="n">name</span><span class="p">:</span> <span class="bp">cls</span><span class="o">.</span><span class="n">convert_value</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">)})</span>
+        <span class="k">return</span> <span class="n">row_dict</span>
+
+    <span class="nd">@classmethod</span>
+    <span class="k">def</span> <span class="nf">convert_value</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">value</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">value</span>
+        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="p">(</span><span class="n">text_type</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">float</span><span class="p">,</span> <span class="nb">bool</span><span class="p">,</span> <span class="nb">dict</span><span class="p">)):</span>
+            <span class="k">return</span> <span class="n">value</span>
+        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">binary_type</span><span class="p">):</span>
+            <span class="k">return</span> <span class="n">b64encode</span><span class="p">(</span><span class="n">value</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">&#39;ascii&#39;</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">UUID</span><span class="p">):</span>
+            <span class="k">return</span> <span class="n">b64encode</span><span class="p">(</span><span class="n">value</span><span class="o">.</span><span class="n">bytes</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">&#39;ascii&#39;</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="p">(</span><span class="n">datetime</span><span class="p">,</span> <span class="n">Date</span><span class="p">)):</span>
+            <span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">Decimal</span><span class="p">):</span>
+            <span class="k">return</span> <span class="nb">float</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">Time</span><span class="p">):</span>
+            <span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">value</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
+        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="n">SortedSet</span><span class="p">)):</span>
+            <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">convert_array_types</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="s1">&#39;_fields&#39;</span><span class="p">):</span>
+            <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">convert_user_type</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">):</span>
+            <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">convert_tuple_type</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">OrderedMapSerializedKey</span><span class="p">):</span>
+            <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">convert_map_type</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">&#39;unexpected value: &#39;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">value</span><span class="p">))</span>
+
+    <span class="nd">@classmethod</span>
+    <span class="k">def</span> <span class="nf">convert_array_types</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+        <span class="k">return</span> <span class="p">[</span><span class="bp">cls</span><span class="o">.</span><span class="n">convert_value</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">nested_value</span><span class="p">)</span> <span class="k">for</span> <span class="n">nested_value</span> <span class="ow">in</span> <span class="n">value</span><span class="p">]</span>
+
+<div class="viewcode-block" id="CassandraToGoogleCloudStorageOperator.convert_user_type"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.cassandra_to_gcs.CassandraToGoogleCloudStorageOperator.convert_user_type">[docs]</a>    <span class="nd">@classmethod</span>
+    <span class="k">def</span> <span class="nf">convert_user_type</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Converts a user type to RECORD that contains n fields, where n is the</span>
+<span class="sd">        number of attributes. Each element in the user type class will be converted to its</span>
+<span class="sd">        corresponding data type in BQ.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">names</span> <span class="o">=</span> <span class="n">value</span><span class="o">.</span><span class="n">_fields</span>
+        <span class="n">values</span> <span class="o">=</span> <span class="p">[</span><span class="bp">cls</span><span class="o">.</span><span class="n">convert_value</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">name</span><span class="p">))</span> <span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">names</span><span class="p">]</span>
+        <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">generate_data_dict</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">values</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="CassandraToGoogleCloudStorageOperator.convert_tuple_type"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.cassandra_to_gcs.CassandraToGoogleCloudStorageOperator.convert_tuple_type">[docs]</a>    <span class="nd">@classmethod</span>
+    <span class="k">def</span> <span class="nf">convert_tuple_type</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Converts a tuple to RECORD that contains n fields, each will be converted</span>
+<span class="sd">        to its corresponding data type in bq and will be named &#39;field_&lt;index&gt;&#39;, where</span>
+<span class="sd">        index is determined by the order of the tuple elments defined in cassandra.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">names</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;field_&#39;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">value</span><span class="p">))]</span>
+        <span class="n">values</span> <span class="o">=</span> <span class="p">[</span><span class="bp">cls</span><span class="o">.</span><span class="n">convert_value</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span> <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">value</span><span class="p">)]</span>
+        <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">generate_data_dict</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">values</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="CassandraToGoogleCloudStorageOperator.convert_map_type"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.cassandra_to_gcs.CassandraToGoogleCloudStorageOperator.convert_map_type">[docs]</a>    <span class="nd">@classmethod</span>
+    <span class="k">def</span> <span class="nf">convert_map_type</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Converts a map to a repeated RECORD that contains two fields: &#39;key&#39; and &#39;value&#39;,</span>
+<span class="sd">        each will be converted to its corresopnding data type in BQ.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">converted_map</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">value</span><span class="o">.</span><span class="n">keys</span><span class="p">(),</span> <span class="n">value</span><span class="o">.</span><span class="n">values</span><span class="p">()):</span>
+            <span class="n">converted_map</span><span class="o">.</span><span class="n">append</span><span class="p">({</span>
+                <span class="s1">&#39;key&#39;</span><span class="p">:</span> <span class="bp">cls</span><span class="o">.</span><span class="n">convert_value</span><span class="p">(</span><span class="s1">&#39;key&#39;</span><span class="p">,</span> <span class="n">k</span><span class="p">),</span>
+                <span class="s1">&#39;value&#39;</span><span class="p">:</span> <span class="bp">cls</span><span class="o">.</span><span class="n">convert_value</span><span class="p">(</span><span class="s1">&#39;value&#39;</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span>
+            <span class="p">})</span>
+        <span class="k">return</span> <span class="n">converted_map</span></div>
+
+    <span class="nd">@classmethod</span>
+    <span class="k">def</span> <span class="nf">generate_schema_dict</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="nb">type</span><span class="p">):</span>
+        <span class="n">field_schema</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
+        <span class="n">field_schema</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s1">&#39;name&#39;</span><span class="p">:</span> <span class="n">name</span><span class="p">})</span>
+        <span class="n">field_schema</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="bp">cls</span><span class="o">.</span><span class="n">get_bq_type</span><span class="p">(</span><span class="nb">type</span><span class="p">)})</span>
+        <span class="n">field_schema</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s1">&#39;mode&#39;</span><span class="p">:</span> <span class="bp">cls</span><span class="o">.</span><span class="n">get_bq_mode</span><span class="p">(</span><span class="nb">type</span><span class="p">)})</span>
+        <span class="n">fields</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">get_bq_fields</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="nb">type</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">fields</span><span class="p">:</span>
+            <span class="n">field_schema</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s1">&#39;fields&#39;</span><span class="p">:</span> <span class="n">fields</span><span class="p">})</span>
+        <span class="k">return</span> <span class="n">field_schema</span>
+
+    <span class="nd">@classmethod</span>
+    <span class="k">def</span> <span class="nf">get_bq_fields</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="nb">type</span><span class="p">):</span>
+        <span class="n">fields</span> <span class="o">=</span> <span class="p">[]</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="bp">cls</span><span class="o">.</span><span class="n">is_simple_type</span><span class="p">(</span><span class="nb">type</span><span class="p">):</span>
+            <span class="n">names</span><span class="p">,</span> <span class="n">types</span> <span class="o">=</span> <span class="p">[],</span> <span class="p">[]</span>
+
+            <span class="k">if</span> <span class="bp">cls</span><span class="o">.</span><span class="n">is_array_type</span><span class="p">(</span><span class="nb">type</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">cls</span><span class="o">.</span><span class="n">is_record_type</span><span class="p">(</span><span class="nb">type</span><span class="o">.</span><span class="n">subtypes</span><span class="p">[</span><span class="mi">0</span><span class="p">]):</span>
+                <span class="n">names</span> <span class="o">=</span> <span class="nb">type</span><span class="o">.</span><span class="n">subtypes</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">fieldnames</span>
+                <span class="n">types</span> <span class="o">=</span> <span class="nb">type</span><span class="o">.</span><span class="n">subtypes</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">subtypes</span>
+            <span class="k">elif</span> <span class="bp">cls</span><span class="o">.</span><span class="n">is_record_type</span><span class="p">(</span><span class="nb">type</span><span class="p">):</span>
+                <span class="n">names</span> <span class="o">=</span> <span class="nb">type</span><span class="o">.</span><span class="n">fieldnames</span>
+                <span class="n">types</span> <span class="o">=</span> <span class="nb">type</span><span class="o">.</span><span class="n">subtypes</span>
+
+            <span class="k">if</span> <span class="n">types</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">names</span> <span class="ow">and</span> <span class="nb">type</span><span class="o">.</span><span class="n">cassname</span> <span class="o">==</span> <span class="s1">&#39;TupleType&#39;</span><span class="p">:</span>
+                <span class="n">names</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;field_&#39;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">types</span><span class="p">))]</span>
+            <span class="k">elif</span> <span class="n">types</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">names</span> <span class="ow">and</span> <span class="nb">type</span><span class="o">.</span><span class="n">cassname</span> <span class="o">==</span> <span class="s1">&#39;MapType&#39;</span><span class="p">:</span>
+                <span class="n">names</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;key&#39;</span><span class="p">,</span> <span class="s1">&#39;value&#39;</span><span class="p">]</span>
+
+            <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="nb">type</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">types</span><span class="p">):</span>
+                <span class="n">field</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">generate_schema_dict</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="nb">type</span><span class="p">)</span>
+                <span class="n">fields</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">field</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">fields</span>
+
+    <span class="nd">@classmethod</span>
+    <span class="k">def</span> <span class="nf">is_simple_type</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="nb">type</span><span class="p">):</span>
+        <span class="k">return</span> <span class="nb">type</span><span class="o">.</span><span class="n">cassname</span> <span class="ow">in</span> <span class="n">CassandraToGoogleCloudStorageOperator</span><span class="o">.</span><span class="n">CQL_TYPE_MAP</span>
+
+    <span class="nd">@classmethod</span>
+    <span class="k">def</span> <span class="nf">is_array_type</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="nb">type</span><span class="p">):</span>
+        <span class="k">return</span> <span class="nb">type</span><span class="o">.</span><span class="n">cassname</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;ListType&#39;</span><span class="p">,</span> <span class="s1">&#39;SetType&#39;</span><span class="p">]</span>
+
+    <span class="nd">@classmethod</span>
+    <span class="k">def</span> <span class="nf">is_record_type</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="nb">type</span><span class="p">):</span>
+        <span class="k">return</span> <span class="nb">type</span><span class="o">.</span><span class="n">cassname</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;UserType&#39;</span><span class="p">,</span> <span class="s1">&#39;TupleType&#39;</span><span class="p">,</span> <span class="s1">&#39;MapType&#39;</span><span class="p">]</span>
+
+    <span class="nd">@classmethod</span>
+    <span class="k">def</span> <span class="nf">get_bq_type</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="nb">type</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">cls</span><span class="o">.</span><span class="n">is_simple_type</span><span class="p">(</span><span class="nb">type</span><span class="p">):</span>
+            <span class="k">return</span> <span class="n">CassandraToGoogleCloudStorageOperator</span><span class="o">.</span><span class="n">CQL_TYPE_MAP</span><span class="p">[</span><span class="nb">type</span><span class="o">.</span><span class="n">cassname</span><span class="p">]</span>
+        <span class="k">elif</span> <span class="bp">cls</span><span class="o">.</span><span class="n">is_record_type</span><span class="p">(</span><span class="nb">type</span><span class="p">):</span>
+            <span class="k">return</span> <span class="s1">&#39;RECORD&#39;</span>
+        <span class="k">elif</span> <span class="bp">cls</span><span class="o">.</span><span class="n">is_array_type</span><span class="p">(</span><span class="nb">type</span><span class="p">):</span>
+            <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">get_bq_type</span><span class="p">(</span><span class="nb">type</span><span class="o">.</span><span class="n">subtypes</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">&#39;Not a supported type: &#39;</span> <span class="o">+</span> <span class="nb">type</span><span class="o">.</span><span class="n">cassname</span><span class="p">)</span>
+
+    <span class="nd">@classmethod</span>
+    <span class="k">def</span> <span class="nf">get_bq_mode</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="nb">type</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">cls</span><span class="o">.</span><span class="n">is_array_type</span><span class="p">(</span><span class="nb">type</span><span class="p">)</span> <span class="ow">or</span> <span class="nb">type</span><span class="o">.</span><span class="n">cassname</span> <span class="o">==</span> <span class="s1">&#39;MapType&#39;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="s1">&#39;REPEATED&#39;</span>
+        <span class="k">elif</span> <span class="bp">cls</span><span class="o">.</span><span class="n">is_record_type</span><span class="p">(</span><span class="nb">type</span><span class="p">)</span> <span class="ow">or</span> <span class="bp">cls</span><span class="o">.</span><span class="n">is_simple_type</span><span class="p">(</span><span class="nb">type</span><span class="p">):</span>
+            <span class="k">return</span> <span class="s1">&#39;NULLABLE&#39;</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">&#39;Not a supported type: &#39;</span> <span class="o">+</span> <span class="nb">type</span><span class="o">.</span><span class="n">cassname</span><span class="p">)</span></div>
+</pre></div>
+
+           </div>
+           
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            LANGUAGE:'None',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/_modules/airflow/contrib/operators/databricks_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/databricks_operator.html b/_modules/airflow/contrib/operators/databricks_operator.html
index c39f769..f401636 100644
--- a/_modules/airflow/contrib/operators/databricks_operator.html
+++ b/_modules/airflow/contrib/operators/databricks_operator.html
@@ -24,26 +24,17 @@
 
   
 
-  
-  
-    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-  
-
-  
-
-  
-        <link rel="index" title="Index"
-              href="../../../../genindex.html"/>
-        <link rel="search" title="Search" href="../../../../search.html"/>
-    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
-        <link rel="up" title="Module code" href="../../../index.html"/> 
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
 
   
   <script src="../../../../_static/js/modernizr.min.js"></script>
 
 </head>
 
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
 
    
   <div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
     <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
 
       
-      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+      <nav class="wy-nav-top" aria-label="top navigation">
         
           <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
           <a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
       </nav>
 
 
-      
       <div class="wy-nav-content">
+        
         <div class="rst-content">
+        
           
 
 
@@ -156,8 +148,6 @@
     
       <li class="wy-breadcrumbs-aside">
         
-            
-        
       </li>
     
   </ul>
@@ -178,9 +168,9 @@
 <span class="c1"># to you under the Apache License, Version 2.0 (the</span>
 <span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
 <span class="c1"># with the License.  You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
 <span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
 <span class="c1"># Unless required by applicable law or agreed to in writing,</span>
 <span class="c1"># software distributed under the License is distributed on an</span>
 <span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -259,8 +249,7 @@
 <span class="sd">        (i.e. ``spark_jar_task``, ``notebook_task``..) to this operator will</span>
 <span class="sd">        be merged with this json dictionary if they are provided.</span>
 <span class="sd">        If there are conflicts during the merge, the named parameters will</span>
-<span class="sd">        take precedence and override the top level json keys. This field will be</span>
-<span class="sd">        templated.</span>
+<span class="sd">        take precedence and override the top level json keys. (templated)</span>
 
 <span class="sd">        .. seealso::</span>
 <span class="sd">            For more information about templating see :ref:`jinja-templating`.</span>
@@ -377,13 +366,14 @@
 <span class="sd">        Coerces content or all values of content if it is a dict to a string. The</span>
 <span class="sd">        function will throw if content contains non-string or non-numeric types.</span>
 
-<span class="sd">        The reason why we have this function is because the ``self.json`` field must be a dict</span>
-<span class="sd">        with only string values. This is because ``render_template`` will fail for numerical values.</span>
+<span class="sd">        The reason why we have this function is because the ``self.json`` field must be a</span>
+<span class="sd">         dict with only string values. This is because ``render_template`` will fail</span>
+<span class="sd">        for numerical values.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">c</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_deep_string_coerce</span>
         <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">):</span>
             <span class="k">return</span> <span class="n">content</span>
-        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">integer_types</span><span class="o">+</span><span class="p">(</span><span class="nb">float</span><span class="p">,)):</span>
+        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">integer_types</span> <span class="o">+</span> <span class="p">(</span><span class="nb">float</span><span class="p">,)):</span>
             <span class="c1"># Databricks can tolerate either numeric or string types in the API backend.</span>
             <span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
         <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
@@ -393,8 +383,8 @@
                     <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="n">content</span><span class="o">.</span><span class="n">items</span><span class="p">())}</span>
         <span class="k">else</span><span class="p">:</span>
             <span class="n">param_type</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
-            <span class="n">msg</span> <span class="o">=</span> <span class="s1">&#39;Type </span><span class="si">{0}</span><span class="s1"> used for parameter </span><span class="si">{1}</span><span class="s1"> is not a number or a string&#39;</span> \
-                    <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">param_type</span><span class="p">,</span> <span class="n">json_path</span><span class="p">)</span>
+            <span class="n">msg</span> <span class="o">=</span> <span class="s1">&#39;Type </span><span class="si">{0}</span><span class="s1"> used for parameter </span><span class="si">{1}</span><span class="s1"> is not a number or a string&#39;</span>\
+                <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">param_type</span><span class="p">,</span> <span class="n">json_path</span><span class="p">)</span>
             <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">_log_run_page_url</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">url</span><span class="p">):</span>
@@ -443,9 +433,7 @@
 </pre></div>
 
            </div>
-           <div class="articleComments">
-            
-           </div>
+           
           </div>
           <footer>
   
@@ -457,7 +445,7 @@
 
     </p>
   </div>
-  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
 
 </footer>
 
@@ -476,6 +464,7 @@
         var DOCUMENTATION_OPTIONS = {
             URL_ROOT:'../../../../',
             VERSION:'',
+            LANGUAGE:'None',
             COLLAPSE_INDEX:false,
             FILE_SUFFIX:'.html',
             HAS_SOURCE:  true,
@@ -488,19 +477,13 @@
 
   
 
-  
-  
-    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-  
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
 
-  
-  
   <script type="text/javascript">
       jQuery(function () {
-          SphinxRtdTheme.StickyNav.enable();
+          SphinxRtdTheme.Navigation.enable(true);
       });
-  </script>
-   
+  </script> 
 
 </body>
 </html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/_modules/airflow/contrib/operators/dataflow_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/dataflow_operator.html b/_modules/airflow/contrib/operators/dataflow_operator.html
index d5d4041..09cc5c1 100644
--- a/_modules/airflow/contrib/operators/dataflow_operator.html
+++ b/_modules/airflow/contrib/operators/dataflow_operator.html
@@ -24,26 +24,17 @@
 
   
 
-  
-  
-    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-  
-
-  
-
-  
-        <link rel="index" title="Index"
-              href="../../../../genindex.html"/>
-        <link rel="search" title="Search" href="../../../../search.html"/>
-    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
-        <link rel="up" title="Module code" href="../../../index.html"/> 
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
 
   
   <script src="../../../../_static/js/modernizr.min.js"></script>
 
 </head>
 
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
 
    
   <div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
     <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
 
       
-      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+      <nav class="wy-nav-top" aria-label="top navigation">
         
           <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
           <a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
       </nav>
 
 
-      
       <div class="wy-nav-content">
+        
         <div class="rst-content">
+        
           
 
 
@@ -156,8 +148,6 @@
     
       <li class="wy-breadcrumbs-aside">
         
-            
-        
       </li>
     
   </ul>
@@ -178,9 +168,9 @@
 <span class="c1"># to you under the Apache License, Version 2.0 (the</span>
 <span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
 <span class="c1"># with the License.  You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
 <span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
 <span class="c1"># Unless required by applicable law or agreed to in writing,</span>
 <span class="c1"># software distributed under the License is distributed on an</span>
 <span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -548,9 +538,7 @@
 </pre></div>
 
            </div>
-           <div class="articleComments">
-            
-           </div>
+           
           </div>
           <footer>
   
@@ -562,7 +550,7 @@
 
     </p>
   </div>
-  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
 
 </footer>
 
@@ -581,6 +569,7 @@
         var DOCUMENTATION_OPTIONS = {
             URL_ROOT:'../../../../',
             VERSION:'',
+            LANGUAGE:'None',
             COLLAPSE_INDEX:false,
             FILE_SUFFIX:'.html',
             HAS_SOURCE:  true,
@@ -593,19 +582,13 @@
 
   
 
-  
-  
-    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-  
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
 
-  
-  
   <script type="text/javascript">
       jQuery(function () {
-          SphinxRtdTheme.StickyNav.enable();
+          SphinxRtdTheme.Navigation.enable(true);
       });
-  </script>
-   
+  </script> 
 
 </body>
 </html>
\ No newline at end of file


Mime
View raw message