airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maximebeauche...@apache.org
Subject [19/22] incubator-airflow-site git commit: Latest docs version as of 1.8.x
Date Mon, 06 Mar 2017 17:10:24 GMT
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/5e574012/api.html
----------------------------------------------------------------------
diff --git a/api.html b/api.html
new file mode 100644
index 0000000..7aca3bb
--- /dev/null
+++ b/api.html
@@ -0,0 +1,279 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>Experimental Rest API &mdash; Airflow Documentation</title>
+  
+
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="genindex.html"/>
+        <link rel="search" title="Search" href="search.html"/>
+    <link rel="top" title="Airflow Documentation" href="index.html"/>
+        <link rel="next" title="Integration" href="integration.html"/>
+        <link rel="prev" title="Security" href="security.html"/> 
+
+  
+  <script src="_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main
navigation">
+          
+            
+            
+                <ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="start.html">Quick
Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="profiling.html">Data
Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="cli.html">Command
Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="scheduler.html">Scheduling
&amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li>
+<li class="toctree-l1 current"><a class="current reference internal" href="#">Experimental
Rest API</a><ul>
+<li class="toctree-l2"><a class="reference internal" href="#endpoints">Endpoints</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#cli">CLI</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#authentication">Authentication</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="index.html">Airflow</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="index.html">Docs</a> &raquo;</li>
+        
+      <li>Experimental Rest API</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+            <a href="_sources/api.rst.txt" rel="nofollow"> View page source</a>
+          
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <div class="section" id="experimental-rest-api">
+<h1>Experimental Rest API<a class="headerlink" href="#experimental-rest-api" title="Permalink
to this headline">¶</a></h1>
+<p>Airflow exposes an experimental Rest API. It is available through the webserver.
Endpoints are
+available at /api/experimental/. Please note that we expect the endpoint definitions to change.</p>
+<div class="section" id="endpoints">
+<h2>Endpoints<a class="headerlink" href="#endpoints" title="Permalink to this headline">¶</a></h2>
+<p>This is a place holder until the swagger definitions are active</p>
+<ul class="simple">
+<li>/api/experimental/dags/&lt;DAG_ID&gt;/tasks/&lt;TASK_ID&gt; returns
info for a task (GET).</li>
+<li>/api/experimental/dags/&lt;DAG_ID&gt;/dag_runs creates a dag_run for a
given dag id (POST).</li>
+</ul>
+</div>
+<div class="section" id="cli">
+<h2>CLI<a class="headerlink" href="#cli" title="Permalink to this headline">¶</a></h2>
+<p>For some functions the cli can use the API. To configure the CLI to use the API
when available
+configure as follows:</p>
+<div class="highlight-bash"><div class="highlight"><pre><span></span><span
class="o">[</span>cli<span class="o">]</span>
+<span class="nv">api_client</span> <span class="o">=</span> airflow.api.client.json_client
+<span class="nv">endpoint_url</span> <span class="o">=</span> http://&lt;WEBSERVER&gt;:&lt;PORT&gt;
+</pre></div>
+</div>
+</div>
+<div class="section" id="authentication">
+<h2>Authentication<a class="headerlink" href="#authentication" title="Permalink
to this headline">¶</a></h2>
+<p>Only Kerberos authentication is currently supported for the API. To enable this
set the following
+in the configuration:</p>
+<div class="highlight-bash"><div class="highlight"><pre><span></span><span
class="o">[</span>api<span class="o">]</span>
+<span class="nv">auth_backend</span> <span class="o">=</span> airflow.api.auth.backend.default
+
+<span class="o">[</span>kerberos<span class="o">]</span>
+<span class="nv">keytab</span> <span class="o">=</span> &lt;KEYTAB&gt;
+</pre></div>
+</div>
+<p>The Kerberos service is configured as <cite>airflow/fully.qualified.domainname&#64;REALM</cite>.
Make sure this
+principal exists in the keytab file.</p>
+</div>
+</div>
+
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
+      
+        <a href="integration.html" class="btn btn-neutral float-right" title="Integration"
accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
+      
+      
+        <a href="security.html" class="btn btn-neutral" title="Security" accesskey="p"
rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
+      
+    </div>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a>
provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'./',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="_static/jquery.js"></script>
+      <script type="text/javascript" src="_static/underscore.js"></script>
+      <script type="text/javascript" src="_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/5e574012/integration.html
----------------------------------------------------------------------
diff --git a/integration.html b/integration.html
new file mode 100644
index 0000000..a55aa41
--- /dev/null
+++ b/integration.html
@@ -0,0 +1,424 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>Integration &mdash; Airflow Documentation</title>
+  
+
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="genindex.html"/>
+        <link rel="search" title="Search" href="search.html"/>
+    <link rel="top" title="Airflow Documentation" href="index.html"/>
+        <link rel="next" title="FAQ" href="faq.html"/>
+        <link rel="prev" title="Experimental Rest API" href="api.html"/> 
+
+  
+  <script src="_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main
navigation">
+          
+            
+            
+                <ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="start.html">Quick
Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="profiling.html">Data
Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="cli.html">Command
Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="scheduler.html">Scheduling
&amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="api.html">Experimental
Rest API</a></li>
+<li class="toctree-l1 current"><a class="current reference internal" href="#">Integration</a><ul>
+<li class="toctree-l2"><a class="reference internal" href="#aws-amazon-webservices">AWS:
Amazon Webservices</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#gcp-google-cloud-platform">GCP:
Google Cloud Platform</a><ul>
+<li class="toctree-l3"><a class="reference internal" href="#bigquery">BigQuery</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="#bigquery-operators">BigQuery
Operators</a></li>
+<li class="toctree-l4"><a class="reference internal" href="#bigqueryhook">BigQueryHook</a></li>
+</ul>
+</li>
+<li class="toctree-l3"><a class="reference internal" href="#cloud-dataflow">Cloud
DataFlow</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="#dataflow-operators">DataFlow
Operators</a></li>
+<li class="toctree-l4"><a class="reference internal" href="#dataflowhook">DataFlowHook</a></li>
+</ul>
+</li>
+<li class="toctree-l3"><a class="reference internal" href="#cloud-dataproc">Cloud
DataProc</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="#dataproc-operators">DataProc
Operators</a></li>
+<li class="toctree-l4"><a class="reference internal" href="#dataprocpysparkoperator">DataProcPySparkOperator</a></li>
+</ul>
+</li>
+<li class="toctree-l3"><a class="reference internal" href="#cloud-datastore">Cloud
Datastore</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="#datastore-operators">Datastore
Operators</a></li>
+</ul>
+</li>
+<li class="toctree-l3"><a class="reference internal" href="#cloud-storage">Cloud
Storage</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="#storage-operators">Storage
Operators</a></li>
+<li class="toctree-l4"><a class="reference internal" href="#googlecloudstoragehook">GoogleCloudStorageHook</a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="index.html">Airflow</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="index.html">Docs</a> &raquo;</li>
+        
+      <li>Integration</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+            <a href="_sources/integration.rst.txt" rel="nofollow"> View page source</a>
+          
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <div class="section" id="integration">
+<h1>Integration<a class="headerlink" href="#integration" title="Permalink to this
headline">¶</a></h1>
+<ul class="simple">
+<li><a class="reference internal" href="#aws"><span class="std std-ref">AWS:
Amazon Webservices</span></a></li>
+<li><a class="reference internal" href="#gcp"><span class="std std-ref">GCP:
Google Cloud Platform</span></a></li>
+</ul>
+<div class="section" id="aws-amazon-webservices">
+<span id="aws"></span><h2>AWS: Amazon Webservices<a class="headerlink"
href="#aws-amazon-webservices" title="Permalink to this headline">¶</a></h2>
+<p>&#8212;</p>
+</div>
+<div class="section" id="gcp-google-cloud-platform">
+<span id="gcp"></span><h2>GCP: Google Cloud Platform<a class="headerlink"
href="#gcp-google-cloud-platform" title="Permalink to this headline">¶</a></h2>
+<p>Airflow has extensive support for the Google Cloud Platform. But note that most
Hooks and
+Operators are in the contrib section. Meaning that they have a <em>beta</em>
status, meaning that
+they can have breaking changes between minor releases.</p>
+<div class="section" id="bigquery">
+<h3>BigQuery<a class="headerlink" href="#bigquery" title="Permalink to this headline">¶</a></h3>
+<div class="section" id="bigquery-operators">
+<h4>BigQuery Operators<a class="headerlink" href="#bigquery-operators" title="Permalink
to this headline">¶</a></h4>
+<ul class="simple">
+<li><a class="reference internal" href="#bigquerycheckoperator"><span class="std
std-ref">BigQueryCheckOperator</span></a> : Performs checks against a SQL query
that will return a single row with different values.</li>
+<li><a class="reference internal" href="#bigqueryvaluecheckoperator"><span
class="std std-ref">BigQueryValueCheckOperator</span></a> : Performs a simple
value check using SQL code.</li>
+<li><a class="reference internal" href="#bigqueryintervalcheckoperator"><span
class="std std-ref">BigQueryIntervalCheckOperator</span></a> : Checks that
the values of metrics given as SQL expressions are within a certain tolerance of the ones
from days_back before.</li>
+<li><a class="reference internal" href="#bigqueryoperator"><span class="std
std-ref">BigQueryOperator</span></a> : Executes BigQuery SQL queries in a specific
BigQuery database.</li>
+<li><a class="reference internal" href="#bigquerytobigqueryoperator"><span
class="std std-ref">BigQueryToBigQueryOperator</span></a> : Copy a BigQuery
table to another BigQuery table.</li>
+<li><a class="reference internal" href="#bigquerytocloudstorageoperator"><span
class="std std-ref">BigQueryToCloudStorageOperator</span></a> : Transfers a
BigQuery table to a Google Cloud Storage bucket</li>
+</ul>
+<div class="section" id="bigquerycheckoperator">
+<span id="id1"></span><h5>BigQueryCheckOperator<a class="headerlink"
href="#bigquerycheckoperator" title="Permalink to this headline">¶</a></h5>
+</div>
+<div class="section" id="bigqueryvaluecheckoperator">
+<span id="id2"></span><h5>BigQueryValueCheckOperator<a class="headerlink"
href="#bigqueryvaluecheckoperator" title="Permalink to this headline">¶</a></h5>
+</div>
+<div class="section" id="bigqueryintervalcheckoperator">
+<span id="id3"></span><h5>BigQueryIntervalCheckOperator<a class="headerlink"
href="#bigqueryintervalcheckoperator" title="Permalink to this headline">¶</a></h5>
+</div>
+<div class="section" id="bigqueryoperator">
+<span id="id4"></span><h5>BigQueryOperator<a class="headerlink" href="#bigqueryoperator"
title="Permalink to this headline">¶</a></h5>
+</div>
+<div class="section" id="bigquerytobigqueryoperator">
+<span id="id5"></span><h5>BigQueryToBigQueryOperator<a class="headerlink"
href="#bigquerytobigqueryoperator" title="Permalink to this headline">¶</a></h5>
+</div>
+<div class="section" id="bigquerytocloudstorageoperator">
+<span id="id6"></span><h5>BigQueryToCloudStorageOperator<a class="headerlink"
href="#bigquerytocloudstorageoperator" title="Permalink to this headline">¶</a></h5>
+</div>
+</div>
+<div class="section" id="bigqueryhook">
+<h4>BigQueryHook<a class="headerlink" href="#bigqueryhook" title="Permalink to this
headline">¶</a></h4>
+</div>
+</div>
+<div class="section" id="cloud-dataflow">
+<h3>Cloud DataFlow<a class="headerlink" href="#cloud-dataflow" title="Permalink
to this headline">¶</a></h3>
+<div class="section" id="dataflow-operators">
+<h4>DataFlow Operators<a class="headerlink" href="#dataflow-operators" title="Permalink
to this headline">¶</a></h4>
+<ul class="simple">
+<li><a class="reference internal" href="#dataflowjavaoperator"><span class="std
std-ref">DataFlowJavaOperator</span></a> :</li>
+</ul>
+<div class="section" id="dataflowjavaoperator">
+<span id="id7"></span><h5>DataFlowJavaOperator<a class="headerlink"
href="#dataflowjavaoperator" title="Permalink to this headline">¶</a></h5>
+<div class="code python highlight-default"><div class="highlight"><pre><span></span><span
class="n">default_args</span> <span class="o">=</span> <span class="p">{</span>
+    <span class="s1">&#39;owner&#39;</span><span class="p">:</span>
<span class="s1">&#39;airflow&#39;</span><span class="p">,</span>
+    <span class="s1">&#39;depends_on_past&#39;</span><span class="p">:</span>
<span class="kc">False</span><span class="p">,</span>
+    <span class="s1">&#39;start_date&#39;</span><span class="p">:</span>
+        <span class="p">(</span><span class="mi">2016</span><span
class="p">,</span> <span class="mi">8</span><span class="p">,</span>
<span class="mi">1</span><span class="p">),</span>
+    <span class="s1">&#39;email&#39;</span><span class="p">:</span>
<span class="p">[</span><span class="s1">&#39;alex@vanboxel.be&#39;</span><span
class="p">],</span>
+    <span class="s1">&#39;email_on_failure&#39;</span><span class="p">:</span>
<span class="kc">False</span><span class="p">,</span>
+    <span class="s1">&#39;email_on_retry&#39;</span><span class="p">:</span>
<span class="kc">False</span><span class="p">,</span>
+    <span class="s1">&#39;retries&#39;</span><span class="p">:</span>
<span class="mi">1</span><span class="p">,</span>
+    <span class="s1">&#39;retry_delay&#39;</span><span class="p">:</span>
<span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span
class="o">=</span><span class="mi">30</span><span class="p">),</span>
+    <span class="s1">&#39;dataflow_default_options&#39;</span><span
class="p">:</span> <span class="p">{</span>
+        <span class="s1">&#39;project&#39;</span><span class="p">:</span>
<span class="s1">&#39;my-gcp-project&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;zone&#39;</span><span class="p">:</span>
<span class="s1">&#39;us-central1-f&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;stagingLocation&#39;</span><span class="p">:</span>
<span class="s1">&#39;gs://bucket/tmp/dataflow/staging/&#39;</span><span
class="p">,</span>
+    <span class="p">}</span>
+<span class="p">}</span>
+
+<span class="n">dag</span> <span class="o">=</span> <span class="n">DAG</span><span
class="p">(</span><span class="s1">&#39;test-dag&#39;</span><span
class="p">,</span> <span class="n">default_args</span><span class="o">=</span><span
class="n">default_args</span><span class="p">)</span>
+
+<span class="n">task</span> <span class="o">=</span> <span class="n">DataFlowJavaOperator</span><span
class="p">(</span>
+    <span class="n">gcp_conn_id</span><span class="o">=</span><span
class="s1">&#39;gcp_default&#39;</span><span class="p">,</span>
+    <span class="n">task_id</span><span class="o">=</span><span
class="s1">&#39;normalize-cal&#39;</span><span class="p">,</span>
+    <span class="n">jar</span><span class="o">=</span><span class="s1">&#39;{{var.value.gcp_dataflow_base}}pipeline-ingress-cal-normalize-1.0.jar&#39;</span><span
class="p">,</span>
+    <span class="n">options</span><span class="o">=</span><span
class="p">{</span>
+        <span class="s1">&#39;autoscalingAlgorithm&#39;</span><span
class="p">:</span> <span class="s1">&#39;BASIC&#39;</span><span
class="p">,</span>
+        <span class="s1">&#39;maxNumWorkers&#39;</span><span class="p">:</span>
<span class="s1">&#39;50&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;start&#39;</span><span class="p">:</span>
<span class="s1">&#39;{{ds}}&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;partitionType&#39;</span><span class="p">:</span>
<span class="s1">&#39;DAY&#39;</span>
+
+    <span class="p">},</span>
+    <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span
class="p">)</span>
+</pre></div>
+</div>
+</div>
+</div>
+<div class="section" id="dataflowhook">
+<h4>DataFlowHook<a class="headerlink" href="#dataflowhook" title="Permalink to this
headline">¶</a></h4>
+</div>
+</div>
+<div class="section" id="cloud-dataproc">
+<h3>Cloud DataProc<a class="headerlink" href="#cloud-dataproc" title="Permalink
to this headline">¶</a></h3>
+<div class="section" id="dataproc-operators">
+<h4>DataProc Operators<a class="headerlink" href="#dataproc-operators" title="Permalink
to this headline">¶</a></h4>
+<ul class="simple">
+<li><a class="reference internal" href="#dataprocpigoperator"><span class="std
std-ref">DataProcPigOperator</span></a> : Start a Pig query Job on a Cloud
DataProc cluster.</li>
+<li><a class="reference internal" href="#dataprochiveoperator"><span class="std
std-ref">DataProcHiveOperator</span></a> : Start a Hive query Job on a Cloud
DataProc cluster.</li>
+<li><a class="reference internal" href="#dataprocsparksqloperator"><span class="std
std-ref">DataProcSparkSqlOperator</span></a> : Start a Spark SQL query Job
on a Cloud DataProc cluster.</li>
+<li><a class="reference internal" href="#dataprocsparkoperator"><span class="std
std-ref">DataProcSparkOperator</span></a> : Start a Spark Job on a Cloud DataProc
cluster.</li>
+<li><a class="reference internal" href="#dataprochadoopoperator"><span class="std
std-ref">DataProcHadoopOperator</span></a> : Start a Hadoop Job on a Cloud
DataProc cluster.</li>
+<li><a class="reference internal" href="#dataprocpysparkoperator"><span class="std
std-ref">DataProcPySparkOperator</span></a> : Start a PySpark Job on a Cloud
DataProc cluster.</li>
+</ul>
+<div class="section" id="dataprocpigoperator">
+<span id="id8"></span><h5>DataProcPigOperator<a class="headerlink" href="#dataprocpigoperator"
title="Permalink to this headline">¶</a></h5>
+</div>
+<div class="section" id="dataprochiveoperator">
+<span id="id9"></span><h5>DataProcHiveOperator<a class="headerlink"
href="#dataprochiveoperator" title="Permalink to this headline">¶</a></h5>
+</div>
+<div class="section" id="dataprocsparksqloperator">
+<span id="id10"></span><h5>DataProcSparkSqlOperator<a class="headerlink"
href="#dataprocsparksqloperator" title="Permalink to this headline">¶</a></h5>
+</div>
+<div class="section" id="dataprocsparkoperator">
+<span id="id11"></span><h5>DataProcSparkOperator<a class="headerlink"
href="#dataprocsparkoperator" title="Permalink to this headline">¶</a></h5>
+</div>
+<div class="section" id="dataprochadoopoperator">
+<span id="id12"></span><h5>DataProcHadoopOperator<a class="headerlink"
href="#dataprochadoopoperator" title="Permalink to this headline">¶</a></h5>
+</div>
+</div>
+<div class="section" id="dataprocpysparkoperator">
+<span id="id13"></span><h4>DataProcPySparkOperator<a class="headerlink"
href="#dataprocpysparkoperator" title="Permalink to this headline">¶</a></h4>
+</div>
+</div>
+<div class="section" id="cloud-datastore">
+<h3>Cloud Datastore<a class="headerlink" href="#cloud-datastore" title="Permalink
to this headline">¶</a></h3>
+<div class="section" id="datastore-operators">
+<h4>Datastore Operators<a class="headerlink" href="#datastore-operators" title="Permalink
to this headline">¶</a></h4>
+</div>
+</div>
+<div class="section" id="cloud-storage">
+<h3>Cloud Storage<a class="headerlink" href="#cloud-storage" title="Permalink to
this headline">¶</a></h3>
+<div class="section" id="storage-operators">
+<h4>Storage Operators<a class="headerlink" href="#storage-operators" title="Permalink
to this headline">¶</a></h4>
+<ul class="simple">
+<li><a class="reference internal" href="#googlecloudstoragedownloadoperator"><span
class="std std-ref">GoogleCloudStorageDownloadOperator</span></a> : Downloads
a file from Google Cloud Storage.</li>
+<li><a class="reference internal" href="#googlecloudstoragetobigqueryoperator"><span
class="std std-ref">GoogleCloudStorageToBigQueryOperator</span></a> : Loads
files from Google cloud storage into BigQuery.</li>
+</ul>
+<div class="section" id="googlecloudstoragedownloadoperator">
+<span id="id14"></span><h5>GoogleCloudStorageDownloadOperator<a class="headerlink"
href="#googlecloudstoragedownloadoperator" title="Permalink to this headline">¶</a></h5>
+</div>
+<div class="section" id="googlecloudstoragetobigqueryoperator">
+<span id="id15"></span><h5>GoogleCloudStorageToBigQueryOperator<a class="headerlink"
href="#googlecloudstoragetobigqueryoperator" title="Permalink to this headline">¶</a></h5>
+</div>
+</div>
+<div class="section" id="googlecloudstoragehook">
+<h4>GoogleCloudStorageHook<a class="headerlink" href="#googlecloudstoragehook" title="Permalink
to this headline">¶</a></h4>
+</div>
+</div>
+</div>
+</div>
+
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
+      
+        <a href="faq.html" class="btn btn-neutral float-right" title="FAQ" accesskey="n"
rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
+      
+      
+        <a href="api.html" class="btn btn-neutral" title="Experimental Rest API" accesskey="p"
rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
+      
+    </div>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a>
provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'./',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="_static/jquery.js"></script>
+      <script type="text/javascript" src="_static/underscore.js"></script>
+      <script type="text/javascript" src="_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file


Mime
View raw message