airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maximebeauche...@apache.org
Subject [01/34] incubator-airflow-site git commit: Initial commit
Date Sun, 05 Jun 2016 05:23:51 GMT
Repository: incubator-airflow-site
Updated Branches:
  refs/heads/asf-site [created] 9e19165ca


http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9e19165c/start.html
----------------------------------------------------------------------
diff --git a/start.html b/start.html
new file mode 100644
index 0000000..6bf0db6
--- /dev/null
+++ b/start.html
@@ -0,0 +1,256 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>Quick Start &mdash; Airflow Documentation</title>
+  
+
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+    <link rel="top" title="Airflow Documentation" href="index.html"/>
+        <link rel="next" title="Installation" href="installation.html"/>
+        <link rel="prev" title="License" href="license.html"/> 
+
+  
+  <script src="_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+                <ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="license.html">License</a></li>
+<li class="toctree-l1 current"><a class="current reference internal" href="#">Quick Start</a><ul>
+<li class="toctree-l2"><a class="reference internal" href="#what-s-next">What&#8217;s Next?</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+        <a href="index.html">Airflow</a>
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+ 
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href="index.html">Docs</a> &raquo;</li>
+      
+    <li>Quick Start</li>
+      <li class="wy-breadcrumbs-aside">
+        
+          
+            <a href="_sources/start.txt" rel="nofollow"> View page source</a>
+          
+        
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <div class="section" id="quick-start">
+<h1>Quick Start<a class="headerlink" href="#quick-start" title="Permalink to this headline">¶</a></h1>
+<p>The installation is quick and straightforward.</p>
+<div class="highlight-bash"><div class="highlight"><pre><span></span><span class="c1"># airflow needs a home, ~/airflow is the default,</span>
+<span class="c1"># but you can lay foundation somewhere else if you prefer</span>
+<span class="c1"># (optional)</span>
+<span class="nb">export</span> <span class="nv">AIRFLOW_HOME</span><span class="o">=</span>~/airflow
+
+<span class="c1"># install from pypi using pip</span>
+pip install airflow
+
+<span class="c1"># initialize the database</span>
+airflow initdb
+
+<span class="c1"># start the web server, default port is 8080</span>
+airflow webserver -p 8080
+</pre></div>
+</div>
+<p>Upon running these commands, Airflow will create the <code class="docutils literal"><span class="pre">$AIRFLOW_HOME</span></code> folder
+and lay an &#8220;airflow.cfg&#8221; file with defaults that get you going fast. You can
+inspect the file either in <code class="docutils literal"><span class="pre">$AIRFLOW_HOME/airflow.cfg</span></code>, or through the UI in
+the <code class="docutils literal"><span class="pre">Admin-&gt;Configuration</span></code> menu. The PID file for the webserver will be stored
+in <code class="docutils literal"><span class="pre">$AIRFLOW_HOME/airflow-webserver.pid</span></code> or in <code class="docutils literal"><span class="pre">/run/airflow/webserver.pid</span></code>
+if started by systemd.</p>
+<p>Out of the box, Airflow uses a sqlite database, which you should outgrow
+fairly quickly since no parallelization is possible using this database
+backend. It works in conjunction with the <code class="docutils literal"><span class="pre">SequentialExecutor</span></code> which will
+only run task instances sequentially. While this is very limiting, it allows
+you to get up and running quickly and take a tour of the UI and the
+command line utilities.</p>
+<p>Here are a few commands that will trigger a few task instances. You should
+be able to see the status of the jobs change in the <code class="docutils literal"><span class="pre">example1</span></code> DAG as you
+run the commands below.</p>
+<div class="highlight-bash"><div class="highlight"><pre><span></span><span class="c1"># run your first task instance</span>
+airflow run example_bash_operator runme_0 2015-01-01
+<span class="c1"># run a backfill over 2 days</span>
+airflow backfill example_bash_operator -s 2015-01-01 -e 2015-01-02
+</pre></div>
+</div>
+<div class="section" id="what-s-next">
+<h2>What&#8217;s Next?<a class="headerlink" href="#what-s-next" title="Permalink to this headline">¶</a></h2>
+<p>From this point, you can head to the <a class="reference internal" href="tutorial.html"><span class="doc">Tutorial</span></a> section for further examples or the <a class="reference internal" href="configuration.html"><span class="doc">Configuration</span></a> section if you&#8217;re ready to get your hands dirty.</p>
+</div>
+</div>
+
+
+           </div>
+          </div>
+          <footer>
+  
+    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
+      
+        <a href="installation.html" class="btn btn-neutral float-right" title="Installation" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a>
+      
+      
+        <a href="license.html" class="btn btn-neutral" title="License" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a>
+      
+    </div>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright 2014, Maxime Beauchemin, Airbnb.
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'./',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true
+        };
+    </script>
+      <script type="text/javascript" src="_static/jquery.js"></script>
+      <script type="text/javascript" src="_static/underscore.js"></script>
+      <script type="text/javascript" src="_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9e19165c/tutorial.html
----------------------------------------------------------------------
diff --git a/tutorial.html b/tutorial.html
new file mode 100644
index 0000000..d0adb06
--- /dev/null
+++ b/tutorial.html
@@ -0,0 +1,622 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>Tutorial &mdash; Airflow Documentation</title>
+  
+
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+    <link rel="top" title="Airflow Documentation" href="index.html"/>
+        <link rel="next" title="Configuration" href="configuration.html"/>
+        <link rel="prev" title="Installation" href="installation.html"/> 
+
+  
+  <script src="_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+                <ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li>
+<li class="toctree-l1 current"><a class="current reference internal" href="#">Tutorial</a><ul>
+<li class="toctree-l2"><a class="reference internal" href="#example-pipeline-definition">Example Pipeline definition</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#it-s-a-dag-definition-file">It&#8217;s a DAG definition file</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#importing-modules">Importing Modules</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#default-arguments">Default Arguments</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#instantiate-a-dag">Instantiate a DAG</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#tasks">Tasks</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#templating-with-jinja">Templating with Jinja</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#setting-up-dependencies">Setting up Dependencies</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#recap">Recap</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#testing">Testing</a><ul>
+<li class="toctree-l3"><a class="reference internal" href="#running-the-script">Running the Script</a></li>
+<li class="toctree-l3"><a class="reference internal" href="#command-line-metadata-validation">Command Line Metadata Validation</a></li>
+<li class="toctree-l3"><a class="reference internal" href="#id1">Testing</a></li>
+<li class="toctree-l3"><a class="reference internal" href="#backfill">Backfill</a></li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="#what-s-next">What&#8217;s Next?</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+        <a href="index.html">Airflow</a>
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+ 
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href="index.html">Docs</a> &raquo;</li>
+      
+    <li>Tutorial</li>
+      <li class="wy-breadcrumbs-aside">
+        
+          
+            <a href="_sources/tutorial.txt" rel="nofollow"> View page source</a>
+          
+        
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <div class="section" id="tutorial">
+<h1>Tutorial<a class="headerlink" href="#tutorial" title="Permalink to this headline">¶</a></h1>
+<p>This tutorial walks you through some of the fundamental Airflow concepts,
+objects, and their usage while writing your first pipeline.</p>
+<div class="section" id="example-pipeline-definition">
+<h2>Example Pipeline definition<a class="headerlink" href="#example-pipeline-definition" title="Permalink to this headline">¶</a></h2>
+<p>Here is an example of a basic pipeline definition. Do not worry if this looks
+complicated, a line by line explanation follows below.</p>
+<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">Code that goes along with the Airflow tutorial located at:</span>
+<span class="sd">https://github.com/airbnb/airflow/blob/master/airflow/example_dags/tutorial.py</span>
+<span class="sd">&quot;&quot;&quot;</span>
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">DAG</span>
+<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="k">import</span> <span class="n">BashOperator</span>
+<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
+
+
+<span class="n">default_args</span> <span class="o">=</span> <span class="p">{</span>
+    <span class="s1">&#39;owner&#39;</span><span class="p">:</span> <span class="s1">&#39;airflow&#39;</span><span class="p">,</span>
+    <span class="s1">&#39;depends_on_past&#39;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="s1">&#39;start_date&#39;</span><span class="p">:</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2015</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span>
+    <span class="s1">&#39;email&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;airflow@airflow.com&#39;</span><span class="p">],</span>
+    <span class="s1">&#39;email_on_failure&#39;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="s1">&#39;email_on_retry&#39;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="s1">&#39;retries&#39;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
+    <span class="s1">&#39;retry_delay&#39;</span><span class="p">:</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="mi">5</span><span class="p">),</span>
+    <span class="c1"># &#39;queue&#39;: &#39;bash_queue&#39;,</span>
+    <span class="c1"># &#39;pool&#39;: &#39;backfill&#39;,</span>
+    <span class="c1"># &#39;priority_weight&#39;: 10,</span>
+    <span class="c1"># &#39;end_date&#39;: datetime(2016, 1, 1),</span>
+<span class="p">}</span>
+
+<span class="n">dag</span> <span class="o">=</span> <span class="n">DAG</span><span class="p">(</span><span class="s1">&#39;tutorial&#39;</span><span class="p">,</span> <span class="n">default_args</span><span class="o">=</span><span class="n">default_args</span><span class="p">)</span>
+
+<span class="c1"># t1, t2 and t3 are examples of tasks created by instatiating operators</span>
+<span class="n">t1</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span>
+    <span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;print_date&#39;</span><span class="p">,</span>
+    <span class="n">bash_command</span><span class="o">=</span><span class="s1">&#39;date&#39;</span><span class="p">,</span>
+    <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
+
+<span class="n">t2</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span>
+    <span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;sleep&#39;</span><span class="p">,</span>
+    <span class="n">bash_command</span><span class="o">=</span><span class="s1">&#39;sleep 5&#39;</span><span class="p">,</span>
+    <span class="n">retries</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
+    <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
+
+<span class="n">templated_command</span> <span class="o">=</span> <span class="s2">&quot;&quot;&quot;</span>
+<span class="s2">    {</span><span class="si">% f</span><span class="s2">or i in range(5) %}</span>
+<span class="s2">        echo &quot;{{ ds }}&quot;</span>
+<span class="s2">        echo &quot;{{ macros.ds_add(ds, 7)}}&quot;</span>
+<span class="s2">        echo &quot;{{ params.my_param }}&quot;</span>
+<span class="s2">    {</span><span class="si">% e</span><span class="s2">ndfor %}</span>
+<span class="s2">&quot;&quot;&quot;</span>
+
+<span class="n">t3</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span>
+    <span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;templated&#39;</span><span class="p">,</span>
+    <span class="n">bash_command</span><span class="o">=</span><span class="n">templated_command</span><span class="p">,</span>
+    <span class="n">params</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;my_param&#39;</span><span class="p">:</span> <span class="s1">&#39;Parameter I passed in&#39;</span><span class="p">},</span>
+    <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
+
+<span class="n">t2</span><span class="o">.</span><span class="n">set_upstream</span><span class="p">(</span><span class="n">t1</span><span class="p">)</span>
+<span class="n">t3</span><span class="o">.</span><span class="n">set_upstream</span><span class="p">(</span><span class="n">t1</span><span class="p">)</span>
+</pre></div>
+</div>
+</div>
+<div class="section" id="it-s-a-dag-definition-file">
+<h2>It&#8217;s a DAG definition file<a class="headerlink" href="#it-s-a-dag-definition-file" title="Permalink to this headline">¶</a></h2>
+<p>One thing to wrap your head around (it may not be very intuitive for everyone
+at first) is that this Airflow Python script is really
+just a configuration file specifying the DAG&#8217;s structure as code.
+The actual tasks defined here will run in a different context from
+the context of this script. Different tasks run on different workers
+at different points in time, which means that this script cannot be used
+to cross communicate between tasks. Note that for this
+purpose we have a more advanced feature called <code class="docutils literal"><span class="pre">XCom</span></code>.</p>
+<p>People sometimes think of the DAG definition file as a place where they
+can do some actual data processing - that is not the case at all!
+The script&#8217;s purpose is to define a DAG object. It needs to evaluate
+quickly (seconds, not minutes) since the scheduler will execute it
+periodically to reflect the changes if any.</p>
+</div>
+<div class="section" id="importing-modules">
+<h2>Importing Modules<a class="headerlink" href="#importing-modules" title="Permalink to this headline">¶</a></h2>
+<p>An Airflow pipeline is just a Python script that happens to define an
+Airflow DAG object. Let&#8217;s start by importing the libraries we will need.</p>
+<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="c1"># The DAG object; we&#39;ll need this to instantiate a DAG</span>
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">DAG</span>
+
+<span class="c1"># Operators; we need this to operate!</span>
+<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="k">import</span> <span class="n">BashOperator</span>
+</pre></div>
+</div>
+</div>
+<div class="section" id="default-arguments">
+<h2>Default Arguments<a class="headerlink" href="#default-arguments" title="Permalink to this headline">¶</a></h2>
+<p>We&#8217;re about to create a DAG and some tasks, and we have the choice to
+explicitly pass a set of arguments to each task&#8217;s constructor
+(which would become redundant), or (better!) we can define a dictionary
+of default parameters that we can use when creating tasks.</p>
+<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
+
+<span class="n">default_args</span> <span class="o">=</span> <span class="p">{</span>
+    <span class="s1">&#39;owner&#39;</span><span class="p">:</span> <span class="s1">&#39;airflow&#39;</span><span class="p">,</span>
+    <span class="s1">&#39;depends_on_past&#39;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="s1">&#39;start_date&#39;</span><span class="p">:</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2015</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span>
+    <span class="s1">&#39;email&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;airflow@airflow.com&#39;</span><span class="p">],</span>
+    <span class="s1">&#39;email_on_failure&#39;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="s1">&#39;email_on_retry&#39;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="s1">&#39;retries&#39;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
+    <span class="s1">&#39;retry_delay&#39;</span><span class="p">:</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="mi">5</span><span class="p">),</span>
+    <span class="c1"># &#39;queue&#39;: &#39;bash_queue&#39;,</span>
+    <span class="c1"># &#39;pool&#39;: &#39;backfill&#39;,</span>
+    <span class="c1"># &#39;priority_weight&#39;: 10,</span>
+    <span class="c1"># &#39;end_date&#39;: datetime(2016, 1, 1),</span>
+<span class="p">}</span>
+</pre></div>
+</div>
+<p>For more information about the BaseOperator&#8217;s parameters and what they do,
+refer to the :py:class:<code class="docutils literal"><span class="pre">airflow.models.BaseOperator</span></code> documentation.</p>
+<p>Also, note that you could easily define different sets of arguments that
+would serve different purposes. An example of that would be to have
+different settings between a production and development environment.</p>
+</div>
+<div class="section" id="instantiate-a-dag">
+<h2>Instantiate a DAG<a class="headerlink" href="#instantiate-a-dag" title="Permalink to this headline">¶</a></h2>
+<p>We&#8217;ll need a DAG object to nest our tasks into. Here we pass a string
+that defines the <code class="docutils literal"><span class="pre">dag_id</span></code>, which serves as a unique identifier for your DAG.
+We also pass the default argument dictionary that we just defined and
+define a <code class="docutils literal"><span class="pre">schedule_interval</span></code> of 1 day for the DAG.</p>
+<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="n">dag</span> <span class="o">=</span> <span class="n">DAG</span><span class="p">(</span>
+    <span class="s1">&#39;tutorial&#39;</span><span class="p">,</span> <span class="n">default_args</span><span class="o">=</span><span class="n">default_args</span><span class="p">,</span> <span class="n">schedule_interval</span><span class="o">=</span><span class="n">timedelta</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>
+</pre></div>
+</div>
+</div>
+<div class="section" id="tasks">
+<h2>Tasks<a class="headerlink" href="#tasks" title="Permalink to this headline">¶</a></h2>
+<p>Tasks are generated when instantiating operator objects. An object
+instantiated from an operator is called a constructor. The first argument
+<code class="docutils literal"><span class="pre">task_id</span></code> acts as a unique identifier for the task.</p>
+<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="n">t1</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span>
+    <span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;print_date&#39;</span><span class="p">,</span>
+    <span class="n">bash_command</span><span class="o">=</span><span class="s1">&#39;date&#39;</span><span class="p">,</span>
+    <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
+
+<span class="n">t2</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span>
+    <span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;sleep&#39;</span><span class="p">,</span>
+    <span class="n">bash_command</span><span class="o">=</span><span class="s1">&#39;sleep 5&#39;</span><span class="p">,</span>
+    <span class="n">retries</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
+    <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
+</pre></div>
+</div>
+<p>Notice how we pass a mix of operator specific arguments (<code class="docutils literal"><span class="pre">bash_command</span></code>) and
+an argument common to all operators (<code class="docutils literal"><span class="pre">retries</span></code>) inherited
+from BaseOperator to the operator&#8217;s constructor. This is simpler than
+passing every argument for every constructor call. Also, notice that in
+the second task we override the <code class="docutils literal"><span class="pre">retries</span></code> parameter with <code class="docutils literal"><span class="pre">3</span></code>.</p>
+<p>The precedence rules for a task are as follows:</p>
+<ol class="arabic simple">
+<li>Explicitly passed arguments</li>
+<li>Values that exist in the <code class="docutils literal"><span class="pre">default_args</span></code> dictionary</li>
+<li>The operator&#8217;s default value, if one exists</li>
+</ol>
+<p>A task must include or inherit the arguments <code class="docutils literal"><span class="pre">task_id</span></code> and <code class="docutils literal"><span class="pre">owner</span></code>,
+otherwise Airflow will raise an exception.</p>
+</div>
+<div class="section" id="templating-with-jinja">
+<h2>Templating with Jinja<a class="headerlink" href="#templating-with-jinja" title="Permalink to this headline">¶</a></h2>
+<p>Airflow leverages the power of
+<a class="reference external" href="http://jinja.pocoo.org/docs/dev/">Jinja Templating</a>  and provides
+the pipeline author
+with a set of built-in parameters and macros. Airflow also provides
+hooks for the pipeline author to define their own parameters, macros and
+templates.</p>
+<p>This tutorial barely scratches the surface of what you can do with
+templating in Airflow, but the goal of this section is to let you know
+this feature exists, get you familiar with double curly brackets, and
+point to the most common template variable: <code class="docutils literal"><span class="pre">{{</span> <span class="pre">ds</span> <span class="pre">}}</span></code>.</p>
+<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="n">templated_command</span> <span class="o">=</span> <span class="s2">&quot;&quot;&quot;</span>
+<span class="s2">    {</span><span class="si">% f</span><span class="s2">or i in range(5) %}</span>
+<span class="s2">        echo &quot;{{ ds }}&quot;</span>
+<span class="s2">        echo &quot;{{ macros.ds_add(ds, 7) }}&quot;</span>
+<span class="s2">        echo &quot;{{ params.my_param }}&quot;</span>
+<span class="s2">    {</span><span class="si">% e</span><span class="s2">ndfor %}</span>
+<span class="s2">&quot;&quot;&quot;</span>
+
+<span class="n">t3</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span>
+    <span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;templated&#39;</span><span class="p">,</span>
+    <span class="n">bash_command</span><span class="o">=</span><span class="n">templated_command</span><span class="p">,</span>
+    <span class="n">params</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;my_param&#39;</span><span class="p">:</span> <span class="s1">&#39;Parameter I passed in&#39;</span><span class="p">},</span>
+    <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
+</pre></div>
+</div>
+<p>Notice that the <code class="docutils literal"><span class="pre">templated_command</span></code> contains code logic in <code class="docutils literal"><span class="pre">{%</span> <span class="pre">%}</span></code> blocks,
+references parameters like <code class="docutils literal"><span class="pre">{{</span> <span class="pre">ds</span> <span class="pre">}}</span></code>, calls a function as in
+<code class="docutils literal"><span class="pre">{{</span> <span class="pre">macros.ds_add(ds,</span> <span class="pre">7)}}</span></code>, and references a user-defined parameter
+in <code class="docutils literal"><span class="pre">{{</span> <span class="pre">params.my_param</span> <span class="pre">}}</span></code>.</p>
+<p>The <code class="docutils literal"><span class="pre">params</span></code> hook in <code class="docutils literal"><span class="pre">BaseOperator</span></code> allows you to pass a dictionary of
+parameters and/or objects to your templates. Please take the time
+to understand how the parameter <code class="docutils literal"><span class="pre">my_param</span></code> makes it through to the template.</p>
+<p>Files can also be passed to the <code class="docutils literal"><span class="pre">bash_command</span></code> argument, like
+<code class="docutils literal"><span class="pre">bash_command='templated_command.sh'</span></code>, where the file location is relative to
+the directory containing the pipeline file (<code class="docutils literal"><span class="pre">tutorial.py</span></code> in this case). This
+may be desirable for many reasons, like separating your script&#8217;s logic and
+pipeline code, allowing for proper code highlighting in files composed in
+different languages, and general flexibility in structuring pipelines. It is
+also possible to define your <code class="docutils literal"><span class="pre">template_searchpath</span></code> as pointing to any folder
+locations in the DAG constructor call.</p>
+<p>For more information on the variables and macros that can be referenced
+in templates, make sure to read through the <a class="reference internal" href="code.html#macros"><span class="std std-ref">Macros</span></a> section</p>
+</div>
+<div class="section" id="setting-up-dependencies">
+<h2>Setting up Dependencies<a class="headerlink" href="#setting-up-dependencies" title="Permalink to this headline">¶</a></h2>
+<p>We have two simple tasks that do not depend on each other. Here&#8217;s a few ways
+you can define dependencies between them:</p>
+<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="n">t2</span><span class="o">.</span><span class="n">set_upstream</span><span class="p">(</span><span class="n">t1</span><span class="p">)</span>
+
+<span class="c1"># This means that t2 will depend on t1</span>
+<span class="c1"># running successfully to run</span>
+<span class="c1"># It is equivalent to</span>
+<span class="c1"># t1.set_downstream(t2)</span>
+
+<span class="n">t3</span><span class="o">.</span><span class="n">set_upstream</span><span class="p">(</span><span class="n">t1</span><span class="p">)</span>
+
+<span class="c1"># all of this is equivalent to</span>
+<span class="c1"># dag.set_dependency(&#39;print_date&#39;, &#39;sleep&#39;)</span>
+<span class="c1"># dag.set_dependency(&#39;print_date&#39;, &#39;templated&#39;)</span>
+</pre></div>
+</div>
+<p>Note that when executing your script, Airflow will raise exceptions when
+it finds cycles in your DAG or when a dependency is referenced more
+than once.</p>
+</div>
+<div class="section" id="recap">
+<h2>Recap<a class="headerlink" href="#recap" title="Permalink to this headline">¶</a></h2>
+<p>Alright, so we have a pretty basic DAG. At this point your code should look
+something like this:</p>
+<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">Code that goes along with the Airflow located at:</span>
+<span class="sd">http://airflow.readthedocs.org/en/latest/tutorial.html</span>
+<span class="sd">&quot;&quot;&quot;</span>
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">DAG</span>
+<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="k">import</span> <span class="n">BashOperator</span>
+<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
+
+
+<span class="n">default_args</span> <span class="o">=</span> <span class="p">{</span>
+    <span class="s1">&#39;owner&#39;</span><span class="p">:</span> <span class="s1">&#39;airflow&#39;</span><span class="p">,</span>
+    <span class="s1">&#39;depends_on_past&#39;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="s1">&#39;start_date&#39;</span><span class="p">:</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2015</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span>
+    <span class="s1">&#39;email&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;airflow@airflow.com&#39;</span><span class="p">],</span>
+    <span class="s1">&#39;email_on_failure&#39;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="s1">&#39;email_on_retry&#39;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="s1">&#39;retries&#39;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
+    <span class="s1">&#39;retry_delay&#39;</span><span class="p">:</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="mi">5</span><span class="p">),</span>
+    <span class="c1"># &#39;queue&#39;: &#39;bash_queue&#39;,</span>
+    <span class="c1"># &#39;pool&#39;: &#39;backfill&#39;,</span>
+    <span class="c1"># &#39;priority_weight&#39;: 10,</span>
+    <span class="c1"># &#39;end_date&#39;: datetime(2016, 1, 1),</span>
+<span class="p">}</span>
+
+<span class="n">dag</span> <span class="o">=</span> <span class="n">DAG</span><span class="p">(</span>
+    <span class="s1">&#39;tutorial&#39;</span><span class="p">,</span> <span class="n">default_args</span><span class="o">=</span><span class="n">default_args</span><span class="p">,</span> <span class="n">schedule_interval</span><span class="o">=</span><span class="n">timedelta</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>
+
+<span class="c1"># t1, t2 and t3 are examples of tasks created by instatiating operators</span>
+<span class="n">t1</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span>
+    <span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;print_date&#39;</span><span class="p">,</span>
+    <span class="n">bash_command</span><span class="o">=</span><span class="s1">&#39;date&#39;</span><span class="p">,</span>
+    <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
+
+<span class="n">t2</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span>
+    <span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;sleep&#39;</span><span class="p">,</span>
+    <span class="n">bash_command</span><span class="o">=</span><span class="s1">&#39;sleep 5&#39;</span><span class="p">,</span>
+    <span class="n">retries</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
+    <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
+
+<span class="n">templated_command</span> <span class="o">=</span> <span class="s2">&quot;&quot;&quot;</span>
+<span class="s2">    {</span><span class="si">% f</span><span class="s2">or i in range(5) %}</span>
+<span class="s2">        echo &quot;{{ ds }}&quot;</span>
+<span class="s2">        echo &quot;{{ macros.ds_add(ds, 7)}}&quot;</span>
+<span class="s2">        echo &quot;{{ params.my_param }}&quot;</span>
+<span class="s2">    {</span><span class="si">% e</span><span class="s2">ndfor %}</span>
+<span class="s2">&quot;&quot;&quot;</span>
+
+<span class="n">t3</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span>
+    <span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;templated&#39;</span><span class="p">,</span>
+    <span class="n">bash_command</span><span class="o">=</span><span class="n">templated_command</span><span class="p">,</span>
+    <span class="n">params</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;my_param&#39;</span><span class="p">:</span> <span class="s1">&#39;Parameter I passed in&#39;</span><span class="p">},</span>
+    <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
+
+<span class="n">t2</span><span class="o">.</span><span class="n">set_upstream</span><span class="p">(</span><span class="n">t1</span><span class="p">)</span>
+<span class="n">t3</span><span class="o">.</span><span class="n">set_upstream</span><span class="p">(</span><span class="n">t1</span><span class="p">)</span>
+</pre></div>
+</div>
+</div>
+<div class="section" id="testing">
+<h2>Testing<a class="headerlink" href="#testing" title="Permalink to this headline">¶</a></h2>
+<div class="section" id="running-the-script">
+<h3>Running the Script<a class="headerlink" href="#running-the-script" title="Permalink to this headline">¶</a></h3>
+<p>Time to run some tests. First let&#8217;s make sure that the pipeline
+parses. Let&#8217;s assume we&#8217;re saving the code from the previous step in
+<code class="docutils literal"><span class="pre">tutorial.py</span></code> in the DAGs folder referenced in your <code class="docutils literal"><span class="pre">airflow.cfg</span></code>.
+The default location for your DAGs is <code class="docutils literal"><span class="pre">~/airflow/dags</span></code>.</p>
+<div class="highlight-bash"><div class="highlight"><pre><span></span>python ~/airflow/dags/tutorial.py
+</pre></div>
+</div>
+<p>If the script does not raise an exception it means that you haven&#8217;t done
+anything horribly wrong, and that your Airflow environment is somewhat
+sound.</p>
+</div>
+<div class="section" id="command-line-metadata-validation">
+<h3>Command Line Metadata Validation<a class="headerlink" href="#command-line-metadata-validation" title="Permalink to this headline">¶</a></h3>
+<p>Let&#8217;s run a few commands to validate this script further.</p>
+<div class="highlight-bash"><div class="highlight"><pre><span></span><span class="c1"># print the list of active DAGs</span>
+airflow list_dags
+
+<span class="c1"># prints the list of tasks the &quot;tutorial&quot; dag_id</span>
+airflow list_tasks tutorial
+
+<span class="c1"># prints the hierarchy of tasks in the tutorial DAG</span>
+airflow list_tasks tutorial --tree
+</pre></div>
+</div>
+</div>
+<div class="section" id="id1">
+<h3>Testing<a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h3>
+<p>Let&#8217;s test by running the actual task instances on a specific date. The
+date specified in this context is an <code class="docutils literal"><span class="pre">execution_date</span></code>, which simulates the
+scheduler running your task or dag at a specific date + time:</p>
+<div class="highlight-bash"><div class="highlight"><pre><span></span><span class="c1"># command layout: command subcommand dag_id task_id date</span>
+
+<span class="c1"># testing print_date</span>
+airflow <span class="nb">test</span> tutorial print_date 2015-06-01
+
+<span class="c1"># testing sleep</span>
+airflow <span class="nb">test</span> tutorial sleep 2015-06-01
+</pre></div>
+</div>
+<p>Now remember what we did with templating earlier? See how this template
+gets rendered and executed by running this command:</p>
+<div class="highlight-bash"><div class="highlight"><pre><span></span><span class="c1"># testing templated</span>
+airflow <span class="nb">test</span> tutorial templated 2015-06-01
+</pre></div>
+</div>
+<p>This should result in displaying a verbose log of events and ultimately
+running your bash command and printing the result.</p>
+<p>Note that the <code class="docutils literal"><span class="pre">airflow</span> <span class="pre">test</span></code> command runs task instances locally, outputs
+their log to stdout (on screen), doesn&#8217;t bother with dependencies, and
+doesn&#8217;t communicate state (running, success, failed, ...) to the database.
+It simply allows testing a single task instance.</p>
+</div>
+<div class="section" id="backfill">
+<h3>Backfill<a class="headerlink" href="#backfill" title="Permalink to this headline">¶</a></h3>
+<p>Everything looks like it&#8217;s running fine so let&#8217;s run a backfill.
+<code class="docutils literal"><span class="pre">backfill</span></code> will respect your dependencies, emit logs into files and talk to
+the database to record status. If you do have a webserver up, you&#8217;ll be able
+to track the progress. <code class="docutils literal"><span class="pre">airflow</span> <span class="pre">webserver</span></code> will start a web server if you
+are interested in tracking the progress visually as your backfill progresses.</p>
+<p>Note that if you use <code class="docutils literal"><span class="pre">depends_on_past=True</span></code>, individual task instances
+will depend on the success of the preceding task instance, except for the
+start_date specified itself, for which this dependency is disregarded.</p>
+<p>The date range in this context is a <code class="docutils literal"><span class="pre">start_date</span></code> and optionally an <code class="docutils literal"><span class="pre">end_date</span></code>,
+which are used to populate the run schedule with task instances from this dag.</p>
+<div class="highlight-bash"><div class="highlight"><pre><span></span><span class="c1"># optional, start a web server in debug mode in the background</span>
+<span class="c1"># airflow webserver --debug &amp;</span>
+
+<span class="c1"># start your backfill on a date range</span>
+airflow backfill tutorial -s 2015-06-01 -e 2015-06-07
+</pre></div>
+</div>
+</div>
+</div>
+<div class="section" id="what-s-next">
+<h2>What&#8217;s Next?<a class="headerlink" href="#what-s-next" title="Permalink to this headline">¶</a></h2>
+<p>That&#8217;s it, you&#8217;ve written, tested and backfilled your very first Airflow
+pipeline. Merging your code into a code repository that has a master scheduler
+running against it should get it to get triggered and run every day.</p>
+<p>Here&#8217;s a few things you might want to do next:</p>
+<ul>
+<li><p class="first">Take an in-depth tour of the UI - click all the things!</p>
+</li>
+<li><p class="first">Keep reading the docs! Especially the sections on:</p>
+<blockquote>
+<div><ul class="simple">
+<li>Command line interface</li>
+<li>Operators</li>
+<li>Macros</li>
+</ul>
+</div></blockquote>
+</li>
+<li><p class="first">Write your first pipeline!</p>
+</li>
+</ul>
+</div>
+</div>
+
+
+           </div>
+          </div>
+          <footer>
+  
+    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
+      
+        <a href="configuration.html" class="btn btn-neutral float-right" title="Configuration" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a>
+      
+      
+        <a href="installation.html" class="btn btn-neutral" title="Installation" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a>
+      
+    </div>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright 2014, Maxime Beauchemin, Airbnb.
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'./',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true
+        };
+    </script>
+      <script type="text/javascript" src="_static/jquery.js"></script>
+      <script type="text/javascript" src="_static/underscore.js"></script>
+      <script type="text/javascript" src="_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9e19165c/ui.html
----------------------------------------------------------------------
diff --git a/ui.html b/ui.html
new file mode 100644
index 0000000..df7f6f7
--- /dev/null
+++ b/ui.html
@@ -0,0 +1,296 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>UI / Screenshots &mdash; Airflow Documentation</title>
+  
+
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+    <link rel="top" title="Airflow Documentation" href="index.html"/>
+        <link rel="next" title="Concepts" href="concepts.html"/>
+        <link rel="prev" title="Configuration" href="configuration.html"/> 
+
+  
+  <script src="_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+                <ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="configuration.html">Configuration</a></li>
+<li class="toctree-l1 current"><a class="current reference internal" href="#">UI / Screenshots</a><ul>
+<li class="toctree-l2"><a class="reference internal" href="#dags-view">DAGs View</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#tree-view">Tree View</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#graph-view">Graph View</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#variable-view">Variable View</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#gantt-chart">Gantt Chart</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#task-duration">Task Duration</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#code-view">Code View</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#task-instance-context-menu">Task Instance Context Menu</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+        <a href="index.html">Airflow</a>
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+ 
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href="index.html">Docs</a> &raquo;</li>
+      
+    <li>UI / Screenshots</li>
+      <li class="wy-breadcrumbs-aside">
+        
+          
+            <a href="_sources/ui.txt" rel="nofollow"> View page source</a>
+          
+        
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <div class="section" id="ui-screenshots">
+<h1>UI / Screenshots<a class="headerlink" href="#ui-screenshots" title="Permalink to this headline">¶</a></h1>
+<p>The Airflow UI make it easy to monitor and troubleshoot your data pipelines.
+Here&#8217;s a quick overview of some of the features and visualizations you
+can find in the Airflow UI.</p>
+<div class="section" id="dags-view">
+<h2>DAGs View<a class="headerlink" href="#dags-view" title="Permalink to this headline">¶</a></h2>
+<p>List of the DAGs in your environment, and a set of shortcuts to useful pages.
+You can see exactly how many tasks succeeded, failed, or are currently
+running at a glance.</p>
+<hr class="docutils" />
+<img alt="_images/dags.png" src="_images/dags.png" />
+</div>
+<hr class="docutils" />
+<div class="section" id="tree-view">
+<h2>Tree View<a class="headerlink" href="#tree-view" title="Permalink to this headline">¶</a></h2>
+<p>A tree representation of the DAG that spans across time. If a pipeline is
+late, you can quickly see where the different steps are and identify
+the blocking ones.</p>
+<hr class="docutils" />
+<img alt="_images/tree.png" src="_images/tree.png" />
+</div>
+<hr class="docutils" />
+<div class="section" id="graph-view">
+<h2>Graph View<a class="headerlink" href="#graph-view" title="Permalink to this headline">¶</a></h2>
+<p>The graph view is perhaps the most comprehensive. Visualize your DAG&#8217;s
+dependencies and their current status for a specific run.</p>
+<hr class="docutils" />
+<img alt="_images/graph.png" src="_images/graph.png" />
+</div>
+<hr class="docutils" />
+<div class="section" id="variable-view">
+<h2>Variable View<a class="headerlink" href="#variable-view" title="Permalink to this headline">¶</a></h2>
+<p>The variable view allows you to list, create, edit or delete the key-value pair
+of a variable used during jobs. Value of a variable will be hidden if the key contains
+any words in (&#8216;password&#8217;, &#8216;secret&#8217;, &#8216;passwd&#8217;, &#8216;authorization&#8217;, &#8216;api_key&#8217;, &#8216;apikey&#8217;, &#8216;access_token&#8217;)
+by default, but can be configured to show in clear-text.</p>
+<hr class="docutils" />
+<img alt="_images/variable_hidden.png" src="_images/variable_hidden.png" />
+</div>
+<hr class="docutils" />
+<div class="section" id="gantt-chart">
+<h2>Gantt Chart<a class="headerlink" href="#gantt-chart" title="Permalink to this headline">¶</a></h2>
+<p>The Gantt chart lets you analyse task duration and overlap. You can quickly
+identify bottlenecks and where the bulk of the time is spent for specific
+DAG runs.</p>
+<hr class="docutils" />
+<img alt="_images/gantt.png" src="_images/gantt.png" />
+</div>
+<hr class="docutils" />
+<div class="section" id="task-duration">
+<h2>Task Duration<a class="headerlink" href="#task-duration" title="Permalink to this headline">¶</a></h2>
+<p>The duration of your different tasks over the past N runs. This view lets
+you find outliers and quickly understand where the time is spent in your
+DAG over many runs.</p>
+<hr class="docutils" />
+<img alt="_images/duration.png" src="_images/duration.png" />
+</div>
+<hr class="docutils" />
+<div class="section" id="code-view">
+<h2>Code View<a class="headerlink" href="#code-view" title="Permalink to this headline">¶</a></h2>
+<p>Transparency is everything. While the code for your pipeline is in source
+control, this is a quick way to get to the code that generates the DAG and
+provide yet more context.</p>
+<hr class="docutils" />
+<img alt="_images/code.png" src="_images/code.png" />
+</div>
+<hr class="docutils" />
+<div class="section" id="task-instance-context-menu">
+<h2>Task Instance Context Menu<a class="headerlink" href="#task-instance-context-menu" title="Permalink to this headline">¶</a></h2>
+<p>From the pages seen above (tree view, graph view, gantt, ...), it is always
+possible to click on a task instance, and get to this rich context menu
+that can take you to more detailed metadata, and perform some actions.</p>
+<hr class="docutils" />
+<img alt="_images/context.png" src="_images/context.png" />
+</div>
+</div>
+
+
+           </div>
+          </div>
+          <footer>
+  
+    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
+      
+        <a href="concepts.html" class="btn btn-neutral float-right" title="Concepts" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a>
+      
+      
+        <a href="configuration.html" class="btn btn-neutral" title="Configuration" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a>
+      
+    </div>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright 2014, Maxime Beauchemin, Airbnb.
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'./',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true
+        };
+    </script>
+      <script type="text/javascript" src="_static/jquery.js"></script>
+      <script type="text/javascript" src="_static/underscore.js"></script>
+      <script type="text/javascript" src="_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file


Mime
View raw message