http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/plugins.html ---------------------------------------------------------------------- diff --git a/plugins.html b/plugins.html index 68c615d..be1cfbb 100644 --- a/plugins.html +++ b/plugins.html @@ -30,6 +30,9 @@ + <link rel="index" title="Index" + href="genindex.html"/> + <link rel="search" title="Search" href="search.html"/> <link rel="top" title="Airflow Documentation" href="index.html"/> <link rel="next" title="Security" href="security.html"/> <link rel="prev" title="Scheduling & Triggers" href="scheduler.html"/> @@ -41,6 +44,7 @@ <body class="wy-body-for-nav" role="document"> + <div class="wy-grid-for-nav"> @@ -97,6 +101,8 @@ </ul> </li> <li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="api.html">Experimental Rest API</a></li> +<li class="toctree-l1"><a class="reference internal" href="integration.html">Integration</a></li> <li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a></li> </ul> @@ -111,8 +117,10 @@ <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="index.html">Airflow</a> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="index.html">Airflow</a> + </nav> @@ -121,23 +129,40 @@ <div class="rst-content"> - + + + + + + + + + + + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> - <li><a href="index.html">Docs</a> »</li> - - <li>Plugins</li> + + <li><a href="index.html">Docs</a> »</li> + + <li>Plugins</li> + + <li class="wy-breadcrumbs-aside"> - - <a href="_sources/plugins.txt" rel="nofollow"> View page source</a> + + <a href="_sources/plugins.rst.txt" rel="nofollow"> View page source</a> </li> + </ul> + + <hr/> </div> <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> @@ -226,18 +251,22 @@ definitions in Airflow.</p> <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span> <span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="k">import</span> <span class="n">BaseExecutor</span> -<span class="c1"># Will show up under airflow.hooks.PluginHook</span> +<span class="c1"># Will show up under airflow.hooks.test_plugin.PluginHook</span> <span class="k">class</span> <span class="nc">PluginHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span> <span class="k">pass</span> -<span class="c1"># Will show up under airflow.operators.PluginOperator</span> +<span class="c1"># Will show up under airflow.operators.test_plugin.PluginOperator</span> <span class="k">class</span> <span class="nc">PluginOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> <span class="k">pass</span> -<span class="c1"># Will show up under airflow.executors.PluginExecutor</span> +<span class="c1"># Will show up under airflow.executors.test_plugin.PluginExecutor</span> <span class="k">class</span> <span class="nc">PluginExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">):</span> <span class="k">pass</span> +<span class="c1"># Will show up under airflow.macros.test_plugin.plugin_macro</span> +<span class="k">def</span> <span class="nf">plugin_macro</span><span class="p">():</span> + <span class="k">pass</span> + <span class="c1"># Creating a flask admin BaseView</span> <span class="k">class</span> <span class="nc">TestView</span><span class="p">(</span><span class="n">BaseView</span><span class="p">):</span> <span class="nd">@expose</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)</span> @@ -262,10 +291,11 @@ definitions in Airflow.</p> <span class="k">class</span> <span class="nc">AirflowTestPlugin</span><span class="p">(</span><span class="n">AirflowPlugin</span><span class="p">):</span> <span class="n">name</span> <span class="o">=</span> <span class="s2">"test_plugin"</span> <span class="n">operators</span> <span class="o">=</span> <span class="p">[</span><span class="n">PluginOperator</span><span class="p">]</span> - <span class="n">flask_blueprints</span> <span class="o">=</span> <span class="p">[</span><span class="n">bp</span><span class="p">]</span> <span class="n">hooks</span> <span class="o">=</span> <span class="p">[</span><span class="n">PluginHook</span><span class="p">]</span> <span class="n">executors</span> <span class="o">=</span> <span class="p">[</span><span class="n">PluginExecutor</span><span class="p">]</span> + <span class="n">macros</span> <span class="o">=</span> <span class="p">[</span><span class="n">plugin_macro</span><span class="p">]</span> <span class="n">admin_views</span> <span class="o">=</span> <span class="p">[</span><span class="n">v</span><span class="p">]</span> + <span class="n">flask_blueprints</span> <span class="o">=</span> <span class="p">[</span><span class="n">bp</span><span class="p">]</span> <span class="n">menu_links</span> <span class="o">=</span> <span class="p">[</span><span class="n">ml</span><span class="p">]</span> </pre></div> </div> @@ -274,15 +304,18 @@ definitions in Airflow.</p> </div> + <div class="articleComments"> + + </div> </div> <footer> <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> - <a href="security.html" class="btn btn-neutral float-right" title="Security" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a> + <a href="security.html" class="btn btn-neutral float-right" title="Security" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a> - <a href="scheduler.html" class="btn btn-neutral" title="Scheduling & Triggers" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a> + <a href="scheduler.html" class="btn btn-neutral" title="Scheduling & Triggers" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a> </div> @@ -315,7 +348,8 @@ definitions in Airflow.</p> VERSION:'', COLLAPSE_INDEX:false, FILE_SUFFIX:'.html', - HAS_SOURCE: true + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' }; </script> <script type="text/javascript" src="_static/jquery.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/profiling.html ---------------------------------------------------------------------- diff --git a/profiling.html b/profiling.html index 795ec81..350c015 100644 --- a/profiling.html +++ b/profiling.html @@ -30,6 +30,9 @@ + <link rel="index" title="Index" + href="genindex.html"/> + <link rel="search" title="Search" href="search.html"/> <link rel="top" title="Airflow Documentation" href="index.html"/> <link rel="next" title="Command Line Interface" href="cli.html"/> <link rel="prev" title="Concepts" href="concepts.html"/> @@ -41,6 +44,7 @@ <body class="wy-body-for-nav" role="document"> + <div class="wy-grid-for-nav"> @@ -99,6 +103,8 @@ <li class="toctree-l1"><a class="reference internal" href="scheduler.html">Scheduling & Triggers</a></li> <li class="toctree-l1"><a class="reference internal" href="plugins.html">Plugins</a></li> <li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="api.html">Experimental Rest API</a></li> +<li class="toctree-l1"><a class="reference internal" href="integration.html">Integration</a></li> <li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a></li> </ul> @@ -113,8 +119,10 @@ <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="index.html">Airflow</a> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="index.html">Airflow</a> + </nav> @@ -123,23 +131,40 @@ <div class="rst-content"> - + + + + + + + + + + + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> - <li><a href="index.html">Docs</a> »</li> - - <li>Data Profiling</li> + + <li><a href="index.html">Docs</a> »</li> + + <li>Data Profiling</li> + + <li class="wy-breadcrumbs-aside"> - - <a href="_sources/profiling.txt" rel="nofollow"> View page source</a> + + <a href="_sources/profiling.rst.txt" rel="nofollow"> View page source</a> </li> + </ul> + + <hr/> </div> <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> @@ -181,15 +206,18 @@ directly in the URL.</p> </div> + <div class="articleComments"> + + </div> </div> <footer> <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> - <a href="cli.html" class="btn btn-neutral float-right" title="Command Line Interface" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a> + <a href="cli.html" class="btn btn-neutral float-right" title="Command Line Interface" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a> - <a href="concepts.html" class="btn btn-neutral" title="Concepts" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a> + <a href="concepts.html" class="btn btn-neutral" title="Concepts" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a> </div> @@ -222,7 +250,8 @@ directly in the URL.</p> VERSION:'', COLLAPSE_INDEX:false, FILE_SUFFIX:'.html', - HAS_SOURCE: true + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' }; </script> <script type="text/javascript" src="_static/jquery.js"></script> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/project.html ---------------------------------------------------------------------- diff --git a/project.html b/project.html index d2eb7f2..1bc09d0 100644 --- a/project.html +++ b/project.html @@ -30,6 +30,9 @@ + <link rel="index" title="Index" + href="genindex.html"/> + <link rel="search" title="Search" href="search.html"/> <link rel="top" title="Airflow Documentation" href="index.html"/> <link rel="next" title="License" href="license.html"/> <link rel="prev" title="Apache Airflow (incubating) Documentation" href="index.html"/> @@ -41,6 +44,7 @@ <body class="wy-body-for-nav" role="document"> + <div class="wy-grid-for-nav"> @@ -97,6 +101,8 @@ <li class="toctree-l1"><a class="reference internal" href="scheduler.html">Scheduling & Triggers</a></li> <li class="toctree-l1"><a class="reference internal" href="plugins.html">Plugins</a></li> <li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="api.html">Experimental Rest API</a></li> +<li class="toctree-l1"><a class="reference internal" href="integration.html">Integration</a></li> <li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a></li> </ul> @@ -111,8 +117,10 @@ <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="index.html">Airflow</a> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="index.html">Airflow</a> + </nav> @@ -121,23 +129,40 @@ <div class="rst-content"> - + + + + + + + + + + + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> - <li><a href="index.html">Docs</a> »</li> - - <li>Project</li> + + <li><a href="index.html">Docs</a> »</li> + + <li>Project</li> + + <li class="wy-breadcrumbs-aside"> - - <a href="_sources/project.txt" rel="nofollow"> View page source</a> + + <a href="_sources/project.rst.txt" rel="nofollow"> View page source</a> </li> + </ul> + + <hr/> </div> <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> @@ -147,11 +172,10 @@ <h1>Project<a class="headerlink" href="#project" title="Permalink to this headline">¶</a></h1> <div class="section" id="history"> <h2>History<a class="headerlink" href="#history" title="Permalink to this headline">¶</a></h2> -<p>Airflow was started in the fall of 2014 by Maxime Beauchemin at Airbnb. +<p>Airflow was started in October 2014 by Maxime Beauchemin at Airbnb. It was open source from the very first commit and officially brought under -the Airbnb Github and announced in the spring of 2015.</p> -<p>The project joined the Apache Software Foundation’s incubation program in the -winter of 2016.</p> +the Airbnb Github and announced in June 2015.</p> +<p>The project joined the Apache Software Foundation’s incubation program in March 2016.</p> </div> <div class="section" id="committers"> <h2>Committers<a class="headerlink" href="#committers" title="Permalink to this headline">¶</a></h2> @@ -172,24 +196,14 @@ Contributor page:</a></p> <div class="section" id="resources-links"> <h2>Resources & links<a class="headerlink" href="#resources-links" title="Permalink to this headline">¶</a></h2> <ul class="simple"> +<li><a class="reference external" href="http://airflow.apache.org/">Airflow’s official documentation</a></li> <li>Mailing list (send emails to -<code class="docutils literal"><span class="pre">dev-subscribe@airflow.incubator.apache.org</span></code> and +<code class="docutils literal"><span class="pre">dev-subscribe@airflow.incubator.apache.org</span></code> and/or <code class="docutils literal"><span class="pre">commits-subscribe@airflow.incubator.apache.org</span></code> to subscribe to each)</li> -<li><a class="reference external" href="https://issues.apache.org/jira/browse/AIRFLOW">Issues</a></li> -<li><a class="reference external" href="http://nerds.airbnb.com/airflow/">Airbnb Blog Post about Airflow</a></li> -<li><a class="reference external" href="https://cwiki.apache.org/confluence/display/AIRFLOW/Common+Pitfalls">Airflow Common Pitfalls</a></li> -<li><a class="reference external" href="https://www.youtube.com/watch?v=oYp49mBwH60">Hadoop Summit Airflow Video</a></li> -<li><a class="reference external" href="http://agari.com/blog/airflow-agari">Airflow at Agari Blog Post</a></li> -<li><a class="reference external" href="https://youtu.be/dgaoqOZlvEA">Talk: Best practices with Airflow (nov 2015)</a></li> -<li><a class="reference external" href="https://www.linkedin.com/pulse/airflow-lesson-1-triggerdagrunoperator-siddharth-anand?published=t">Airflow Lesson 1: TriggerDagRunOperator</a></li> -<li><a class="reference external" href="https://github.com/puckel/docker-airflow">Docker Airflow (externally maintained)</a></li> -<li><a class="reference external" href="https://medium.com/handy-tech/airflow-tips-tricks-and-pitfalls-9ba53fba14eb#.o2snqeoz7">Airflow: Tips, Tricks, and Pitfalls @ Handy</a></li> -<li><a class="reference external" href="https://github.com/bahchis/airflow-cookbook">Airflow Chef recipe (community contributed)</a> , -<a class="reference external" href="https://supermarket.chef.io/cookbooks/airflow">another here</a></li> -<li>Airflow Puppet Module (community contributed) <a class="reference external" href="https://github.com/similarweb/puppet-airflow">puppet-airflow</a>, -<a class="reference external" href="https://forge.puppetlabs.com/similarweb/airflow">airflow</a></li> +<li><a class="reference external" href="https://issues.apache.org/jira/browse/AIRFLOW">Issues on Apache’s Jira</a></li> <li><a class="reference external" href="https://gitter.im/airbnb/airflow">Gitter (chat) Channel</a></li> +<li><a class="reference external" href="https://cwiki.apache.org/confluence/display/AIRFLOW/Airflow+Links">More resources and links to Airflow related content on the Wiki</a></li> </ul> </div> <div class="section" id="roadmap"> @@ -200,15 +214,18 @@ to subscribe to each)</li> </div> + <div class="articleComments"> + + </div> </div> <footer> <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> - <a href="license.html" class="btn btn-neutral float-right" title="License" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a> + <a href="license.html" class="btn btn-neutral float-right" title="License" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a> - <a href="index.html" class="btn btn-neutral" title="Apache Airflow (incubating) Documentation" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a> + <a href="index.html" class="btn btn-neutral" title="Apache Airflow (incubating) Documentation" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a> </div> @@ -241,7 +258,8 @@ to subscribe to each)</li> VERSION:'', COLLAPSE_INDEX:false, FILE_SUFFIX:'.html', - HAS_SOURCE: true + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' }; </script> <script type="text/javascript" src="_static/jquery.js"></script> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/py-modindex.html ---------------------------------------------------------------------- diff --git a/py-modindex.html b/py-modindex.html index d377e08..3f84d0f 100644 --- a/py-modindex.html +++ b/py-modindex.html @@ -30,6 +30,9 @@ + <link rel="index" title="Index" + href="genindex.html"/> + <link rel="search" title="Search" href="search.html"/> <link rel="top" title="Airflow Documentation" href="index.html"/> @@ -42,6 +45,7 @@ <body class="wy-body-for-nav" role="document"> + <div class="wy-grid-for-nav"> @@ -92,6 +96,8 @@ <li class="toctree-l1"><a class="reference internal" href="scheduler.html">Scheduling & Triggers</a></li> <li class="toctree-l1"><a class="reference internal" href="plugins.html">Plugins</a></li> <li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="api.html">Experimental Rest API</a></li> +<li class="toctree-l1"><a class="reference internal" href="integration.html">Integration</a></li> <li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a></li> </ul> @@ -106,8 +112,10 @@ <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="index.html">Airflow</a> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="index.html">Airflow</a> + </nav> @@ -120,17 +128,34 @@ + + + + + + + + + + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> - <li><a href="index.html">Docs</a> »</li> - - <li></li> + + <li><a href="index.html">Docs</a> »</li> + + <li></li> + + <li class="wy-breadcrumbs-aside"> - + </li> + </ul> + + <hr/> </div> <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> @@ -143,8 +168,8 @@ <a href="#cap-a"><strong>a</strong></a> </div> - <table class="indextable modindextable" cellspacing="0" cellpadding="2"> - <tr class="pcap"><td></td><td> </td><td></td></tr> + <table class="indextable modindextable"> + <tr class="pcap"><td></td><td> </td><td></td></tr> <tr class="cap" id="cap-a"><td></td><td> <strong>a</strong></td><td></td></tr> <tr> @@ -155,48 +180,51 @@ <em></em></td></tr> <tr class="cg-1"> <td></td> - <td> + <td>    <a href="code.html#module-airflow.contrib.hooks"><code class="xref">airflow.contrib.hooks</code></a></td><td> <em></em></td></tr> <tr class="cg-1"> <td></td> - <td> + <td>    <a href="code.html#module-airflow.contrib.operators"><code class="xref">airflow.contrib.operators</code></a></td><td> <em></em></td></tr> <tr class="cg-1"> <td></td> - <td> + <td>    <a href="code.html#module-airflow.executors"><code class="xref">airflow.executors</code></a></td><td> <em></em></td></tr> <tr class="cg-1"> <td></td> - <td> + <td>    <a href="code.html#module-airflow.hooks"><code class="xref">airflow.hooks</code></a></td><td> <em></em></td></tr> <tr class="cg-1"> <td></td> - <td> + <td>    <a href="code.html#module-airflow.macros"><code class="xref">airflow.macros</code></a></td><td> <em></em></td></tr> <tr class="cg-1"> <td></td> - <td> + <td>    <a href="code.html#module-airflow.macros.hive"><code class="xref">airflow.macros.hive</code></a></td><td> <em></em></td></tr> <tr class="cg-1"> <td></td> - <td> + <td>    <a href="code.html#module-airflow.models"><code class="xref">airflow.models</code></a></td><td> <em></em></td></tr> <tr class="cg-1"> <td></td> - <td> + <td>    <a href="code.html#module-airflow.operators"><code class="xref">airflow.operators</code></a></td><td> <em></em></td></tr> </table> </div> + <div class="articleComments"> + + </div> </div> <footer> @@ -229,7 +257,8 @@ VERSION:'', COLLAPSE_INDEX:false, FILE_SUFFIX:'.html', - HAS_SOURCE: true + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' }; </script> <script type="text/javascript" src="_static/jquery.js"></script> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/scheduler.html ---------------------------------------------------------------------- diff --git a/scheduler.html b/scheduler.html index 7f0bb2c..3f3aa9a 100644 --- a/scheduler.html +++ b/scheduler.html @@ -30,6 +30,9 @@ + <link rel="index" title="Index" + href="genindex.html"/> + <link rel="search" title="Search" href="search.html"/> <link rel="top" title="Airflow Documentation" href="index.html"/> <link rel="next" title="Plugins" href="plugins.html"/> <link rel="prev" title="Command Line Interface" href="cli.html"/> @@ -41,6 +44,7 @@ <body class="wy-body-for-nav" role="document"> + <div class="wy-grid-for-nav"> @@ -90,12 +94,15 @@ <li class="toctree-l1"><a class="reference internal" href="cli.html">Command Line Interface</a></li> <li class="toctree-l1 current"><a class="current reference internal" href="#">Scheduling & Triggers</a><ul> <li class="toctree-l2"><a class="reference internal" href="#dag-runs">DAG Runs</a></li> +<li class="toctree-l2"><a class="reference internal" href="#backfill-and-catchup">Backfill and Catchup</a></li> <li class="toctree-l2"><a class="reference internal" href="#external-triggers">External Triggers</a></li> <li class="toctree-l2"><a class="reference internal" href="#to-keep-in-mind">To Keep in Mind</a></li> </ul> </li> <li class="toctree-l1"><a class="reference internal" href="plugins.html">Plugins</a></li> <li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="api.html">Experimental Rest API</a></li> +<li class="toctree-l1"><a class="reference internal" href="integration.html">Integration</a></li> <li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a></li> </ul> @@ -110,8 +117,10 @@ <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="index.html">Airflow</a> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="index.html">Airflow</a> + </nav> @@ -120,23 +129,40 @@ <div class="rst-content"> - + + + + + + + + + + + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> - <li><a href="index.html">Docs</a> »</li> - - <li>Scheduling & Triggers</li> + + <li><a href="index.html">Docs</a> »</li> + + <li>Scheduling & Triggers</li> + + <li class="wy-breadcrumbs-aside"> - - <a href="_sources/scheduler.txt" rel="nofollow"> View page source</a> + + <a href="_sources/scheduler.rst.txt" rel="nofollow"> View page source</a> </li> + </ul> + + <hr/> </div> <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> @@ -157,6 +183,8 @@ execute <code class="docutils literal"><span class="pre">airflow</span> <span cl the run stamped <code class="docutils literal"><span class="pre">2016-01-01</span></code> will be trigger soon after <code class="docutils literal"><span class="pre">2016-01-01T23:59</span></code>. In other words, the job instance is started once the period it covers has ended.</p> +<p><strong>Let’s Repeat That</strong> The scheduler runs your job one <code class="docutils literal"><span class="pre">schedule_interval</span></code> AFTER the +start date, at the END of the period.</p> <p>The scheduler starts an instance of the executor specified in the your <code class="docutils literal"><span class="pre">airflow.cfg</span></code>. If it happens to be the <code class="docutils literal"><span class="pre">LocalExecutor</span></code>, tasks will be executed as subprocesses; in the case of <code class="docutils literal"><span class="pre">CeleryExecutor</span></code> and @@ -190,11 +218,11 @@ use one of these cron “preset”:</p> <tr class="row-even"><td><code class="docutils literal"><span class="pre">None</span></code></td> <td>Don’t schedule, use for exclusively “externally triggered” DAGs</td> -<td> </td> +<td> </td> </tr> <tr class="row-odd"><td><code class="docutils literal"><span class="pre">@once</span></code></td> <td>Schedule once and only once</td> -<td> </td> +<td> </td> </tr> <tr class="row-even"><td><code class="docutils literal"><span class="pre">@hourly</span></code></td> <td>Run once an hour at the beginning of the hour</td> @@ -228,6 +256,27 @@ should be triggered and come to a crawl. It might also create undesired processing when changing the shape of your DAG, by say adding in new tasks.</p> </div> +<div class="section" id="backfill-and-catchup"> +<h2>Backfill and Catchup<a class="headerlink" href="#backfill-and-catchup" title="Permalink to this headline">¶</a></h2> +<p>An Airflow DAG with a <code class="docutils literal"><span class="pre">start_date</span></code>, possibly an <code class="docutils literal"><span class="pre">end_date</span></code>, and a <code class="docutils literal"><span class="pre">schedule_interval</span></code> defines a +series of intervals which the scheduler turn into individual Dag Runs and execute. A key capability of +Airflow is that these DAG Runs are atomic, idempotent items, and the scheduler, by default, will examine +the lifetime of the DAG (from start to end/now, one interval at a time) and kick off a DAG Run for any +interval that has not been run (or has been cleared). This concept is called Catchup.</p> +<p>If your DAG is written to handle it’s own catchup (IE not limited to the interval, but instead to “Now” +for instance.), then you will want to turn catchup off (Either on the DAG itself with <code class="docutils literal"><span class="pre">dag.catchup</span> <span class="pre">=</span> +<span class="pre">False</span></code>) or by default at the configuration file level with <code class="docutils literal"><span class="pre">catchup_by_default</span> <span class="pre">=</span> <span class="pre">False</span></code>. What this +will do, is to instruct the scheduler to only create a DAG Run for the most current instance of the DAG +interval series.</p> +<p>In the example above, if the DAG is picked up by the scheduler daemon on 2016-01-02 at 6 AM, (or from the +command line), a single DAG Run will be created, with an <code class="docutils literal"><span class="pre">execution_date</span></code> of 2016-01-01, and the next +one will be created just after midnight on the morning of 2016-01-03 with an execution date of 2016-01-02.</p> +<p>If the <code class="docutils literal"><span class="pre">dag.catchup</span></code> value had been True instead, the scheduler would have created a DAG Run for each +completed interval between 2015-12-01 and 2016-01-02 (but not yet one for 2016-01-02, as that interval +hasn’t completed) and the scheduler will execute them sequentially. This behavior is great for atomic +datasets that can easily be split into periods. Turning catchup off is great if your DAG Runs perform +backfill internally.</p> +</div> <div class="section" id="external-triggers"> <h2>External Triggers<a class="headerlink" href="#external-triggers" title="Permalink to this headline">¶</a></h2> <p>Note that <code class="docutils literal"><span class="pre">DAG</span> <span class="pre">Runs</span></code> can also be created manually through the CLI while @@ -259,15 +308,18 @@ whether the scheduler should look into triggering tasks for that run.</li> </div> + <div class="articleComments"> + + </div> </div> <footer> <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> - <a href="plugins.html" class="btn btn-neutral float-right" title="Plugins" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a> + <a href="plugins.html" class="btn btn-neutral float-right" title="Plugins" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a> - <a href="cli.html" class="btn btn-neutral" title="Command Line Interface" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a> + <a href="cli.html" class="btn btn-neutral" title="Command Line Interface" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a> </div> @@ -300,7 +352,8 @@ whether the scheduler should look into triggering tasks for that run.</li> VERSION:'', COLLAPSE_INDEX:false, FILE_SUFFIX:'.html', - HAS_SOURCE: true + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' }; </script> <script type="text/javascript" src="_static/jquery.js"></script> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/search.html ---------------------------------------------------------------------- diff --git a/search.html b/search.html index 9935561..eed50cd 100644 --- a/search.html +++ b/search.html @@ -30,6 +30,9 @@ + <link rel="index" title="Index" + href="genindex.html"/> + <link rel="search" title="Search" href="#"/> <link rel="top" title="Airflow Documentation" href="index.html"/> @@ -39,6 +42,7 @@ <body class="wy-body-for-nav" role="document"> + <div class="wy-grid-for-nav"> @@ -89,6 +93,8 @@ <li class="toctree-l1"><a class="reference internal" href="scheduler.html">Scheduling & Triggers</a></li> <li class="toctree-l1"><a class="reference internal" href="plugins.html">Plugins</a></li> <li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="api.html">Experimental Rest API</a></li> +<li class="toctree-l1"><a class="reference internal" href="integration.html">Integration</a></li> <li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a></li> </ul> @@ -103,8 +109,10 @@ <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="index.html">Airflow</a> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="index.html">Airflow</a> + </nav> @@ -117,15 +125,32 @@ + + + + + + + + + + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> - <li><a href="index.html">Docs</a> »</li> - - <li></li> + + <li><a href="index.html">Docs</a> »</li> + + <li></li> + + <li class="wy-breadcrumbs-aside"> </li> + </ul> + + <hr/> </div> <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> @@ -146,6 +171,9 @@ </div> </div> + <div class="articleComments"> + + </div> </div> <footer> @@ -178,7 +206,8 @@ VERSION:'', COLLAPSE_INDEX:false, FILE_SUFFIX:'.html', - HAS_SOURCE: true + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' }; </script> <script type="text/javascript" src="_static/jquery.js"></script>