Latest docs version as of 1.8.x
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/commit/9c75ee9e Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/tree/9c75ee9e Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/diff/9c75ee9e Branch: refs/heads/asf-site Commit: 9c75ee9e46cc5ec459979dbfd2c911743c07fd63 Parents: 4af0850 Author: Maxime Beauchemin <maxime.beauche...@apache.org> Authored: Mon Mar 6 08:43:12 2017 -0800 Committer: Maxime Beauchemin <maxime.beauche...@apache.org> Committed: Mon Mar 6 08:43:12 2017 -0800 ---------------------------------------------------------------------- .../contrib/executors/mesos_executor.html | 81 +- .../contrib/operators/hipchat_operator.html | 72 +- _modules/airflow/executors/local_executor.html | 73 +- .../airflow/executors/sequential_executor.html | 59 +- _modules/airflow/macros.html | 79 +- _modules/airflow/macros/hive.html | 61 +- _modules/airflow/models.html | 1993 ++++++++++-------- _modules/airflow/operators/sensors.html | 242 ++- _modules/bash_operator.html | 77 +- _modules/dagrun_operator.html | 67 +- _modules/dbapi_hook.html | 130 +- _modules/dummy_operator.html | 49 +- _modules/email_operator.html | 61 +- _modules/ftp_hook.html | 69 +- _modules/generic_transfer.html | 57 +- _modules/http_hook.html | 61 +- _modules/http_operator.html | 71 +- _modules/index.html | 74 +- _modules/mysql_hook.html | 95 +- _modules/mysql_operator.html | 61 +- _modules/presto_check_operator.html | 60 +- _modules/presto_hook.html | 63 +- _modules/python_operator.html | 69 +- _modules/sensors.html | 242 ++- _modules/sqlite_hook.html | 49 +- _modules/ssh_execute_operator.html | 75 +- _modules/ssh_hook.html | 81 +- _static/basic.css | 68 +- _static/comment-bright.png | Bin 3500 -> 756 bytes _static/comment-close.png | Bin 3578 -> 829 bytes _static/comment.png | Bin 3445 -> 641 bytes _static/css/badge_only.css | 2 +- _static/css/theme.css | 4 +- _static/down-pressed.png | Bin 347 -> 222 bytes _static/down.png | Bin 347 -> 202 bytes _static/file.png | Bin 358 -> 286 bytes _static/fonts/Inconsolata-Bold.ttf | Bin 66352 -> 109948 bytes _static/fonts/Inconsolata-Regular.ttf | Bin 84548 -> 96964 bytes _static/fonts/Lato-Bold.ttf | Bin 121788 -> 656544 bytes _static/fonts/Lato-Regular.ttf | Bin 120196 -> 656568 bytes _static/fonts/fontawesome-webfont.eot | Bin 56006 -> 76518 bytes _static/fonts/fontawesome-webfont.svg | 207 +- _static/fonts/fontawesome-webfont.ttf | Bin 112160 -> 152796 bytes _static/fonts/fontawesome-webfont.woff | Bin 65452 -> 90412 bytes _static/jquery.js | 8 +- _static/js/theme.js | 58 +- _static/minus.png | Bin 173 -> 90 bytes _static/plus.png | Bin 173 -> 90 bytes _static/searchtools.js | 115 +- _static/up-pressed.png | Bin 345 -> 214 bytes _static/up.png | Bin 345 -> 203 bytes cli.html | 482 +++-- code.html | 1933 +++-------------- concepts.html | 129 +- configuration.html | 97 +- faq.html | 61 +- genindex.html | 1541 ++++---------- index.html | 115 +- installation.html | 51 +- license.html | 51 +- objects.inv | Bin 2326 -> 2147 bytes plugins.html | 64 +- profiling.html | 51 +- project.html | 76 +- py-modindex.html | 63 +- scheduler.html | 79 +- search.html | 41 +- searchindex.js | 2 +- security.html | 147 +- start.html | 51 +- tutorial.html | 51 +- ui.html | 51 +- 72 files changed, 5097 insertions(+), 4672 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/_modules/airflow/contrib/executors/mesos_executor.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/executors/mesos_executor.html b/_modules/airflow/contrib/executors/mesos_executor.html index e2d8f61..dbca37d 100644 --- a/_modules/airflow/contrib/executors/mesos_executor.html +++ b/_modules/airflow/contrib/executors/mesos_executor.html @@ -30,6 +30,9 @@ + <link rel="index" title="Index" + href="../../../../genindex.html"/> + <link rel="search" title="Search" href="../../../../search.html"/> <link rel="top" title="Airflow Documentation" href="../../../../index.html"/> <link rel="up" title="Module code" href="../../../index.html"/> @@ -40,6 +43,7 @@ <body class="wy-body-for-nav" role="document"> + <div class="wy-grid-for-nav"> @@ -90,6 +94,8 @@ <li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li> </ul> @@ -104,8 +110,10 @@ <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="../../../../index.html">Airflow</a> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../../../../index.html">Airflow</a> + </nav> @@ -118,19 +126,36 @@ + + + + + + + + + + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> - <li><a href="../../../../index.html">Docs</a> »</li> - + + <li><a href="../../../../index.html">Docs</a> »</li> + <li><a href="../../../index.html">Module code</a> »</li> - - <li>airflow.contrib.executors.mesos_executor</li> + + <li>airflow.contrib.executors.mesos_executor</li> + + <li class="wy-breadcrumbs-aside"> - + </li> + </ul> + + <hr/> </div> <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> @@ -151,21 +176,21 @@ <span class="c1"># See the License for the specific language governing permissions and</span> <span class="c1"># limitations under the License.</span> -<span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span> +<span class="kn">from</span> <span class="nn">future</span> <span class="k">import</span> <span class="n">standard_library</span> <span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span> -<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span> +<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span> <span class="kn">import</span> <span class="nn">logging</span> -<span class="kn">from</span> <span class="nn">queue</span> <span class="kn">import</span> <span class="n">Queue</span> +<span class="kn">from</span> <span class="nn">queue</span> <span class="k">import</span> <span class="n">Queue</span> <span class="kn">import</span> <span class="nn">mesos.interface</span> -<span class="kn">from</span> <span class="nn">mesos.interface</span> <span class="kn">import</span> <span class="n">mesos_pb2</span> +<span class="kn">from</span> <span class="nn">mesos.interface</span> <span class="k">import</span> <span class="n">mesos_pb2</span> <span class="kn">import</span> <span class="nn">mesos.native</span> -<span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">configuration</span> -<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="kn">import</span> <span class="n">BaseExecutor</span> -<span class="kn">from</span> <span class="nn">airflow.settings</span> <span class="kn">import</span> <span class="n">Session</span> -<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="kn">import</span> <span class="n">State</span> -<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span> +<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span> +<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="k">import</span> <span class="n">BaseExecutor</span> +<span class="kn">from</span> <span class="nn">airflow.settings</span> <span class="k">import</span> <span class="n">Session</span> +<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span> +<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span> <span class="n">DEFAULT_FRAMEWORK_NAME</span> <span class="o">=</span> <span class="s1">'Airflow'</span> @@ -206,13 +231,13 @@ <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">)</span> <span class="ow">and</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span> <span class="c1"># Import here to work around a circular import error</span> - <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">Connection</span> + <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span> <span class="c1"># Update the Framework ID in the database.</span> <span class="n">session</span> <span class="o">=</span> <span class="n">Session</span><span class="p">()</span> <span class="n">conn_id</span> <span class="o">=</span> <span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">+</span> <span class="n">get_framework_name</span><span class="p">()</span> <span class="n">connection</span> <span class="o">=</span> <span class="n">Session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Connection</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span> - <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span> + <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> <span class="n">connection</span> <span class="o">=</span> <span class="n">Connection</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">,</span> <span class="n">conn_type</span><span class="o">=</span><span class="s1">'mesos_framework-id'</span><span class="p">,</span> <span class="n">extra</span><span class="o">=</span><span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span> <span class="k">else</span><span class="p">:</span> @@ -286,7 +311,7 @@ <span class="n">mem</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span> <span class="n">command</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">CommandInfo</span><span class="p">()</span> - <span class="n">command</span><span class="o">.</span><span class="n">shell</span> <span class="o">=</span> <span class="bp">True</span> + <span class="n">command</span><span class="o">.</span><span class="n">shell</span> <span class="o">=</span> <span class="kc">True</span> <span class="n">command</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">cmd</span> <span class="n">task</span><span class="o">.</span><span class="n">command</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">command</span><span class="p">)</span> @@ -306,7 +331,7 @@ <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span> <span class="c1"># The map may not contain an item if the framework re-registered after a failover.</span> <span class="c1"># Discard these tasks.</span> - <span class="n">logging</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">"Unrecognised task key </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">)</span> + <span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Unrecognised task key </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">)</span> <span class="k">return</span> <span class="k">if</span> <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_FINISHED</span><span class="p">:</span> @@ -357,23 +382,23 @@ <span class="n">task_memory</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">)</span> <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">):</span> - <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="bp">True</span> + <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">True</span> <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span> <span class="c1"># Import here to work around a circular import error</span> - <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">Connection</span> + <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span> <span class="c1"># Query the database to get the ID of the Mesos Framework, if available.</span> <span class="n">conn_id</span> <span class="o">=</span> <span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">+</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span> <span class="n">session</span> <span class="o">=</span> <span class="n">Session</span><span class="p">()</span> <span class="n">connection</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Connection</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span> - <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span> + <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> <span class="c1"># Set the Framework ID to let the scheduler reconnect with running tasks.</span> <span class="n">framework</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">extra</span> <span class="n">framework</span><span class="o">.</span><span class="n">failover_timeout</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">)</span> <span class="k">else</span><span class="p">:</span> - <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="bp">False</span> + <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">False</span> <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'MesosFramework master : </span><span class="si">%s</span><span class="s1">, name : </span><span class="si">%s</span><span class="s1">, cpu : </span><span class="si">%s</span><span class="s1">, mem : </span><span class="si">%s</span><span class="s1">, checkpoint : </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">master</span><span class="p">,</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_cpu</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_memory</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span><span class="p">))</span> @@ -411,7 +436,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span> <span class="o">=</span> <span class="n">driver</span> <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">start</span><span class="p">()</span> - <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span> + <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span> <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> @@ -425,6 +450,9 @@ </pre></div> </div> + <div class="articleComments"> + + </div> </div> <footer> @@ -457,7 +485,8 @@ VERSION:'', COLLAPSE_INDEX:false, FILE_SUFFIX:'.html', - HAS_SOURCE: true + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' }; </script> <script type="text/javascript" src="../../../../_static/jquery.js"></script> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/_modules/airflow/contrib/operators/hipchat_operator.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/hipchat_operator.html b/_modules/airflow/contrib/operators/hipchat_operator.html index 1505a9b..194afad 100644 --- a/_modules/airflow/contrib/operators/hipchat_operator.html +++ b/_modules/airflow/contrib/operators/hipchat_operator.html @@ -30,6 +30,9 @@ + <link rel="index" title="Index" + href="../../../../genindex.html"/> + <link rel="search" title="Search" href="../../../../search.html"/> <link rel="top" title="Airflow Documentation" href="../../../../index.html"/> <link rel="up" title="Module code" href="../../../index.html"/> @@ -40,6 +43,7 @@ <body class="wy-body-for-nav" role="document"> + <div class="wy-grid-for-nav"> @@ -90,6 +94,8 @@ <li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li> </ul> @@ -104,8 +110,10 @@ <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="../../../../index.html">Airflow</a> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../../../../index.html">Airflow</a> + </nav> @@ -118,19 +126,36 @@ + + + + + + + + + + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> - <li><a href="../../../../index.html">Docs</a> »</li> - + + <li><a href="../../../../index.html">Docs</a> »</li> + <li><a href="../../../index.html">Module code</a> »</li> - - <li>airflow.contrib.operators.hipchat_operator</li> + + <li>airflow.contrib.operators.hipchat_operator</li> + + <li class="wy-breadcrumbs-aside"> - + </li> + </ul> + + <hr/> </div> <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> @@ -151,9 +176,11 @@ <span class="c1"># See the License for the specific language governing permissions and</span> <span class="c1"># limitations under the License.</span> -<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> -<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> -<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span> +<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span> + +<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span> +<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span> +<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span> <span class="kn">import</span> <span class="nn">logging</span> <span class="kn">import</span> <span class="nn">requests</span> <span class="kn">import</span> <span class="nn">json</span> @@ -166,7 +193,6 @@ <span class="sd"> at https://www.hipchat.com/docs/apiv2. Before using any HipChat API operators you need</span> <span class="sd"> to get an authentication token at https://www.hipchat.com/docs/apiv2/auth.</span> <span class="sd"> In the future additional HipChat operators will be derived from this class as well.</span> - <span class="sd"> :param token: HipChat REST API authentication token</span> <span class="sd"> :type token: str</span> <span class="sd"> :param base_url: HipChat REST API base url.</span> @@ -181,15 +207,14 @@ <span class="nb">super</span><span class="p">(</span><span class="n">HipChatAPIOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">token</span> <span class="o">=</span> <span class="n">token</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">=</span> <span class="n">base_url</span> - <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">=</span> <span class="bp">None</span> - <span class="bp">self</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="bp">None</span> - <span class="bp">self</span><span class="o">.</span><span class="n">body</span> <span class="o">=</span> <span class="bp">None</span> + <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">=</span> <span class="kc">None</span> + <span class="bp">self</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="kc">None</span> + <span class="bp">self</span><span class="o">.</span><span class="n">body</span> <span class="o">=</span> <span class="kc">None</span> <span class="k">def</span> <span class="nf">prepare_request</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> <span class="sd">"""</span> <span class="sd"> Used by the execute function. Set the request method, url, and body of HipChat's</span> <span class="sd"> REST API call.</span> - <span class="sd"> Override in child class. Each HipChatAPI child operator is responsible for having</span> <span class="sd"> a prepare_request method call which sets self.method, self.url, and self.body.</span> <span class="sd"> """</span> @@ -215,7 +240,6 @@ <span class="sd">"""</span> <span class="sd"> Send notification to a specific HipChat room.</span> <span class="sd"> More info: https://www.hipchat.com/docs/apiv2/method/send_room_notification</span> - <span class="sd"> :param room_id: Room in which to send notification on HipChat</span> <span class="sd"> :type room_id: str</span> <span class="sd"> :param message: The message body</span> @@ -245,11 +269,11 @@ <span class="s1">'message_format'</span><span class="p">:</span> <span class="s1">'html'</span><span class="p">,</span> <span class="s1">'color'</span><span class="p">:</span> <span class="s1">'yellow'</span><span class="p">,</span> <span class="s1">'frm'</span><span class="p">:</span> <span class="s1">'airflow'</span><span class="p">,</span> - <span class="s1">'attach_to'</span><span class="p">:</span> <span class="bp">None</span><span class="p">,</span> - <span class="s1">'notify'</span><span class="p">:</span> <span class="bp">False</span><span class="p">,</span> - <span class="s1">'card'</span><span class="p">:</span> <span class="bp">None</span> + <span class="s1">'attach_to'</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span> + <span class="s1">'notify'</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span> + <span class="s1">'card'</span><span class="p">:</span> <span class="kc">None</span> <span class="p">}</span> - <span class="k">for</span> <span class="p">(</span><span class="n">prop</span><span class="p">,</span> <span class="n">default</span><span class="p">)</span> <span class="ow">in</span> <span class="n">default_options</span><span class="o">.</span><span class="n">iteritems</span><span class="p">():</span> + <span class="k">for</span> <span class="p">(</span><span class="n">prop</span><span class="p">,</span> <span class="n">default</span><span class="p">)</span> <span class="ow">in</span> <span class="n">default_options</span><span class="o">.</span><span class="n">items</span><span class="p">():</span> <span class="nb">setattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">prop</span><span class="p">,</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">prop</span><span class="p">,</span> <span class="n">default</span><span class="p">))</span> <span class="k">def</span> <span class="nf">prepare_request</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> @@ -266,10 +290,13 @@ <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">=</span> <span class="s1">'POST'</span> <span class="bp">self</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="s1">'</span><span class="si">%s</span><span class="s1">/room/</span><span class="si">%s</span><span class="s1">/notification'</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">base_url</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">room_id</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">body</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span> - <span class="p">(</span><span class="n">k</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">),</span> <span class="n">v</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">))</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">params</span><span class="o">.</span><span class="n">iteritems</span><span class="p">()</span> <span class="k">if</span> <span class="n">v</span><span class="p">))</span></div> + <span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">k</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">v</span><span class="p">))</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">params</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">v</span><span class="p">))</span></div> </pre></div> </div> + <div class="articleComments"> + + </div> </div> <footer> @@ -302,7 +329,8 @@ VERSION:'', COLLAPSE_INDEX:false, FILE_SUFFIX:'.html', - HAS_SOURCE: true + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' }; </script> <script type="text/javascript" src="../../../../_static/jquery.js"></script> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/_modules/airflow/executors/local_executor.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/executors/local_executor.html b/_modules/airflow/executors/local_executor.html index 250eafd..2e35470 100644 --- a/_modules/airflow/executors/local_executor.html +++ b/_modules/airflow/executors/local_executor.html @@ -30,6 +30,9 @@ + <link rel="index" title="Index" + href="../../../genindex.html"/> + <link rel="search" title="Search" href="../../../search.html"/> <link rel="top" title="Airflow Documentation" href="../../../index.html"/> <link rel="up" title="Module code" href="../../index.html"/> @@ -40,6 +43,7 @@ <body class="wy-body-for-nav" role="document"> + <div class="wy-grid-for-nav"> @@ -90,6 +94,8 @@ <li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li> </ul> @@ -104,8 +110,10 @@ <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="../../../index.html">Airflow</a> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../../../index.html">Airflow</a> + </nav> @@ -118,19 +126,36 @@ + + + + + + + + + + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> - <li><a href="../../../index.html">Docs</a> »</li> - + + <li><a href="../../../index.html">Docs</a> »</li> + <li><a href="../../index.html">Module code</a> »</li> - - <li>airflow.executors.local_executor</li> + + <li>airflow.executors.local_executor</li> + + <li class="wy-breadcrumbs-aside"> - + </li> + </ul> + + <hr/> </div> <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> @@ -155,12 +180,12 @@ <span class="kn">import</span> <span class="nn">subprocess</span> <span class="kn">import</span> <span class="nn">time</span> -<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">range</span> +<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">range</span> -<span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">configuration</span> -<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="kn">import</span> <span class="n">BaseExecutor</span> -<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="kn">import</span> <span class="n">State</span> -<span class="kn">from</span> <span class="nn">airflow.utils.logging</span> <span class="kn">import</span> <span class="n">LoggingMixin</span> +<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span> +<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="k">import</span> <span class="n">BaseExecutor</span> +<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span> +<span class="kn">from</span> <span class="nn">airflow.utils.logging</span> <span class="k">import</span> <span class="n">LoggingMixin</span> <span class="n">PARALLELISM</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'PARALLELISM'</span><span class="p">)</span> @@ -171,24 +196,24 @@ <span class="n">multiprocessing</span><span class="o">.</span><span class="n">Process</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">task_queue</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">result_queue</span> - <span class="bp">self</span><span class="o">.</span><span class="n">daemon</span> <span class="o">=</span> <span class="bp">True</span> + <span class="bp">self</span><span class="o">.</span><span class="n">daemon</span> <span class="o">=</span> <span class="kc">True</span> <span class="k">def</span> <span class="nf">run</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> - <span class="k">while</span> <span class="bp">True</span><span class="p">:</span> + <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> - <span class="k">if</span> <span class="n">key</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span> + <span class="k">if</span> <span class="n">key</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> <span class="c1"># Received poison pill, no more tasks to run</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span> <span class="k">break</span> - <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"{} running {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span> + <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"</span><span class="si">{}</span><span class="s2"> running </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span> <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span> - <span class="n">command</span> <span class="o">=</span> <span class="s2">"exec bash -c '{0}'"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">command</span><span class="p">)</span> + <span class="n">command</span> <span class="o">=</span> <span class="s2">"exec bash -c '</span><span class="si">{0}</span><span class="s2">'"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">command</span><span class="p">)</span> <span class="k">try</span><span class="p">:</span> - <span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">(</span><span class="n">command</span><span class="p">,</span> <span class="n">shell</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span> + <span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">(</span><span class="n">command</span><span class="p">,</span> <span class="n">shell</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> <span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span> <span class="k">except</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">CalledProcessError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> <span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span> - <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"failed to execute task {}:"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)))</span> + <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"failed to execute task </span><span class="si">{}</span><span class="s2">:"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)))</span> <span class="c1"># raise e</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">state</span><span class="p">))</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span> @@ -213,7 +238,7 @@ <span class="k">for</span> <span class="n">w</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">workers</span><span class="p">:</span> <span class="n">w</span><span class="o">.</span><span class="n">start</span><span class="p">()</span> - <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span> + <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span> <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> @@ -224,7 +249,7 @@ <span class="k">def</span> <span class="nf">end</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> <span class="c1"># Sending poison pill to all worker</span> <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">workers</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="bp">None</span><span class="p">,</span> <span class="bp">None</span><span class="p">))</span> + <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">))</span> <span class="c1"># Wait for commands to finish</span> <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">join</span><span class="p">()</span> @@ -232,6 +257,9 @@ </pre></div> </div> + <div class="articleComments"> + + </div> </div> <footer> @@ -264,7 +292,8 @@ VERSION:'', COLLAPSE_INDEX:false, FILE_SUFFIX:'.html', - HAS_SOURCE: true + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' }; </script> <script type="text/javascript" src="../../../_static/jquery.js"></script> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/_modules/airflow/executors/sequential_executor.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/executors/sequential_executor.html b/_modules/airflow/executors/sequential_executor.html index ce84843..d5644c2 100644 --- a/_modules/airflow/executors/sequential_executor.html +++ b/_modules/airflow/executors/sequential_executor.html @@ -30,6 +30,9 @@ + <link rel="index" title="Index" + href="../../../genindex.html"/> + <link rel="search" title="Search" href="../../../search.html"/> <link rel="top" title="Airflow Documentation" href="../../../index.html"/> <link rel="up" title="Module code" href="../../index.html"/> @@ -40,6 +43,7 @@ <body class="wy-body-for-nav" role="document"> + <div class="wy-grid-for-nav"> @@ -90,6 +94,8 @@ <li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li> </ul> @@ -104,8 +110,10 @@ <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="../../../index.html">Airflow</a> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../../../index.html">Airflow</a> + </nav> @@ -118,19 +126,36 @@ + + + + + + + + + + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> - <li><a href="../../../index.html">Docs</a> »</li> - + + <li><a href="../../../index.html">Docs</a> »</li> + <li><a href="../../index.html">Module code</a> »</li> - - <li>airflow.executors.sequential_executor</li> + + <li>airflow.executors.sequential_executor</li> + + <li class="wy-breadcrumbs-aside"> - + </li> + </ul> + + <hr/> </div> <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> @@ -151,11 +176,11 @@ <span class="c1"># See the License for the specific language governing permissions and</span> <span class="c1"># limitations under the License.</span> -<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span> +<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span> <span class="kn">import</span> <span class="nn">subprocess</span> -<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="kn">import</span> <span class="n">BaseExecutor</span> -<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="kn">import</span> <span class="n">State</span> +<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="k">import</span> <span class="n">BaseExecutor</span> +<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span> <div class="viewcode-block" id="SequentialExecutor"><a class="viewcode-back" href="../../../code.html#airflow.executors.SequentialExecutor">[docs]</a><span class="k">class</span> <span class="nc">SequentialExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">):</span> @@ -171,19 +196,19 @@ <span class="nb">super</span><span class="p">(</span><span class="n">SequentialExecutor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span> <span class="bp">self</span><span class="o">.</span><span class="n">commands_to_run</span> <span class="o">=</span> <span class="p">[]</span> - <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span> + <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> <span class="bp">self</span><span class="o">.</span><span class="n">commands_to_run</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,))</span> <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">commands_to_run</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Executing command: {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">command</span><span class="p">))</span> + <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Executing command: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">command</span><span class="p">))</span> <span class="k">try</span><span class="p">:</span> - <span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">(</span><span class="n">command</span><span class="p">,</span> <span class="n">shell</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span> + <span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">(</span><span class="n">command</span><span class="p">,</span> <span class="n">shell</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">change_state</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">)</span> <span class="k">except</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">CalledProcessError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">change_state</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">)</span> - <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Failed to execute task {}:"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)))</span> + <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Failed to execute task </span><span class="si">{}</span><span class="s2">:"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)))</span> <span class="bp">self</span><span class="o">.</span><span class="n">commands_to_run</span> <span class="o">=</span> <span class="p">[]</span> @@ -192,6 +217,9 @@ </pre></div> </div> + <div class="articleComments"> + + </div> </div> <footer> @@ -224,7 +252,8 @@ VERSION:'', COLLAPSE_INDEX:false, FILE_SUFFIX:'.html', - HAS_SOURCE: true + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' }; </script> <script type="text/javascript" src="../../../_static/jquery.js"></script> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/_modules/airflow/macros.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/macros.html b/_modules/airflow/macros.html index 19c18ff..34826d7 100644 --- a/_modules/airflow/macros.html +++ b/_modules/airflow/macros.html @@ -30,6 +30,9 @@ + <link rel="index" title="Index" + href="../../genindex.html"/> + <link rel="search" title="Search" href="../../search.html"/> <link rel="top" title="Airflow Documentation" href="../../index.html"/> <link rel="up" title="Module code" href="../index.html"/> @@ -40,6 +43,7 @@ <body class="wy-body-for-nav" role="document"> + <div class="wy-grid-for-nav"> @@ -90,6 +94,8 @@ <li class="toctree-l1"><a class="reference internal" href="../../scheduler.html">Scheduling & Triggers</a></li> <li class="toctree-l1"><a class="reference internal" href="../../plugins.html">Plugins</a></li> <li class="toctree-l1"><a class="reference internal" href="../../security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../api.html">Experimental Rest API</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../integration.html">Integration</a></li> <li class="toctree-l1"><a class="reference internal" href="../../faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="../../code.html">API Reference</a></li> </ul> @@ -104,8 +110,10 @@ <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="../../index.html">Airflow</a> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../../index.html">Airflow</a> + </nav> @@ -118,19 +126,36 @@ + + + + + + + + + + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> - <li><a href="../../index.html">Docs</a> »</li> - + + <li><a href="../../index.html">Docs</a> »</li> + <li><a href="../index.html">Module code</a> »</li> - - <li>airflow.macros</li> + + <li>airflow.macros</li> + + <li class="wy-breadcrumbs-aside"> - + </li> + </ul> + + <hr/> </div> <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> @@ -151,12 +176,12 @@ <span class="c1"># See the License for the specific language governing permissions and</span> <span class="c1"># limitations under the License.</span> -<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span> -<span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">random</span> -<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span> +<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">absolute_import</span> +<span class="kn">from</span> <span class="nn">random</span> <span class="k">import</span> <span class="n">random</span> +<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span> <span class="kn">import</span> <span class="nn">dateutil</span> <span class="kn">import</span> <span class="nn">time</span> -<span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">hive</span> +<span class="kn">from</span> <span class="nn">.</span> <span class="k">import</span> <span class="n">hive</span> <span class="kn">import</span> <span class="nn">uuid</span> @@ -204,28 +229,33 @@ <span class="k">def</span> <span class="nf">_integrate_plugins</span><span class="p">():</span> <span class="sd">"""Integrate plugins to the context"""</span> <span class="kn">import</span> <span class="nn">sys</span> - <span class="kn">from</span> <span class="nn">airflow.plugins_manager</span> <span class="kn">import</span> <span class="n">macros</span> <span class="k">as</span> <span class="n">_macros</span> - <span class="k">for</span> <span class="n">_macro_module</span> <span class="ow">in</span> <span class="n">_macros</span><span class="p">:</span> - <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="n">_macro_module</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro_module</span> - <span class="nb">globals</span><span class="p">()[</span><span class="n">_macro_module</span><span class="o">.</span><span class="n">_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro_module</span> + <span class="kn">from</span> <span class="nn">airflow.plugins_manager</span> <span class="k">import</span> <span class="n">macros_modules</span> + <span class="k">for</span> <span class="n">macros_module</span> <span class="ow">in</span> <span class="n">macros_modules</span><span class="p">:</span> + <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="n">macros_module</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">macros_module</span> + <span class="nb">globals</span><span class="p">()[</span><span class="n">macros_module</span><span class="o">.</span><span class="n">_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">macros_module</span> <span class="c1">##########################################################</span> <span class="c1"># TODO FIXME Remove in Airflow 2.0</span> - <span class="kn">import</span> <span class="nn">os</span> <span class="kn">as</span> <span class="nn">_os</span> - <span class="k">if</span> <span class="ow">not</span> <span class="n">_os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'AIRFLOW_USE_NEW_IMPORTS'</span><span class="p">,</span> <span class="bp">False</span><span class="p">):</span> - <span class="kn">from</span> <span class="nn">zope.deprecation</span> <span class="kn">import</span> <span class="n">deprecated</span> <span class="k">as</span> <span class="n">_deprecated</span> - <span class="k">for</span> <span class="n">_macro</span> <span class="ow">in</span> <span class="n">_macro_module</span><span class="o">.</span><span class="n">_objects</span><span class="p">:</span> - <span class="nb">globals</span><span class="p">()[</span><span class="n">_macro</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">_deprecated</span><span class="p">(</span> - <span class="n">_macro</span><span class="p">,</span> - <span class="s2">"Importing plugin macro '{i}' directly from "</span> + <span class="kn">import</span> <span class="nn">os</span> <span class="k">as</span> <span class="nn">_os</span> + <span class="k">if</span> <span class="ow">not</span> <span class="n">_os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'AIRFLOW_USE_NEW_IMPORTS'</span><span class="p">,</span> <span class="kc">False</span><span class="p">):</span> + <span class="kn">from</span> <span class="nn">zope.deprecation</span> <span class="k">import</span> <span class="n">deprecated</span> <span class="k">as</span> <span class="n">_deprecated</span> + <span class="k">for</span> <span class="n">_macro</span> <span class="ow">in</span> <span class="n">macros_module</span><span class="o">.</span><span class="n">_objects</span><span class="p">:</span> + <span class="n">macro_name</span> <span class="o">=</span> <span class="n">_macro</span><span class="o">.</span><span class="n">__name__</span> + <span class="nb">globals</span><span class="p">()[</span><span class="n">macro_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro</span> + <span class="n">_deprecated</span><span class="p">(</span> + <span class="n">macro_name</span><span class="p">,</span> + <span class="s2">"Importing plugin macro '</span><span class="si">{i}</span><span class="s2">' directly from "</span> <span class="s2">"'airflow.macros' has been deprecated. Please "</span> <span class="s2">"import from 'airflow.macros.[plugin_module]' "</span> <span class="s2">"instead. Support for direct imports will be dropped "</span> - <span class="s2">"entirely in Airflow 2.0."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="o">=</span><span class="n">_macro</span><span class="p">))</span> + <span class="s2">"entirely in Airflow 2.0."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="o">=</span><span class="n">macro_name</span><span class="p">))</span> </pre></div> </div> + <div class="articleComments"> + + </div> </div> <footer> @@ -258,7 +288,8 @@ VERSION:'', COLLAPSE_INDEX:false, FILE_SUFFIX:'.html', - HAS_SOURCE: true + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' }; </script> <script type="text/javascript" src="../../_static/jquery.js"></script> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/_modules/airflow/macros/hive.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/macros/hive.html b/_modules/airflow/macros/hive.html index 92000d3..8d13632 100644 --- a/_modules/airflow/macros/hive.html +++ b/_modules/airflow/macros/hive.html @@ -30,6 +30,9 @@ + <link rel="index" title="Index" + href="../../../genindex.html"/> + <link rel="search" title="Search" href="../../../search.html"/> <link rel="top" title="Airflow Documentation" href="../../../index.html"/> <link rel="up" title="airflow.macros" href="../macros.html"/> @@ -40,6 +43,7 @@ <body class="wy-body-for-nav" role="document"> + <div class="wy-grid-for-nav"> @@ -90,6 +94,8 @@ <li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li> </ul> @@ -104,8 +110,10 @@ <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="../../../index.html">Airflow</a> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../../../index.html">Airflow</a> + </nav> @@ -118,21 +126,38 @@ + + + + + + + + + + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> - <li><a href="../../../index.html">Docs</a> »</li> - + + <li><a href="../../../index.html">Docs</a> »</li> + <li><a href="../../index.html">Module code</a> »</li> - + <li><a href="../macros.html">airflow.macros</a> »</li> - - <li>airflow.macros.hive</li> + + <li>airflow.macros.hive</li> + + <li class="wy-breadcrumbs-aside"> - + </li> + </ul> + + <hr/> </div> <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> @@ -157,7 +182,7 @@ <div class="viewcode-block" id="max_partition"><a class="viewcode-back" href="../../../code.html#airflow.macros.hive.max_partition">[docs]</a><span class="k">def</span> <span class="nf">max_partition</span><span class="p">(</span> - <span class="n">table</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="s2">"default"</span><span class="p">,</span> <span class="n">field</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="nb">filter</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> + <span class="n">table</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="s2">"default"</span><span class="p">,</span> <span class="n">field</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="nb">filter</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">metastore_conn_id</span><span class="o">=</span><span class="s1">'metastore_default'</span><span class="p">):</span> <span class="sd">'''</span> <span class="sd"> Gets the max partition for a table.</span> @@ -180,7 +205,7 @@ <span class="sd"> >>> max_partition('airflow.static_babynames_partitioned')</span> <span class="sd"> '2015-01-01'</span> <span class="sd"> '''</span> - <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span> + <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveMetastoreHook</span> <span class="k">if</span> <span class="s1">'.'</span> <span class="ow">in</span> <span class="n">table</span><span class="p">:</span> <span class="n">schema</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span> <span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span><span class="n">metastore_conn_id</span><span class="o">=</span><span class="n">metastore_conn_id</span><span class="p">)</span> @@ -188,7 +213,7 @@ <span class="n">schema</span><span class="o">=</span><span class="n">schema</span><span class="p">,</span> <span class="n">table_name</span><span class="o">=</span><span class="n">table</span><span class="p">,</span> <span class="n">field</span><span class="o">=</span><span class="n">field</span><span class="p">,</span> <span class="nb">filter</span><span class="o">=</span><span class="nb">filter</span><span class="p">)</span></div> -<span class="k">def</span> <span class="nf">_closest_date</span><span class="p">(</span><span class="n">target_dt</span><span class="p">,</span> <span class="n">date_list</span><span class="p">,</span> <span class="n">before_target</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span> +<span class="k">def</span> <span class="nf">_closest_date</span><span class="p">(</span><span class="n">target_dt</span><span class="p">,</span> <span class="n">date_list</span><span class="p">,</span> <span class="n">before_target</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> <span class="sd">'''</span> <span class="sd"> This function finds the date in a list closest to the target date.</span> <span class="sd"> An optional parameter can be given to get the closest before or after.</span> @@ -205,7 +230,7 @@ <span class="n">fb</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">d</span><span class="p">:</span> <span class="n">d</span> <span class="o">-</span> <span class="n">target_dt</span> <span class="k">if</span> <span class="n">d</span> <span class="o">>=</span> <span class="n">target_dt</span> <span class="k">else</span> <span class="n">datetime</span><span class="o">.</span><span class="n">timedelta</span><span class="o">.</span><span class="n">max</span> <span class="n">fa</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">d</span><span class="p">:</span> <span class="n">d</span> <span class="o">-</span> <span class="n">target_dt</span> <span class="k">if</span> <span class="n">d</span> <span class="o"><=</span> <span class="n">target_dt</span> <span class="k">else</span> <span class="n">datetime</span><span class="o">.</span><span class="n">timedelta</span><span class="o">.</span><span class="n">min</span> <span class="n">fnone</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">d</span><span class="p">:</span> <span class="n">target_dt</span> <span class="o">-</span> <span class="n">d</span> <span class="k">if</span> <span class="n">d</span> <span class="o"><</span> <span class="n">target_dt</span> <span class="k">else</span> <span class="n">d</span> <span class="o">-</span> <span class="n">target_dt</span> - <span class="k">if</span> <span class="n">before_target</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span> + <span class="k">if</span> <span class="n">before_target</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> <span class="k">return</span> <span class="nb">min</span><span class="p">(</span><span class="n">date_list</span><span class="p">,</span> <span class="n">key</span><span class="o">=</span><span class="n">fnone</span><span class="p">)</span><span class="o">.</span><span class="n">date</span><span class="p">()</span> <span class="k">if</span> <span class="n">before_target</span><span class="p">:</span> <span class="k">return</span> <span class="nb">min</span><span class="p">(</span><span class="n">date_list</span><span class="p">,</span> <span class="n">key</span><span class="o">=</span><span class="n">fb</span><span class="p">)</span><span class="o">.</span><span class="n">date</span><span class="p">()</span> @@ -214,7 +239,7 @@ <div class="viewcode-block" id="closest_ds_partition"><a class="viewcode-back" href="../../../code.html#airflow.macros.hive.closest_ds_partition">[docs]</a><span class="k">def</span> <span class="nf">closest_ds_partition</span><span class="p">(</span> - <span class="n">table</span><span class="p">,</span> <span class="n">ds</span><span class="p">,</span> <span class="n">before</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="s2">"default"</span><span class="p">,</span> + <span class="n">table</span><span class="p">,</span> <span class="n">ds</span><span class="p">,</span> <span class="n">before</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="s2">"default"</span><span class="p">,</span> <span class="n">metastore_conn_id</span><span class="o">=</span><span class="s1">'metastore_default'</span><span class="p">):</span> <span class="sd">'''</span> <span class="sd"> This function finds the date in a list closest to the target date.</span> @@ -233,13 +258,13 @@ <span class="sd"> >>> closest_ds_partition(tbl, '2015-01-02')</span> <span class="sd"> '2015-01-01'</span> <span class="sd"> '''</span> - <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span> + <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveMetastoreHook</span> <span class="k">if</span> <span class="s1">'.'</span> <span class="ow">in</span> <span class="n">table</span><span class="p">:</span> <span class="n">schema</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span> <span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span><span class="n">metastore_conn_id</span><span class="o">=</span><span class="n">metastore_conn_id</span><span class="p">)</span> <span class="n">partitions</span> <span class="o">=</span> <span class="n">hh</span><span class="o">.</span><span class="n">get_partitions</span><span class="p">(</span><span class="n">schema</span><span class="o">=</span><span class="n">schema</span><span class="p">,</span> <span class="n">table_name</span><span class="o">=</span><span class="n">table</span><span class="p">)</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">partitions</span><span class="p">:</span> - <span class="k">return</span> <span class="bp">None</span> + <span class="k">return</span> <span class="kc">None</span> <span class="n">part_vals</span> <span class="o">=</span> <span class="p">[</span><span class="nb">list</span><span class="p">(</span><span class="n">p</span><span class="o">.</span><span class="n">values</span><span class="p">())[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">partitions</span><span class="p">]</span> <span class="k">if</span> <span class="n">ds</span> <span class="ow">in</span> <span class="n">part_vals</span><span class="p">:</span> <span class="k">return</span> <span class="n">ds</span> @@ -252,6 +277,9 @@ </pre></div> </div> + <div class="articleComments"> + + </div> </div> <footer> @@ -284,7 +312,8 @@ VERSION:'', COLLAPSE_INDEX:false, FILE_SUFFIX:'.html', - HAS_SOURCE: true + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' }; </script> <script type="text/javascript" src="../../../_static/jquery.js"></script>