Docs from 1.10.1
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/commit/1f06fa0e Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/tree/1f06fa0e Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/diff/1f06fa0e Branch: refs/heads/asf-site Commit: 1f06fa0e0ead70d24f3c87326734aec89930207d Parents: 7d4d762 Author: Ash Berlin-Taylor <ash_git...@firemirror.com> Authored: Thu Nov 29 15:57:25 2018 +0000 Committer: Ash Berlin-Taylor <ash_git...@firemirror.com> Committed: Thu Nov 29 15:57:25 2018 +0000 ---------------------------------------------------------------------- _images/airflow.gif | Bin 622963 -> 622832 bytes .../contrib/executors/mesos_executor.html | 548 --- .../contrib/hooks/aws_dynamodb_hook.html | 21 +- .../contrib/hooks/aws_firehose_hook.html | 267 ++ _modules/airflow/contrib/hooks/aws_hook.html | 104 +- .../airflow/contrib/hooks/aws_lambda_hook.html | 21 +- .../contrib/hooks/azure_data_lake_hook.html | 21 +- .../contrib/hooks/azure_fileshare_hook.html | 21 +- .../airflow/contrib/hooks/bigquery_hook.html | 139 +- .../airflow/contrib/hooks/cassandra_hook.html | 21 +- .../airflow/contrib/hooks/cloudant_hook.html | 21 +- .../airflow/contrib/hooks/databricks_hook.html | 87 +- .../airflow/contrib/hooks/datadog_hook.html | 21 +- .../airflow/contrib/hooks/datastore_hook.html | 21 +- .../contrib/hooks/discord_webhook_hook.html | 21 +- _modules/airflow/contrib/hooks/emr_hook.html | 36 +- _modules/airflow/contrib/hooks/fs_hook.html | 21 +- _modules/airflow/contrib/hooks/ftp_hook.html | 91 +- .../contrib/hooks/gcp_api_base_hook.html | 23 +- .../contrib/hooks/gcp_container_hook.html | 25 +- .../contrib/hooks/gcp_dataflow_hook.html | 128 +- .../contrib/hooks/gcp_dataproc_hook.html | 21 +- .../contrib/hooks/gcp_function_hook.html | 406 +++ .../contrib/hooks/gcp_mlengine_hook.html | 32 +- .../airflow/contrib/hooks/gcp_pubsub_hook.html | 21 +- .../airflow/contrib/hooks/gcp_sql_hook.html | 474 +++ _modules/airflow/contrib/hooks/gcs_hook.html | 60 +- _modules/airflow/contrib/hooks/imap_hook.html | 491 +++ .../airflow/contrib/hooks/jenkins_hook.html | 21 +- _modules/airflow/contrib/hooks/jira_hook.html | 21 +- _modules/airflow/contrib/hooks/mongo_hook.html | 21 +- _modules/airflow/contrib/hooks/pinot_hook.html | 21 +- _modules/airflow/contrib/hooks/qubole_hook.html | 21 +- _modules/airflow/contrib/hooks/redis_hook.html | 21 +- .../airflow/contrib/hooks/redshift_hook.html | 21 +- .../airflow/contrib/hooks/sagemaker_hook.html | 972 ++++++ .../airflow/contrib/hooks/segment_hook.html | 21 +- _modules/airflow/contrib/hooks/sftp_hook.html | 98 +- .../contrib/hooks/slack_webhook_hook.html | 21 +- .../airflow/contrib/hooks/snowflake_hook.html | 21 +- .../airflow/contrib/hooks/spark_jdbc_hook.html | 21 +- .../airflow/contrib/hooks/spark_sql_hook.html | 21 +- .../contrib/hooks/spark_submit_hook.html | 21 +- _modules/airflow/contrib/hooks/sqoop_hook.html | 21 +- _modules/airflow/contrib/hooks/ssh_hook.html | 315 +- .../airflow/contrib/hooks/vertica_hook.html | 21 +- _modules/airflow/contrib/hooks/wasb_hook.html | 64 +- _modules/airflow/contrib/hooks/winrm_hook.html | 21 +- _modules/airflow/contrib/kubernetes/secret.html | 21 +- .../contrib/operators/awsbatch_operator.html | 47 +- .../operators/bigquery_check_operator.html | 71 +- .../contrib/operators/bigquery_get_data.html | 25 +- .../contrib/operators/bigquery_operator.html | 202 +- .../bigquery_table_delete_operator.html | 25 +- .../contrib/operators/bigquery_to_bigquery.html | 33 +- .../contrib/operators/bigquery_to_gcs.html | 34 +- .../contrib/operators/cassandra_to_gcs.html | 25 +- .../contrib/operators/databricks_operator.html | 37 +- .../contrib/operators/dataflow_operator.html | 104 +- .../contrib/operators/dataproc_operator.html | 130 +- .../operators/datastore_export_operator.html | 25 +- .../operators/datastore_import_operator.html | 25 +- .../operators/discord_webhook_operator.html | 21 +- .../contrib/operators/druid_operator.html | 25 +- .../airflow/contrib/operators/ecs_operator.html | 52 +- .../operators/emr_add_steps_operator.html | 25 +- .../operators/emr_create_job_flow_operator.html | 25 +- .../emr_terminate_job_flow_operator.html | 25 +- .../airflow/contrib/operators/file_to_gcs.html | 34 +- .../airflow/contrib/operators/file_to_wasb.html | 21 +- .../contrib/operators/gcp_compute_operator.html | 394 +++ .../operators/gcp_container_operator.html | 29 +- .../operators/gcp_function_operator.html | 522 +++ .../contrib/operators/gcp_sql_operator.html | 737 ++++ .../operators/gcs_download_operator.html | 25 +- .../contrib/operators/gcs_list_operator.html | 25 +- .../airflow/contrib/operators/gcs_operator.html | 25 +- .../airflow/contrib/operators/gcs_to_bq.html | 25 +- .../airflow/contrib/operators/gcs_to_gcs.html | 25 +- .../airflow/contrib/operators/gcs_to_s3.html | 25 +- .../contrib/operators/hipchat_operator.html | 25 +- .../contrib/operators/hive_to_dynamodb.html | 25 +- .../operators/jenkins_job_trigger_operator.html | 25 +- .../contrib/operators/jira_operator.html | 25 +- .../operators/kubernetes_pod_operator.html | 51 +- .../contrib/operators/mlengine_operator.html | 41 +- .../airflow/contrib/operators/mongo_to_s3.html | 21 +- .../airflow/contrib/operators/mysql_to_gcs.html | 25 +- .../operators/postgres_to_gcs_operator.html | 25 +- .../contrib/operators/pubsub_operator.html | 41 +- .../operators/qubole_check_operator.html | 29 +- .../contrib/operators/qubole_operator.html | 29 +- .../contrib/operators/s3_list_operator.html | 25 +- .../contrib/operators/s3_to_gcs_operator.html | 25 +- .../operators/sagemaker_base_operator.html | 311 ++ .../sagemaker_endpoint_config_operator.html | 278 ++ .../operators/sagemaker_endpoint_operator.html | 362 ++ .../operators/sagemaker_model_operator.html | 279 ++ .../operators/sagemaker_training_operator.html | 310 ++ .../operators/sagemaker_transform_operator.html | 336 ++ .../operators/sagemaker_tuning_operator.html | 310 ++ .../operators/segment_track_event_operator.html | 25 +- .../contrib/operators/sftp_operator.html | 57 +- .../operators/slack_webhook_operator.html | 25 +- .../contrib/operators/snowflake_operator.html | 25 +- .../contrib/operators/spark_jdbc_operator.html | 25 +- .../contrib/operators/spark_sql_operator.html | 25 +- .../operators/spark_submit_operator.html | 25 +- .../contrib/operators/sqoop_operator.html | 25 +- .../airflow/contrib/operators/ssh_operator.html | 164 +- .../contrib/operators/vertica_operator.html | 25 +- .../contrib/operators/vertica_to_hive.html | 25 +- .../contrib/operators/winrm_operator.html | 25 +- .../sensors/aws_redshift_cluster_sensor.html | 21 +- .../airflow/contrib/sensors/bash_sensor.html | 21 +- .../contrib/sensors/bigquery_sensor.html | 21 +- .../contrib/sensors/cassandra_sensor.html | 271 ++ .../airflow/contrib/sensors/datadog_sensor.html | 21 +- .../contrib/sensors/emr_base_sensor.html | 21 +- .../contrib/sensors/emr_job_flow_sensor.html | 21 +- .../contrib/sensors/emr_step_sensor.html | 25 +- .../airflow/contrib/sensors/file_sensor.html | 21 +- .../airflow/contrib/sensors/ftp_sensor.html | 21 +- .../airflow/contrib/sensors/gcs_sensor.html | 21 +- .../airflow/contrib/sensors/hdfs_sensor.html | 21 +- .../airflow/contrib/sensors/jira_sensor.html | 21 +- .../airflow/contrib/sensors/pubsub_sensor.html | 21 +- .../airflow/contrib/sensors/qubole_sensor.html | 21 +- .../contrib/sensors/redis_key_sensor.html | 21 +- .../contrib/sensors/sagemaker_base_sensor.html | 285 ++ .../sensors/sagemaker_endpoint_sensor.html | 272 ++ .../sensors/sagemaker_training_sensor.html | 313 ++ .../sensors/sagemaker_transform_sensor.html | 273 ++ .../sensors/sagemaker_tuning_sensor.html | 273 ++ .../airflow/contrib/sensors/sftp_sensor.html | 21 +- .../airflow/contrib/sensors/wasb_sensor.html | 21 +- _modules/airflow/executors/celery_executor.html | 27 +- _modules/airflow/executors/local_executor.html | 21 +- .../airflow/executors/sequential_executor.html | 21 +- _modules/airflow/hooks/S3_hook.html | 164 +- _modules/airflow/hooks/dbapi_hook.html | 21 +- _modules/airflow/hooks/docker_hook.html | 21 +- _modules/airflow/hooks/druid_hook.html | 35 +- _modules/airflow/hooks/hdfs_hook.html | 21 +- _modules/airflow/hooks/hive_hooks.html | 21 +- _modules/airflow/hooks/http_hook.html | 33 +- _modules/airflow/hooks/jdbc_hook.html | 21 +- _modules/airflow/hooks/mssql_hook.html | 21 +- _modules/airflow/hooks/mysql_hook.html | 33 +- _modules/airflow/hooks/oracle_hook.html | 21 +- _modules/airflow/hooks/pig_hook.html | 21 +- _modules/airflow/hooks/postgres_hook.html | 21 +- _modules/airflow/hooks/presto_hook.html | 21 +- _modules/airflow/hooks/samba_hook.html | 21 +- _modules/airflow/hooks/slack_hook.html | 21 +- _modules/airflow/hooks/sqlite_hook.html | 21 +- _modules/airflow/hooks/webhdfs_hook.html | 21 +- _modules/airflow/hooks/zendesk_hook.html | 21 +- _modules/airflow/macros.html | 21 +- _modules/airflow/macros/hive.html | 21 +- _modules/airflow/models.html | 242 +- _modules/airflow/operators/bash_operator.html | 25 +- _modules/airflow/operators/check_operator.html | 33 +- _modules/airflow/operators/dagrun_operator.html | 25 +- _modules/airflow/operators/docker_operator.html | 80 +- .../airflow/operators/druid_check_operator.html | 25 +- _modules/airflow/operators/dummy_operator.html | 25 +- _modules/airflow/operators/email_operator.html | 25 +- .../airflow/operators/generic_transfer.html | 25 +- _modules/airflow/operators/hive_operator.html | 33 +- .../airflow/operators/hive_stats_operator.html | 25 +- _modules/airflow/operators/hive_to_druid.html | 25 +- _modules/airflow/operators/hive_to_mysql.html | 25 +- .../operators/hive_to_samba_operator.html | 25 +- _modules/airflow/operators/http_operator.html | 36 +- _modules/airflow/operators/jdbc_operator.html | 25 +- .../airflow/operators/latest_only_operator.html | 25 +- _modules/airflow/operators/mssql_operator.html | 25 +- _modules/airflow/operators/mssql_to_hive.html | 25 +- _modules/airflow/operators/mysql_operator.html | 25 +- _modules/airflow/operators/mysql_to_hive.html | 25 +- _modules/airflow/operators/oracle_operator.html | 25 +- _modules/airflow/operators/pig_operator.html | 33 +- .../airflow/operators/postgres_operator.html | 25 +- .../operators/presto_check_operator.html | 21 +- _modules/airflow/operators/presto_to_mysql.html | 25 +- _modules/airflow/operators/python_operator.html | 33 +- .../operators/redshift_to_s3_operator.html | 25 +- .../operators/s3_file_transform_operator.html | 49 +- .../airflow/operators/s3_to_hive_operator.html | 27 +- .../operators/s3_to_redshift_operator.html | 25 +- _modules/airflow/operators/slack_operator.html | 21 +- _modules/airflow/operators/sqlite_operator.html | 25 +- _modules/airflow/operators/subdag_operator.html | 25 +- .../airflow/sensors/base_sensor_operator.html | 25 +- .../airflow/sensors/external_task_sensor.html | 21 +- _modules/airflow/sensors/hdfs_sensor.html | 21 +- .../airflow/sensors/hive_partition_sensor.html | 21 +- _modules/airflow/sensors/http_sensor.html | 21 +- .../sensors/metastore_partition_sensor.html | 21 +- .../sensors/named_hive_partition_sensor.html | 21 +- _modules/airflow/sensors/s3_key_sensor.html | 21 +- _modules/airflow/sensors/s3_prefix_sensor.html | 23 +- _modules/airflow/sensors/sql_sensor.html | 21 +- _modules/airflow/sensors/time_delta_sensor.html | 21 +- _modules/airflow/sensors/time_sensor.html | 21 +- _modules/airflow/sensors/web_hdfs_sensor.html | 21 +- _modules/index.html | 45 +- _sources/api.rst.txt | 17 + _sources/cli.rst.txt | 17 + _sources/code.rst.txt | 40 +- _sources/concepts.rst.txt | 63 +- _sources/faq.rst.txt | 17 + _sources/howto/executor/use-celery.rst.txt | 19 +- _sources/howto/executor/use-dask.rst.txt | 17 + _sources/howto/executor/use-mesos.rst.txt | 17 + _sources/howto/index.rst.txt | 17 + _sources/howto/initialize-database.rst.txt | 17 + _sources/howto/manage-connections.rst.txt | 101 +- _sources/howto/operator.rst.txt | 493 +++ _sources/howto/run-with-systemd.rst.txt | 17 + _sources/howto/run-with-upstart.rst.txt | 17 + _sources/howto/secure-connections.rst.txt | 28 +- _sources/howto/set-config.rst.txt | 37 +- _sources/howto/use-test-config.rst.txt | 17 + _sources/howto/write-logs.rst.txt | 86 +- _sources/index.rst.txt | 17 + _sources/installation.rst.txt | 84 +- _sources/integration.rst.txt | 161 +- _sources/kubernetes.rst.txt | 34 +- _sources/license.rst.txt | 17 + _sources/lineage.rst.txt | 17 + _sources/plugins.rst.txt | 70 +- _sources/profiling.rst.txt | 19 + _sources/project.rst.txt | 18 +- _sources/scheduler.rst.txt | 24 +- _sources/security.rst.txt | 37 +- _sources/start.rst.txt | 17 + _sources/timezone.rst.txt | 84 +- _sources/tutorial.rst.txt | 58 +- _sources/ui.rst.txt | 19 +- _static/basic.css | 11 + _static/css/theme.css | 6 +- _static/doctools.js | 6 +- _static/documentation_options.js | 291 +- _static/fonts/Inconsolata-Bold.ttf | Bin 0 -> 109948 bytes _static/fonts/Inconsolata-Regular.ttf | Bin 0 -> 96964 bytes _static/fonts/Inconsolata.ttf | Bin 0 -> 63184 bytes _static/fonts/Lato-Bold.ttf | Bin 0 -> 656544 bytes _static/fonts/Lato-Regular.ttf | Bin 0 -> 656568 bytes _static/fonts/RobotoSlab-Bold.ttf | Bin 0 -> 170616 bytes _static/fonts/RobotoSlab-Regular.ttf | Bin 0 -> 169064 bytes _static/js/theme.js | 6 +- _static/searchtools.js | 351 +- api.html | 21 +- cli.html | 1473 ++++---- code.html | 3218 +++++++++++++++--- concepts.html | 59 +- faq.html | 21 +- genindex.html | 545 ++- howto/executor/use-celery.html | 23 +- howto/executor/use-dask.html | 21 +- howto/executor/use-mesos.html | 21 +- howto/index.html | 21 +- howto/initialize-database.html | 21 +- howto/manage-connections.html | 100 +- howto/operator.html | 643 +++- howto/run-with-systemd.html | 21 +- howto/run-with-upstart.html | 21 +- howto/secure-connections.html | 31 +- howto/set-config.html | 40 +- howto/use-test-config.html | 21 +- howto/write-logs.html | 92 +- index.html | 137 +- installation.html | 124 +- integration.html | 1798 +++++++++- kubernetes.html | 58 +- license.html | 21 +- lineage.html | 21 +- objects.inv | Bin 7505 -> 8915 bytes plugins.html | 68 +- profiling.html | 21 +- project.html | 22 +- py-modindex.html | 21 +- scheduler.html | 27 +- search.html | 23 +- searchindex.js | 2 +- security.html | 41 +- start.html | 21 +- timezone.html | 25 +- tutorial.html | 62 +- ui.html | 23 +- 292 files changed, 19787 insertions(+), 6652 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_images/airflow.gif ---------------------------------------------------------------------- diff --git a/_images/airflow.gif b/_images/airflow.gif index 1889b86..7646e37 100644 Binary files a/_images/airflow.gif and b/_images/airflow.gif differ http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/executors/mesos_executor.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/executors/mesos_executor.html b/_modules/airflow/contrib/executors/mesos_executor.html deleted file mode 100644 index a495421..0000000 --- a/_modules/airflow/contrib/executors/mesos_executor.html +++ /dev/null @@ -1,548 +0,0 @@ - - -<!DOCTYPE html> -<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> -<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> -<head> - <meta charset="utf-8"> - - <meta name="viewport" content="width=device-width, initial-scale=1.0"> - - <title>airflow.contrib.executors.mesos_executor — Airflow Documentation</title> - - - - - - - - - - - - - - - - <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" /> - <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" /> - <link rel="index" title="Index" href="../../../../genindex.html" /> - <link rel="search" title="Search" href="../../../../search.html" /> - - - <script src="../../../../_static/js/modernizr.min.js"></script> - -</head> - -<body class="wy-body-for-nav"> - - - <div class="wy-grid-for-nav"> - - - <nav data-toggle="wy-nav-shift" class="wy-nav-side"> - <div class="wy-side-scroll"> - <div class="wy-side-nav-search"> - - - - <a href="../../../../index.html" class="icon icon-home"> Airflow - - - - </a> - - - - - - - -<div role="search"> - <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get"> - <input type="text" name="q" placeholder="Search docs" /> - <input type="hidden" name="check_keywords" value="yes" /> - <input type="hidden" name="area" value="default" /> - </form> -</div> - - - </div> - - <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> - - - - - - - <ul> -<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li> -</ul> - - - - </div> - </div> - </nav> - - <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> - - - <nav class="wy-nav-top" aria-label="top navigation"> - - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="../../../../index.html">Airflow</a> - - </nav> - - - <div class="wy-nav-content"> - - <div class="rst-content"> - - - - - - - - - - - - - - - - - -<div role="navigation" aria-label="breadcrumbs navigation"> - - <ul class="wy-breadcrumbs"> - - <li><a href="../../../../index.html">Docs</a> »</li> - - <li><a href="../../../index.html">Module code</a> »</li> - - <li>airflow.contrib.executors.mesos_executor</li> - - - <li class="wy-breadcrumbs-aside"> - - </li> - - </ul> - - - <hr/> -</div> - <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article";> - <div itemprop="articleBody"> - - <h1>Source code for airflow.contrib.executors.mesos_executor</h1><div class="highlight"><pre> -<span></span><span class="c1"># -*- coding: utf-8 -*-</span> -<span class="c1">#</span> -<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span> -<span class="c1"># or more contributor license agreements. See the NOTICE file</span> -<span class="c1"># distributed with this work for additional information</span> -<span class="c1"># regarding copyright ownership. The ASF licenses this file</span> -<span class="c1"># to you under the Apache License, Version 2.0 (the</span> -<span class="c1"># "License"); you may not use this file except in compliance</span> -<span class="c1"># with the License. You may obtain a copy of the License at</span> -<span class="c1">#</span> -<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> -<span class="c1">#</span> -<span class="c1"># Unless required by applicable law or agreed to in writing,</span> -<span class="c1"># software distributed under the License is distributed on an</span> -<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span> -<span class="c1"># KIND, either express or implied. See the License for the</span> -<span class="c1"># specific language governing permissions and limitations</span> -<span class="c1"># under the License.</span> - -<span class="kn">from</span> <span class="nn">future</span> <span class="k">import</span> <span class="n">standard_library</span> - -<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span> -<span class="kn">from</span> <span class="nn">airflow.www.utils</span> <span class="k">import</span> <span class="n">LoginMixin</span> - - -<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span> -<span class="kn">from</span> <span class="nn">queue</span> <span class="k">import</span> <span class="n">Queue</span> - -<span class="kn">import</span> <span class="nn">mesos.interface</span> -<span class="kn">from</span> <span class="nn">mesos.interface</span> <span class="k">import</span> <span class="n">mesos_pb2</span> -<span class="kn">import</span> <span class="nn">mesos.native</span> - -<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span> -<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="k">import</span> <span class="n">BaseExecutor</span> -<span class="kn">from</span> <span class="nn">airflow.settings</span> <span class="k">import</span> <span class="n">Session</span> -<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span> -<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span> - -<span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span> -<span class="n">DEFAULT_FRAMEWORK_NAME</span> <span class="o">=</span> <span class="s1">'Airflow'</span> -<span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">=</span> <span class="s1">'mesos_framework_'</span> - - -<span class="k">def</span> <span class="nf">get_framework_name</span><span class="p">():</span> - <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">):</span> - <span class="k">return</span> <span class="n">DEFAULT_FRAMEWORK_NAME</span> - <span class="k">return</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">)</span> - - -<span class="c1"># AirflowMesosScheduler, implements Mesos Scheduler interface</span> -<span class="c1"># To schedule airflow jobs on mesos</span> -<span class="k">class</span> <span class="nc">AirflowMesosScheduler</span><span class="p">(</span><span class="n">mesos</span><span class="o">.</span><span class="n">interface</span><span class="o">.</span><span class="n">Scheduler</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span> - <span class="sd">"""</span> -<span class="sd"> Airflow Mesos scheduler implements mesos scheduler interface</span> -<span class="sd"> to schedule airflow tasks on mesos.</span> -<span class="sd"> Basically, it schedules a command like</span> -<span class="sd"> 'airflow run <dag_id> <task_instance_id> <start_date> --local -p=<pickle>'</span> -<span class="sd"> to run on a mesos slave.</span> -<span class="sd"> """</span> - <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> - <span class="n">task_queue</span><span class="p">,</span> - <span class="n">result_queue</span><span class="p">,</span> - <span class="n">task_cpu</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> - <span class="n">task_mem</span><span class="o">=</span><span class="mi">256</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">task_queue</span> - <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">result_queue</span> - <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span> <span class="o">=</span> <span class="n">task_cpu</span> - <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span> <span class="o">=</span> <span class="n">task_mem</span> - <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span> <span class="o">=</span> <span class="mi">0</span> - <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span> <span class="o">=</span> <span class="p">{}</span> - <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DOCKER_IMAGE_SLAVE'</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span> - <span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DOCKER_IMAGE_SLAVE'</span> - <span class="p">)</span> - - <span class="k">def</span> <span class="nf">registered</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">frameworkId</span><span class="p">,</span> <span class="n">masterInfo</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler registered to Mesos with framework ID </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> - <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span> - - <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">)</span> <span class="ow">and</span> \ - <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span> - <span class="c1"># Import here to work around a circular import error</span> - <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span> - - <span class="c1"># Update the Framework ID in the database.</span> - <span class="n">session</span> <span class="o">=</span> <span class="n">Session</span><span class="p">()</span> - <span class="n">conn_id</span> <span class="o">=</span> <span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">+</span> <span class="n">get_framework_name</span><span class="p">()</span> - <span class="n">connection</span> <span class="o">=</span> <span class="n">Session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Connection</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span> - <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> - <span class="n">connection</span> <span class="o">=</span> <span class="n">Connection</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">,</span> <span class="n">conn_type</span><span class="o">=</span><span class="s1">'mesos_framework-id'</span><span class="p">,</span> - <span class="n">extra</span><span class="o">=</span><span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span> - <span class="k">else</span><span class="p">:</span> - <span class="n">connection</span><span class="o">.</span><span class="n">extra</span> <span class="o">=</span> <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span> - - <span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">connection</span><span class="p">)</span> - <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span> - <span class="n">Session</span><span class="o">.</span><span class="n">remove</span><span class="p">()</span> - - <span class="k">def</span> <span class="nf">reregistered</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">masterInfo</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler re-registered to mesos"</span><span class="p">)</span> - - <span class="k">def</span> <span class="nf">disconnected</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler disconnected from mesos"</span><span class="p">)</span> - - <span class="k">def</span> <span class="nf">offerRescinded</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">offerId</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler offer </span><span class="si">%s</span><span class="s2"> rescinded"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">offerId</span><span class="p">))</span> - - <span class="k">def</span> <span class="nf">frameworkMessage</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">executorId</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">,</span> <span class="n">message</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler received framework message </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span> - - <span class="k">def</span> <span class="nf">executorLost</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">executorId</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">,</span> <span class="n">status</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"AirflowScheduler executor </span><span class="si">%s</span><span class="s2"> lost"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">executorId</span><span class="p">))</span> - - <span class="k">def</span> <span class="nf">slaveLost</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"AirflowScheduler slave </span><span class="si">%s</span><span class="s2"> lost"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">slaveId</span><span class="p">))</span> - - <span class="k">def</span> <span class="nf">error</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">message</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"AirflowScheduler driver aborted </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span> - <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"AirflowScheduler driver aborted </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">message</span><span class="p">)</span> - - <span class="k">def</span> <span class="nf">resourceOffers</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">offers</span><span class="p">):</span> - <span class="k">for</span> <span class="n">offer</span> <span class="ow">in</span> <span class="n">offers</span><span class="p">:</span> - <span class="n">tasks</span> <span class="o">=</span> <span class="p">[]</span> - <span class="n">offerCpus</span> <span class="o">=</span> <span class="mi">0</span> - <span class="n">offerMem</span> <span class="o">=</span> <span class="mi">0</span> - <span class="k">for</span> <span class="n">resource</span> <span class="ow">in</span> <span class="n">offer</span><span class="o">.</span><span class="n">resources</span><span class="p">:</span> - <span class="k">if</span> <span class="n">resource</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="s2">"cpus"</span><span class="p">:</span> - <span class="n">offerCpus</span> <span class="o">+=</span> <span class="n">resource</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span> - <span class="k">elif</span> <span class="n">resource</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="s2">"mem"</span><span class="p">:</span> - <span class="n">offerMem</span> <span class="o">+=</span> <span class="n">resource</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span> - - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Received offer </span><span class="si">%s</span><span class="s2"> with cpus: </span><span class="si">%s</span><span class="s2"> and mem: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> - <span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span><span class="p">,</span> <span class="n">offerCpus</span><span class="p">,</span> <span class="n">offerMem</span><span class="p">)</span> - - <span class="n">remainingCpus</span> <span class="o">=</span> <span class="n">offerCpus</span> - <span class="n">remainingMem</span> <span class="o">=</span> <span class="n">offerMem</span> - - <span class="k">while</span> <span class="p">(</span><span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">())</span> <span class="ow">and</span> \ - <span class="n">remainingCpus</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span> <span class="ow">and</span> \ - <span class="n">remainingMem</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span><span class="p">:</span> - <span class="n">key</span><span class="p">,</span> <span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> - <span class="n">tid</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span> - <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span> <span class="o">+=</span> <span class="mi">1</span> - <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span><span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">tid</span><span class="p">)]</span> <span class="o">=</span> <span class="n">key</span> - - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Launching task </span><span class="si">%d</span><span class="s2"> using offer </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">tid</span><span class="p">,</span> <span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span><span class="p">)</span> - - <span class="n">task</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TaskInfo</span><span class="p">()</span> - <span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">tid</span><span class="p">)</span> - <span class="n">task</span><span class="o">.</span><span class="n">slave_id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">offer</span><span class="o">.</span><span class="n">slave_id</span><span class="o">.</span><span class="n">value</span> - <span class="n">task</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="s2">"AirflowTask </span><span class="si">%d</span><span class="s2">"</span> <span class="o">%</span> <span class="n">tid</span> - - <span class="n">cpus</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">add</span><span class="p">()</span> - <span class="n">cpus</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="s2">"cpus"</span> - <span class="n">cpus</span><span class="o">.</span><span class="n">type</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Value</span><span class="o">.</span><span class="n">SCALAR</span> - <span class="n">cpus</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span> - - <span class="n">mem</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">add</span><span class="p">()</span> - <span class="n">mem</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="s2">"mem"</span> - <span class="n">mem</span><span class="o">.</span><span class="n">type</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Value</span><span class="o">.</span><span class="n">SCALAR</span> - <span class="n">mem</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span> - - <span class="n">command</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">CommandInfo</span><span class="p">()</span> - <span class="n">command</span><span class="o">.</span><span class="n">shell</span> <span class="o">=</span> <span class="kc">True</span> - <span class="n">command</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">cmd</span> - <span class="n">task</span><span class="o">.</span><span class="n">command</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">command</span><span class="p">)</span> - - <span class="c1"># If docker image for airflow is specified in config then pull that</span> - <span class="c1"># image before running the above airflow command</span> - <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span><span class="p">:</span> - <span class="n">network</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DockerInfo</span><span class="o">.</span><span class="n">Network</span><span class="o">.</span><span class="n">Value</span><span class="p">(</span><span class="s1">'BRIDGE'</span><span class="p">)</span> - <span class="n">docker</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DockerInfo</span><span class="p">(</span> - <span class="n">image</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span><span class="p">,</span> - <span class="n">force_pull_image</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> - <span class="n">network</span><span class="o">=</span><span class="n">network</span> - <span class="p">)</span> - <span class="n">container</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="p">(</span> - <span class="nb">type</span><span class="o">=</span><span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DOCKER</span><span class="p">,</span> - <span class="n">docker</span><span class="o">=</span><span class="n">docker</span> - <span class="p">)</span> - <span class="n">task</span><span class="o">.</span><span class="n">container</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">container</span><span class="p">)</span> - - <span class="n">tasks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">task</span><span class="p">)</span> - - <span class="n">remainingCpus</span> <span class="o">-=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span> - <span class="n">remainingMem</span> <span class="o">-=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span> - - <span class="n">driver</span><span class="o">.</span><span class="n">launchTasks</span><span class="p">(</span><span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="p">,</span> <span class="n">tasks</span><span class="p">)</span> - - <span class="k">def</span> <span class="nf">statusUpdate</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">update</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span> - <span class="s2">"Task </span><span class="si">%s</span><span class="s2"> is in state </span><span class="si">%s</span><span class="s2">, data </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> - <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">,</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TaskState</span><span class="o">.</span><span class="n">Name</span><span class="p">(</span><span class="n">update</span><span class="o">.</span><span class="n">state</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">update</span><span class="o">.</span><span class="n">data</span><span class="p">)</span> - <span class="p">)</span> - - <span class="k">try</span><span class="p">:</span> - <span class="n">key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span><span class="p">[</span><span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">]</span> - <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span> - <span class="c1"># The map may not contain an item if the framework re-registered</span> - <span class="c1"># after a failover.</span> - <span class="c1"># Discard these tasks.</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Unrecognised task key </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">)</span> - <span class="k">return</span> - - <span class="k">if</span> <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_FINISHED</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">))</span> - <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span> - - <span class="k">if</span> <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_LOST</span> <span class="ow">or</span> \ - <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_KILLED</span> <span class="ow">or</span> \ - <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_FAILED</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">))</span> - <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span> - - -<div class="viewcode-block" id="MesosExecutor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor">[docs]</a><span class="k">class</span> <span class="nc">MesosExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">,</span> <span class="n">LoginMixin</span><span class="p">):</span> - <span class="sd">"""</span> -<span class="sd"> MesosExecutor allows distributing the execution of task</span> -<span class="sd"> instances to multiple mesos workers.</span> - -<span class="sd"> Apache Mesos is a distributed systems kernel which abstracts</span> -<span class="sd"> CPU, memory, storage, and other compute resources away from</span> -<span class="sd"> machines (physical or virtual), enabling fault-tolerant and</span> -<span class="sd"> elastic distributed systems to easily be built and run effectively.</span> -<span class="sd"> See http://mesos.apache.org/</span> -<span class="sd"> """</span> -<div class="viewcode-block" id="MesosExecutor.start"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.start">[docs]</a> <span class="k">def</span> <span class="nf">start</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span> - <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span> - <span class="n">framework</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">FrameworkInfo</span><span class="p">()</span> - <span class="n">framework</span><span class="o">.</span><span class="n">user</span> <span class="o">=</span> <span class="s1">''</span> - - <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting mesos master URL for mesos executor"</span><span class="p">)</span> - <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"mesos.master not provided for mesos executor"</span><span class="p">)</span> - - <span class="n">master</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">)</span> - - <span class="n">framework</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">get_framework_name</span><span class="p">()</span> - - <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">):</span> - <span class="n">task_cpu</span> <span class="o">=</span> <span class="mi">1</span> - <span class="k">else</span><span class="p">:</span> - <span class="n">task_cpu</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">)</span> - - <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">):</span> - <span class="n">task_memory</span> <span class="o">=</span> <span class="mi">256</span> - <span class="k">else</span><span class="p">:</span> - <span class="n">task_memory</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">)</span> - - <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">):</span> - <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">True</span> - - <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span> - <span class="c1"># Import here to work around a circular import error</span> - <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span> - - <span class="c1"># Query the database to get the ID of the Mesos Framework, if available.</span> - <span class="n">conn_id</span> <span class="o">=</span> <span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">+</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span> - <span class="n">session</span> <span class="o">=</span> <span class="n">Session</span><span class="p">()</span> - <span class="n">connection</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Connection</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span> - <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> - <span class="c1"># Set the Framework ID to let the scheduler reconnect</span> - <span class="c1"># with running tasks.</span> - <span class="n">framework</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">extra</span> - - <span class="n">framework</span><span class="o">.</span><span class="n">failover_timeout</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span> - <span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span> - <span class="p">)</span> - <span class="k">else</span><span class="p">:</span> - <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">False</span> - - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span> - <span class="s1">'MesosFramework master : </span><span class="si">%s</span><span class="s1">, name : </span><span class="si">%s</span><span class="s1">, cpu : </span><span class="si">%s</span><span class="s1">, mem : </span><span class="si">%s</span><span class="s1">, checkpoint : </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> - <span class="n">master</span><span class="p">,</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> - <span class="nb">str</span><span class="p">(</span><span class="n">task_cpu</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_memory</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span><span class="p">)</span> - <span class="p">)</span> - - <span class="n">implicit_acknowledgements</span> <span class="o">=</span> <span class="mi">1</span> - - <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'AUTHENTICATE'</span><span class="p">):</span> - <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting authentication principal in the environment"</span><span class="p">)</span> - <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span> - <span class="s2">"mesos.default_principal not provided in authenticated mode"</span><span class="p">)</span> - <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting authentication secret in the environment"</span><span class="p">)</span> - <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span> - <span class="s2">"mesos.default_secret not provided in authenticated mode"</span><span class="p">)</span> - - <span class="n">credential</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Credential</span><span class="p">()</span> - <span class="n">credential</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">)</span> - <span class="n">credential</span><span class="o">.</span><span class="n">secret</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">)</span> - - <span class="n">framework</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">credential</span><span class="o">.</span><span class="n">principal</span> - - <span class="n">driver</span> <span class="o">=</span> <span class="n">mesos</span><span class="o">.</span><span class="n">native</span><span class="o">.</span><span class="n">MesosSchedulerDriver</span><span class="p">(</span> - <span class="n">AirflowMesosScheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="p">,</span> - <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">,</span> - <span class="n">task_cpu</span><span class="p">,</span> - <span class="n">task_memory</span><span class="p">),</span> - <span class="n">framework</span><span class="p">,</span> - <span class="n">master</span><span class="p">,</span> - <span class="n">implicit_acknowledgements</span><span class="p">,</span> - <span class="n">credential</span><span class="p">)</span> - <span class="k">else</span><span class="p">:</span> - <span class="n">framework</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="s1">'Airflow'</span> - <span class="n">driver</span> <span class="o">=</span> <span class="n">mesos</span><span class="o">.</span><span class="n">native</span><span class="o">.</span><span class="n">MesosSchedulerDriver</span><span class="p">(</span> - <span class="n">AirflowMesosScheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="p">,</span> - <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">,</span> - <span class="n">task_cpu</span><span class="p">,</span> - <span class="n">task_memory</span><span class="p">),</span> - <span class="n">framework</span><span class="p">,</span> - <span class="n">master</span><span class="p">,</span> - <span class="n">implicit_acknowledgements</span><span class="p">)</span> - - <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span> <span class="o">=</span> <span class="n">driver</span> - <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">start</span><span class="p">()</span></div> - -<div class="viewcode-block" id="MesosExecutor.execute_async"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.execute_async">[docs]</a> <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">executor_config</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span></div> - -<div class="viewcode-block" id="MesosExecutor.sync"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.sync">[docs]</a> <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> - <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span> - <span class="n">results</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> - <span class="bp">self</span><span class="o">.</span><span class="n">change_state</span><span class="p">(</span><span class="o">*</span><span class="n">results</span><span class="p">)</span></div> - -<div class="viewcode-block" id="MesosExecutor.end"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.end">[docs]</a> <span class="k">def</span> <span class="nf">end</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">join</span><span class="p">()</span> - <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">stop</span><span class="p">()</span></div></div> -</pre></div> - - </div> - - </div> - <footer> - - - <hr/> - - <div role="contentinfo"> - <p> - - </p> - </div> - Built with <a href="http://sphinx-doc.org/";>Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme";>theme</a> provided by <a href="https://readthedocs.org";>Read the Docs</a>. - -</footer> - - </div> - </div> - - </section> - - </div> - - - - - - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> - - - - <script type="text/javascript" src="../../../../_static/js/theme.js"></script> - - <script type="text/javascript"> - jQuery(function () { - SphinxRtdTheme.Navigation.enable(true); - }); - </script> - -</body> -</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html b/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html index b8fea44..4feb169 100644 --- a/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html +++ b/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html @@ -258,20 +258,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> +
