1.10.0
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/commit/11437c14 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/tree/11437c14 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/diff/11437c14 Branch: refs/heads/asf-site Commit: 11437c14a3607f6b85b6610dc776f55d78e7d767 Parents: 28a3eb6 Author: Kaxil Naik <kaxiln...@gmail.com> Authored: Mon Aug 27 17:22:22 2018 +0100 Committer: Kaxil Naik <kaxiln...@gmail.com> Committed: Mon Aug 27 17:22:22 2018 +0100 ---------------------------------------------------------------------- _images/connection_create.png | Bin 0 -> 41547 bytes _images/connection_edit.png | Bin 0 -> 53636 bytes _images/connections.png | Bin 93057 -> 48442 bytes _modules/S3_hook.html | 489 - .../contrib/executors/mesos_executor.html | 89 +- .../contrib/hooks/aws_dynamodb_hook.html | 300 + _modules/airflow/contrib/hooks/aws_hook.html | 410 + .../airflow/contrib/hooks/aws_lambda_hook.html | 302 + .../airflow/contrib/hooks/bigquery_hook.html | 889 +- .../airflow/contrib/hooks/databricks_hook.html | 462 + .../airflow/contrib/hooks/datadog_hook.html | 375 + .../airflow/contrib/hooks/datastore_hook.html | 52 +- .../contrib/hooks/discord_webhook_hook.html | 375 + _modules/airflow/contrib/hooks/emr_hook.html | 35 +- _modules/airflow/contrib/hooks/fs_hook.html | 281 + _modules/airflow/contrib/hooks/ftp_hook.html | 494 + .../contrib/hooks/gcp_api_base_hook.html | 379 + .../contrib/hooks/gcp_dataflow_hook.html | 196 +- .../contrib/hooks/gcp_dataproc_hook.html | 463 + .../contrib/hooks/gcp_mlengine_hook.html | 20 +- .../airflow/contrib/hooks/gcp_pubsub_hook.html | 519 + _modules/airflow/contrib/hooks/gcs_hook.html | 315 +- .../airflow/contrib/hooks/jenkins_hook.html | 283 + _modules/airflow/contrib/hooks/jira_hook.html | 319 + _modules/airflow/contrib/hooks/pinot_hook.html | 340 + _modules/airflow/contrib/hooks/qubole_hook.html | 449 + _modules/airflow/contrib/hooks/redis_hook.html | 328 + .../airflow/contrib/hooks/redshift_hook.html | 348 + _modules/airflow/contrib/hooks/sftp_hook.html | 404 + .../contrib/hooks/slack_webhook_hook.html | 364 + .../airflow/contrib/hooks/spark_jdbc_hook.html | 481 + .../airflow/contrib/hooks/spark_sql_hook.html | 396 + .../contrib/hooks/spark_submit_hook.html | 799 ++ _modules/airflow/contrib/hooks/sqoop_hook.html | 580 + _modules/airflow/contrib/hooks/ssh_hook.html | 470 + .../airflow/contrib/hooks/vertica_hook.html | 288 + _modules/airflow/contrib/hooks/wasb_hook.html | 94 +- _modules/airflow/contrib/kubernetes/secret.html | 276 + .../contrib/operators/awsbatch_operator.html | 403 + .../operators/bigquery_check_operator.html | 37 +- .../contrib/operators/bigquery_get_data.html | 351 + .../contrib/operators/bigquery_operator.html | 399 +- .../bigquery_table_delete_operator.html | 301 + .../contrib/operators/bigquery_to_bigquery.html | 39 +- .../contrib/operators/bigquery_to_gcs.html | 39 +- .../contrib/operators/databricks_operator.html | 45 +- .../contrib/operators/dataflow_operator.html | 241 +- .../contrib/operators/dataproc_operator.html | 866 +- .../operators/datastore_export_operator.html | 344 + .../operators/datastore_import_operator.html | 332 + .../operators/discord_webhook_operator.html | 333 + .../airflow/contrib/operators/ecs_operator.html | 31 +- .../operators/emr_add_steps_operator.html | 33 +- .../operators/emr_create_job_flow_operator.html | 33 +- .../emr_terminate_job_flow_operator.html | 31 +- .../airflow/contrib/operators/file_to_gcs.html | 310 + .../airflow/contrib/operators/file_to_wasb.html | 35 +- .../operators/gcs_download_operator.html | 56 +- .../contrib/operators/gcs_list_operator.html | 326 + .../airflow/contrib/operators/gcs_operator.html | 357 + .../airflow/contrib/operators/gcs_to_bq.html | 288 +- .../airflow/contrib/operators/gcs_to_gcs.html | 365 + .../airflow/contrib/operators/gcs_to_s3.html | 347 + .../contrib/operators/hipchat_operator.html | 35 +- .../operators/jenkins_job_trigger_operator.html | 484 + .../contrib/operators/jira_operator.html | 329 + .../operators/kubernetes_pod_operator.html | 362 + .../contrib/operators/mlengine_operator.html | 260 +- .../airflow/contrib/operators/mysql_to_gcs.html | 524 + .../operators/postgres_to_gcs_operator.html | 481 + .../contrib/operators/pubsub_operator.html | 669 ++ .../contrib/operators/qubole_operator.html | 411 + .../contrib/operators/s3_list_operator.html | 317 + .../contrib/operators/s3_to_gcs_operator.html | 425 + .../contrib/operators/sftp_operator.html | 336 + .../operators/slack_webhook_operator.html | 327 + .../contrib/operators/spark_jdbc_operator.html | 449 + .../contrib/operators/spark_sql_operator.html | 344 + .../operators/spark_submit_operator.html | 405 + .../contrib/operators/sqoop_operator.html | 467 + .../airflow/contrib/operators/ssh_operator.html | 394 + .../contrib/operators/vertica_operator.html | 284 + .../sensors/aws_redshift_cluster_sensor.html | 287 + .../airflow/contrib/sensors/bash_sensor.html | 331 + .../contrib/sensors/bigquery_sensor.html | 306 + .../airflow/contrib/sensors/datadog_sensor.html | 318 + .../contrib/sensors/emr_base_sensor.html | 291 + .../contrib/sensors/emr_job_flow_sensor.html | 288 + .../contrib/sensors/emr_step_sensor.html | 292 + .../airflow/contrib/sensors/file_sensor.html | 306 + .../airflow/contrib/sensors/ftp_sensor.html | 302 + .../airflow/contrib/sensors/gcs_sensor.html | 408 + .../airflow/contrib/sensors/hdfs_sensor.html | 313 + .../airflow/contrib/sensors/jira_sensor.html | 382 + .../airflow/contrib/sensors/pubsub_sensor.html | 340 + .../airflow/contrib/sensors/qubole_sensor.html | 323 + .../contrib/sensors/redis_key_sensor.html | 282 + .../airflow/contrib/sensors/sftp_sensor.html | 287 + .../airflow/contrib/sensors/wasb_sensor.html | 33 +- _modules/airflow/executors/celery_executor.html | 201 +- _modules/airflow/executors/local_executor.html | 240 +- .../airflow/executors/sequential_executor.html | 37 +- _modules/airflow/hooks/S3_hook.html | 200 +- _modules/airflow/hooks/dbapi_hook.html | 515 + _modules/airflow/hooks/hdfs_hook.html | 335 + _modules/airflow/hooks/http_hook.html | 358 + _modules/airflow/hooks/mssql_hook.html | 287 + _modules/airflow/hooks/mysql_hook.html | 341 + _modules/airflow/hooks/oracle_hook.html | 382 + _modules/airflow/hooks/pig_hook.html | 329 + _modules/airflow/hooks/postgres_hook.html | 323 + _modules/airflow/hooks/presto_hook.html | 366 + _modules/airflow/hooks/slack_hook.html | 296 + _modules/airflow/hooks/sqlite_hook.html | 276 + _modules/airflow/hooks/webhdfs_hook.html | 344 + _modules/airflow/macros.html | 31 +- _modules/airflow/macros/hive.html | 52 +- _modules/airflow/models.html | 1004 +- _modules/airflow/operators/bash_operator.html | 376 + _modules/airflow/operators/check_operator.html | 497 + _modules/airflow/operators/dagrun_operator.html | 317 + _modules/airflow/operators/docker_operator.html | 469 - _modules/airflow/operators/dummy_operator.html | 273 + _modules/airflow/operators/email_operator.html | 313 + .../airflow/operators/generic_transfer.html | 315 + _modules/airflow/operators/http_operator.html | 327 + .../airflow/operators/latest_only_operator.html | 300 + _modules/airflow/operators/mssql_operator.html | 291 + _modules/airflow/operators/mysql_operator.html | 295 + _modules/airflow/operators/oracle_operator.html | 290 + _modules/airflow/operators/pig_operator.html | 307 + .../airflow/operators/postgres_operator.html | 297 + .../operators/presto_check_operator.html | 361 + _modules/airflow/operators/presto_to_mysql.html | 314 + _modules/airflow/operators/python_operator.html | 613 + .../operators/redshift_to_s3_operator.html | 118 +- .../operators/s3_file_transform_operator.html | 96 +- .../airflow/operators/s3_to_hive_operator.html | 482 - .../operators/s3_to_redshift_operator.html | 330 + _modules/airflow/operators/sensors.html | 929 -- _modules/airflow/operators/slack_operator.html | 366 + _modules/airflow/operators/sqlite_operator.html | 286 + _modules/airflow/operators/subdag_operator.html | 338 + .../airflow/sensors/base_sensor_operator.html | 310 + .../airflow/sensors/external_task_sensor.html | 336 + _modules/airflow/sensors/hdfs_sensor.html | 352 + .../airflow/sensors/hive_partition_sensor.html | 310 + _modules/airflow/sensors/http_sensor.html | 327 + .../sensors/metastore_partition_sensor.html | 318 + .../sensors/named_hive_partition_sensor.html | 339 + _modules/airflow/sensors/s3_key_sensor.html | 316 + _modules/airflow/sensors/s3_prefix_sensor.html | 303 + _modules/airflow/sensors/sql_sensor.html | 293 + _modules/airflow/sensors/time_delta_sensor.html | 281 + _modules/airflow/sensors/time_sensor.html | 275 + _modules/airflow/sensors/web_hdfs_sensor.html | 279 + _modules/bash_operator.html | 350 - _modules/bigquery_hook.html | 1279 -- _modules/cloudant_hook.html | 275 - _modules/dagrun_operator.html | 310 - _modules/dbapi_hook.html | 499 - _modules/druid_hook.html | 325 - _modules/dummy_operator.html | 266 - _modules/email_operator.html | 297 - _modules/ftp_hook.html | 486 - _modules/gcs_hook.html | 498 - _modules/generic_transfer.html | 308 - _modules/hive_hooks.html | 996 -- _modules/hive_operator.html | 339 - _modules/hive_to_druid.html | 454 - _modules/hive_to_mysql.html | 339 - _modules/hive_to_samba_operator.html | 260 - _modules/http_hook.html | 351 - _modules/http_operator.html | 320 - _modules/index.html | 155 +- _modules/mssql_hook.html | 242 - _modules/mssql_operator.html | 247 - _modules/mssql_to_hive.html | 326 - _modules/mysql_hook.html | 334 - _modules/mysql_operator.html | 288 - _modules/mysql_to_hive.html | 368 - _modules/postgres_hook.html | 316 - _modules/postgres_operator.html | 288 - _modules/presto_check_operator.html | 354 - _modules/presto_hook.html | 346 - _modules/python_operator.html | 591 - _modules/qubole_operator.html | 404 - _modules/s3_to_hive_operator.html | 482 - _modules/sensors.html | 929 -- _modules/slack_operator.html | 352 - _modules/sqlite_hook.html | 269 - _modules/ssh_execute_operator.html | 387 - _modules/ssh_hook.html | 386 - _modules/vertica_hook.html | 247 - _modules/vertica_operator.html | 246 - _modules/vertica_to_hive.html | 330 - _modules/webhdfs_hook.html | 300 - _sources/api.rst.txt | 13 +- _sources/cli.txt | 11 - _sources/code.rst.txt | 336 +- _sources/code.txt | 252 - _sources/concepts.rst.txt | 49 +- _sources/concepts.txt | 759 -- _sources/configuration.rst.txt | 310 - _sources/configuration.txt | 230 - _sources/faq.rst.txt | 43 +- _sources/faq.txt | 142 - _sources/howto/executor/use-celery.rst.txt | 47 + _sources/howto/executor/use-dask.rst.txt | 33 + _sources/howto/executor/use-mesos.rst.txt | 65 + _sources/howto/index.rst.txt | 23 + _sources/howto/initialize-database.rst.txt | 31 + _sources/howto/manage-connections.rst.txt | 135 + _sources/howto/run-with-systemd.rst.txt | 15 + _sources/howto/run-with-upstart.rst.txt | 20 + _sources/howto/secure-connections.rst.txt | 32 + _sources/howto/set-config.rst.txt | 35 + _sources/howto/use-test-config.rst.txt | 17 + _sources/howto/write-logs.rst.txt | 159 + _sources/index.rst.txt | 4 +- _sources/index.txt | 87 - _sources/installation.rst.txt | 29 +- _sources/installation.txt | 90 - _sources/integration.rst.txt | 428 +- _sources/kubernetes.rst.txt | 101 + _sources/license.rst.txt | 35 +- _sources/license.txt | 211 - _sources/lineage.rst.txt | 85 + _sources/plugins.rst.txt | 18 +- _sources/plugins.txt | 139 - _sources/profiling.rst.txt | 2 +- _sources/profiling.txt | 39 - _sources/project.rst.txt | 7 + _sources/project.txt | 59 - _sources/scheduler.rst.txt | 4 +- _sources/scheduler.txt | 101 - _sources/security.rst.txt | 22 +- _sources/security.txt | 253 - _sources/start.rst.txt | 7 +- _sources/start.txt | 49 - _sources/timezone.rst.txt | 143 + _sources/tutorial.rst.txt | 4 +- _sources/tutorial.txt | 429 - _sources/ui.txt | 102 - _static/apache.jpg | Bin 43364 -> 0 bytes _static/basic.css | 2 +- _static/default.css | 3 - _static/doctools.js | 4 +- _static/fonts/Inconsolata.ttf | Bin 63184 -> 0 bytes _static/incubator.jpg | Bin 91227 -> 0 bytes _static/jquery-1.11.1.js | 10308 ----------------- _static/searchtools.js | 2 +- _static/websupport.js | 2 +- api.html | 18 +- cli.html | 256 +- code.html | 10181 ++++++++++++---- concepts.html | 50 +- configuration.html | 520 - faq.html | 50 +- genindex.html | 938 +- howto/executor/use-celery.html | 301 + howto/executor/use-dask.html | 287 + howto/executor/use-mesos.html | 326 + howto/index.html | 295 + howto/initialize-database.html | 292 + howto/manage-connections.html | 381 + howto/run-with-systemd.html | 274 + howto/run-with-upstart.html | 276 + howto/secure-connections.html | 290 + howto/set-config.html | 289 + howto/use-test-config.html | 275 + howto/write-logs.html | 430 + index.html | 163 +- installation.html | 32 +- integration.html | 2801 ++++- kubernetes.html | 369 + license.html | 381 +- lineage.html | 328 + objects.inv | Bin 3480 -> 6013 bytes plugins.html | 22 +- profiling.html | 6 +- project.html | 11 +- py-modindex.html | 34 +- scheduler.html | 8 +- search.html | 4 +- searchindex.js | 2 +- security.html | 41 +- start.html | 11 +- timezone.html | 374 + tutorial.html | 12 +- ui.html | 8 +- 291 files changed, 63681 insertions(+), 38174 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_images/connection_create.png ---------------------------------------------------------------------- diff --git a/_images/connection_create.png b/_images/connection_create.png new file mode 100644 index 0000000..8a574d4 Binary files /dev/null and b/_images/connection_create.png differ http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_images/connection_edit.png ---------------------------------------------------------------------- diff --git a/_images/connection_edit.png b/_images/connection_edit.png new file mode 100644 index 0000000..c6d14da Binary files /dev/null and b/_images/connection_edit.png differ http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_images/connections.png ---------------------------------------------------------------------- diff --git a/_images/connections.png b/_images/connections.png index d07a130..3a28473 100644 Binary files a/_images/connections.png and b/_images/connections.png differ http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/S3_hook.html ---------------------------------------------------------------------- diff --git a/_modules/S3_hook.html b/_modules/S3_hook.html deleted file mode 100644 index ae1091a..0000000 --- a/_modules/S3_hook.html +++ /dev/null @@ -1,489 +0,0 @@ - - -<!DOCTYPE html> -<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> -<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> -<head> - <meta charset="utf-8"> - - <meta name="viewport" content="width=device-width, initial-scale=1.0"> - - <title>S3_hook — Airflow Documentation</title> - - - - - - - - - - - - - - - - - - <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" /> - - - - - - <link rel="index" title="Index" - href="../genindex.html"/> - <link rel="search" title="Search" href="../search.html"/> - <link rel="top" title="Airflow Documentation" href="../index.html"/> - <link rel="up" title="Module code" href="index.html"/> - - - <script src="../_static/js/modernizr.min.js"></script> - -</head> - -<body class="wy-body-for-nav" role="document"> - - - <div class="wy-grid-for-nav"> - - - <nav data-toggle="wy-nav-shift" class="wy-nav-side"> - <div class="wy-side-scroll"> - <div class="wy-side-nav-search"> - - - - <a href="../index.html" class="icon icon-home"> Airflow - - - - </a> - - - - - - - -<div role="search"> - <form id="rtd-search-form" class="wy-form" action="../search.html" method="get"> - <input type="text" name="q" placeholder="Search docs" /> - <input type="hidden" name="check_keywords" value="yes" /> - <input type="hidden" name="area" value="default" /> - </form> -</div> - - - </div> - - <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> - - - - - - - <ul> -<li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li> -<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li> -<li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li> -<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li> -<li class="toctree-l1"><a class="reference internal" href="../tutorial.html">Tutorial</a></li> -<li class="toctree-l1"><a class="reference internal" href="../configuration.html">Configuration</a></li> -<li class="toctree-l1"><a class="reference internal" href="../ui.html">UI / Screenshots</a></li> -<li class="toctree-l1"><a class="reference internal" href="../concepts.html">Concepts</a></li> -<li class="toctree-l1"><a class="reference internal" href="../profiling.html">Data Profiling</a></li> -<li class="toctree-l1"><a class="reference internal" href="../cli.html">Command Line Interface</a></li> -<li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling & Triggers</a></li> -<li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li> -<li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li> -<li class="toctree-l1"><a class="reference internal" href="../api.html">Experimental Rest API</a></li> -<li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li> -<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> -<li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li> -</ul> - - - - </div> - </div> - </nav> - - <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> - - - <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="../index.html">Airflow</a> - - </nav> - - - - <div class="wy-nav-content"> - <div class="rst-content"> - - - - - - - - - - - - - - - - -<div role="navigation" aria-label="breadcrumbs navigation"> - - <ul class="wy-breadcrumbs"> - - <li><a href="../index.html">Docs</a> »</li> - - <li><a href="index.html">Module code</a> »</li> - - <li>S3_hook</li> - - - <li class="wy-breadcrumbs-aside"> - - - - </li> - - </ul> - - - <hr/> -</div> - <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> - <div itemprop="articleBody"> - - <h1>Source code for S3_hook</h1><div class="highlight"><pre> -<span></span><span class="c1"># -*- coding: utf-8 -*-</span> -<span class="c1">#</span> -<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> -<span class="c1"># you may not use this file except in compliance with the License.</span> -<span class="c1"># You may obtain a copy of the License at</span> -<span class="c1">#</span> -<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> -<span class="c1">#</span> -<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> -<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> -<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> -<span class="c1"># See the License for the specific language governing permissions and</span> -<span class="c1"># limitations under the License.</span> - -<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span> -<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span> - -<span class="kn">from</span> <span class="nn">six</span> <span class="k">import</span> <span class="n">BytesIO</span> -<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="k">import</span> <span class="n">urlparse</span> -<span class="kn">import</span> <span class="nn">re</span> -<span class="kn">import</span> <span class="nn">fnmatch</span> - -<div class="viewcode-block" id="S3Hook"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook">[docs]</a><span class="k">class</span> <span class="nc">S3Hook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span> - <span class="sd">"""</span> -<span class="sd"> Interact with AWS S3, using the boto3 library.</span> -<span class="sd"> """</span> - - <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> - <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">'s3'</span><span class="p">)</span> - - <span class="nd">@staticmethod</span> - <span class="k">def</span> <span class="nf">parse_s3_url</span><span class="p">(</span><span class="n">s3url</span><span class="p">):</span> - <span class="n">parsed_url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">s3url</span><span class="p">)</span> - <span class="k">if</span> <span class="ow">not</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span><span class="p">:</span> - <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Please provide a bucket_name'</span><span class="p">)</span> - <span class="k">else</span><span class="p">:</span> - <span class="n">bucket_name</span> <span class="o">=</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span> - <span class="n">key</span> <span class="o">=</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)</span> - <span class="k">return</span> <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> - -<div class="viewcode-block" id="S3Hook.check_for_bucket"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.check_for_bucket">[docs]</a> <span class="k">def</span> <span class="nf">check_for_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span> - <span class="sd">"""</span> -<span class="sd"> Check if bucket_name exists.</span> - -<span class="sd"> :param bucket_name: the name of the bucket</span> -<span class="sd"> :type bucket_name: str</span> -<span class="sd"> """</span> - <span class="k">try</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">head_bucket</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">)</span> - <span class="k">return</span> <span class="kc">True</span> - <span class="k">except</span><span class="p">:</span> - <span class="k">return</span> <span class="kc">False</span></div> - -<div class="viewcode-block" id="S3Hook.get_bucket"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.get_bucket">[docs]</a> <span class="k">def</span> <span class="nf">get_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span> - <span class="sd">"""</span> -<span class="sd"> Returns a boto3.S3.Bucket object</span> - -<span class="sd"> :param bucket_name: the name of the bucket</span> -<span class="sd"> :type bucket_name: str</span> -<span class="sd"> """</span> - <span class="n">s3</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_resource_type</span><span class="p">(</span><span class="s1">'s3'</span><span class="p">)</span> - <span class="k">return</span> <span class="n">s3</span><span class="o">.</span><span class="n">Bucket</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">)</span></div> - -<div class="viewcode-block" id="S3Hook.check_for_prefix"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.check_for_prefix">[docs]</a> <span class="k">def</span> <span class="nf">check_for_prefix</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">):</span> - <span class="sd">"""</span> -<span class="sd"> Checks that a prefix exists in a bucket</span> -<span class="sd"> """</span> - <span class="n">prefix</span> <span class="o">=</span> <span class="n">prefix</span> <span class="o">+</span> <span class="n">delimiter</span> <span class="k">if</span> <span class="n">prefix</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">!=</span> <span class="n">delimiter</span> <span class="k">else</span> <span class="n">prefix</span> - <span class="n">prefix_split</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">r</span><span class="s1">'(\w+[</span><span class="si">{d}</span><span class="s1">])$'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">d</span><span class="o">=</span><span class="n">delimiter</span><span class="p">),</span> <span class="n">prefix</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span> - <span class="n">previous_level</span> <span class="o">=</span> <span class="n">prefix_split</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> - <span class="n">plist</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">list_prefixes</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">previous_level</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">)</span> - <span class="k">return</span> <span class="kc">False</span> <span class="k">if</span> <span class="n">plist</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">prefix</span> <span class="ow">in</span> <span class="n">plist</span></div> - -<div class="viewcode-block" id="S3Hook.list_prefixes"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.list_prefixes">[docs]</a> <span class="k">def</span> <span class="nf">list_prefixes</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span> - <span class="sd">"""</span> -<span class="sd"> Lists prefixes in a bucket under prefix</span> - -<span class="sd"> :param bucket_name: the name of the bucket</span> -<span class="sd"> :type bucket_name: str</span> -<span class="sd"> :param prefix: a key prefix</span> -<span class="sd"> :type prefix: str</span> -<span class="sd"> :param delimiter: the delimiter marks key hierarchy.</span> -<span class="sd"> :type delimiter: str</span> -<span class="sd"> """</span> - <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">list_objects_v2</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span> - <span class="n">Prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span> - <span class="n">Delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span> - <span class="k">return</span> <span class="p">[</span><span class="n">p</span><span class="p">[</span><span class="s1">'Prefix'</span><span class="p">]</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s1">'CommonPrefixes'</span><span class="p">]]</span> <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'CommonPrefixes'</span><span class="p">)</span> <span class="k">else</span> <span class="kc">None</span></div> - -<div class="viewcode-block" id="S3Hook.list_keys"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.list_keys">[docs]</a> <span class="k">def</span> <span class="nf">list_keys</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span> - <span class="sd">"""</span> -<span class="sd"> Lists keys in a bucket under prefix and not containing delimiter</span> - -<span class="sd"> :param bucket_name: the name of the bucket</span> -<span class="sd"> :type bucket_name: str</span> -<span class="sd"> :param prefix: a key prefix</span> -<span class="sd"> :type prefix: str</span> -<span class="sd"> :param delimiter: the delimiter marks key hierarchy.</span> -<span class="sd"> :type delimiter: str</span> -<span class="sd"> """</span> - <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">list_objects_v2</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span> - <span class="n">Prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span> - <span class="n">Delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span> - <span class="k">return</span> <span class="p">[</span><span class="n">k</span><span class="p">[</span><span class="s1">'Key'</span><span class="p">]</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s1">'Contents'</span><span class="p">]]</span> <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Contents'</span><span class="p">)</span> <span class="k">else</span> <span class="kc">None</span></div> - -<div class="viewcode-block" id="S3Hook.check_for_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.check_for_key">[docs]</a> <span class="k">def</span> <span class="nf">check_for_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> - <span class="sd">"""</span> -<span class="sd"> Checks if a key exists in a bucket</span> - -<span class="sd"> :param key: S3 key that will point to the file</span> -<span class="sd"> :type key: str</span> -<span class="sd"> :param bucket_name: Name of the bucket in which the file is stored</span> -<span class="sd"> :type bucket_name: str</span> -<span class="sd"> """</span> - <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span> - <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">key</span><span class="p">)</span> - - <span class="k">try</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">head_object</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">Key</span><span class="o">=</span><span class="n">key</span><span class="p">)</span> - <span class="k">return</span> <span class="kc">True</span> - <span class="k">except</span><span class="p">:</span> - <span class="k">return</span> <span class="kc">False</span></div> - -<div class="viewcode-block" id="S3Hook.get_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.get_key">[docs]</a> <span class="k">def</span> <span class="nf">get_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> - <span class="sd">"""</span> -<span class="sd"> Returns a boto3.s3.Object</span> - -<span class="sd"> :param key: the path to the key</span> -<span class="sd"> :type key: str</span> -<span class="sd"> :param bucket_name: the name of the bucket</span> -<span class="sd"> :type bucket_name: str</span> -<span class="sd"> """</span> - <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span> - <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">key</span><span class="p">)</span> - - <span class="n">obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_resource_type</span><span class="p">(</span><span class="s1">'s3'</span><span class="p">)</span><span class="o">.</span><span class="n">Object</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> - <span class="n">obj</span><span class="o">.</span><span class="n">load</span><span class="p">()</span> - <span class="k">return</span> <span class="n">obj</span></div> - -<div class="viewcode-block" id="S3Hook.read_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.read_key">[docs]</a> <span class="k">def</span> <span class="nf">read_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> - <span class="sd">"""</span> -<span class="sd"> Reads a key from S3</span> - -<span class="sd"> :param key: S3 key that will point to the file</span> -<span class="sd"> :type key: str</span> -<span class="sd"> :param bucket_name: Name of the bucket in which the file is stored</span> -<span class="sd"> :type bucket_name: str</span> -<span class="sd"> """</span> - - <span class="n">obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">)</span> - <span class="k">return</span> <span class="n">obj</span><span class="o">.</span><span class="n">get</span><span class="p">()[</span><span class="s1">'Body'</span><span class="p">]</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span></div> - -<div class="viewcode-block" id="S3Hook.check_for_wildcard_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.check_for_wildcard_key">[docs]</a> <span class="k">def</span> <span class="nf">check_for_wildcard_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> - <span class="n">wildcard_key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span> - <span class="sd">"""</span> -<span class="sd"> Checks that a key matching a wildcard expression exists in a bucket</span> -<span class="sd"> """</span> - <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_wildcard_key</span><span class="p">(</span><span class="n">wildcard_key</span><span class="o">=</span><span class="n">wildcard_key</span><span class="p">,</span> - <span class="n">bucket_name</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span> - <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span></div> - -<div class="viewcode-block" id="S3Hook.get_wildcard_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.get_wildcard_key">[docs]</a> <span class="k">def</span> <span class="nf">get_wildcard_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span> - <span class="sd">"""</span> -<span class="sd"> Returns a boto3.s3.Object object matching the regular expression</span> - -<span class="sd"> :param regex_key: the path to the key</span> -<span class="sd"> :type regex_key: str</span> -<span class="sd"> :param bucket_name: the name of the bucket</span> -<span class="sd"> :type bucket_name: str</span> -<span class="sd"> """</span> - <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span> - <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">wildcard_key</span><span class="p">)</span> - - <span class="n">prefix</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">r</span><span class="s1">'[*]'</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> - <span class="n">klist</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">list_keys</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span> - <span class="k">if</span> <span class="n">klist</span><span class="p">:</span> - <span class="n">key_matches</span> <span class="o">=</span> <span class="p">[</span><span class="n">k</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">klist</span> <span class="k">if</span> <span class="n">fnmatch</span><span class="o">.</span><span class="n">fnmatch</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">)]</span> - <span class="k">if</span> <span class="n">key_matches</span><span class="p">:</span> - <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="n">key_matches</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">bucket_name</span><span class="p">)</span></div> - -<div class="viewcode-block" id="S3Hook.load_file"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.load_file">[docs]</a> <span class="k">def</span> <span class="nf">load_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> - <span class="n">filename</span><span class="p">,</span> - <span class="n">key</span><span class="p">,</span> - <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> - <span class="n">replace</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> - <span class="n">encrypt</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> - <span class="sd">"""</span> -<span class="sd"> Loads a local file to S3</span> - -<span class="sd"> :param filename: name of the file to load.</span> -<span class="sd"> :type filename: str</span> -<span class="sd"> :param key: S3 key that will point to the file</span> -<span class="sd"> :type key: str</span> -<span class="sd"> :param bucket_name: Name of the bucket in which to store the file</span> -<span class="sd"> :type bucket_name: str</span> -<span class="sd"> :param replace: A flag to decide whether or not to overwrite the key</span> -<span class="sd"> if it already exists. If replace is False and the key exists, an</span> -<span class="sd"> error will be raised.</span> -<span class="sd"> :type replace: bool</span> -<span class="sd"> :param encrypt: If True, the file will be encrypted on the server-side</span> -<span class="sd"> by S3 and will be stored in an encrypted form while at rest in S3.</span> -<span class="sd"> :type encrypt: bool</span> -<span class="sd"> """</span> - <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span> - <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">key</span><span class="p">)</span> - - <span class="k">if</span> <span class="ow">not</span> <span class="n">replace</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">check_for_key</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span> - <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"The key </span><span class="si">{key}</span><span class="s2"> already exists."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">))</span> - - <span class="n">extra_args</span><span class="o">=</span><span class="p">{}</span> - <span class="k">if</span> <span class="n">encrypt</span><span class="p">:</span> - <span class="n">extra_args</span><span class="p">[</span><span class="s1">'ServerSideEncryption'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"AES256"</span> - - <span class="n">client</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span> - <span class="n">client</span><span class="o">.</span><span class="n">upload_file</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">ExtraArgs</span><span class="o">=</span><span class="n">extra_args</span><span class="p">)</span></div> - -<div class="viewcode-block" id="S3Hook.load_string"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.load_string">[docs]</a> <span class="k">def</span> <span class="nf">load_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> - <span class="n">string_data</span><span class="p">,</span> - <span class="n">key</span><span class="p">,</span> - <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> - <span class="n">replace</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> - <span class="n">encrypt</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> - <span class="n">encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">):</span> - <span class="sd">"""</span> -<span class="sd"> Loads a string to S3</span> - -<span class="sd"> This is provided as a convenience to drop a string in S3. It uses the</span> -<span class="sd"> boto infrastructure to ship a file to s3. </span> - -<span class="sd"> :param string_data: string to set as content for the key.</span> -<span class="sd"> :type string_data: str</span> -<span class="sd"> :param key: S3 key that will point to the file</span> -<span class="sd"> :type key: str</span> -<span class="sd"> :param bucket_name: Name of the bucket in which to store the file</span> -<span class="sd"> :type bucket_name: str</span> -<span class="sd"> :param replace: A flag to decide whether or not to overwrite the key</span> -<span class="sd"> if it already exists</span> -<span class="sd"> :type replace: bool</span> -<span class="sd"> :param encrypt: If True, the file will be encrypted on the server-side</span> -<span class="sd"> by S3 and will be stored in an encrypted form while at rest in S3.</span> -<span class="sd"> :type encrypt: bool</span> -<span class="sd"> """</span> - <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span> - <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">key</span><span class="p">)</span> - - <span class="k">if</span> <span class="ow">not</span> <span class="n">replace</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">check_for_key</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span> - <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"The key </span><span class="si">{key}</span><span class="s2"> already exists."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">))</span> - - <span class="n">extra_args</span><span class="o">=</span><span class="p">{}</span> - <span class="k">if</span> <span class="n">encrypt</span><span class="p">:</span> - <span class="n">extra_args</span><span class="p">[</span><span class="s1">'ServerSideEncryption'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"AES256"</span> - - <span class="n">filelike_buffer</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="n">string_data</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">encoding</span><span class="p">))</span> - - <span class="n">client</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span> - <span class="n">client</span><span class="o">.</span><span class="n">upload_fileobj</span><span class="p">(</span><span class="n">filelike_buffer</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">ExtraArgs</span><span class="o">=</span><span class="n">extra_args</span><span class="p">)</span></div></div> -</pre></div> - - </div> - <div class="articleComments"> - - </div> - </div> - <footer> - - - <hr/> - - <div role="contentinfo"> - <p> - - </p> - </div> - Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. - -</footer> - - </div> - </div> - - </section> - - </div> - - - - - - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../', - VERSION:'', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../_static/jquery.js"></script> - <script type="text/javascript" src="../_static/underscore.js"></script> - <script type="text/javascript" src="../_static/doctools.js"></script> - - - - - - <script type="text/javascript" src="../_static/js/theme.js"></script> - - - - - <script type="text/javascript"> - jQuery(function () { - SphinxRtdTheme.StickyNav.enable(); - }); - </script> - - -</body> -</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/executors/mesos_executor.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/executors/mesos_executor.html b/_modules/airflow/contrib/executors/mesos_executor.html index f23df55..312fffe 100644 --- a/_modules/airflow/contrib/executors/mesos_executor.html +++ b/_modules/airflow/contrib/executors/mesos_executor.html @@ -91,7 +91,7 @@ <li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li> -<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li> @@ -99,8 +99,10 @@ <li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li> </ul> @@ -169,17 +171,22 @@ <h1>Source code for airflow.contrib.executors.mesos_executor</h1><div class="highlight"><pre> <span></span><span class="c1"># -*- coding: utf-8 -*-</span> <span class="c1">#</span> -<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> -<span class="c1"># you may not use this file except in compliance with the License.</span> -<span class="c1"># You may obtain a copy of the License at</span> -<span class="c1">#</span> -<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> -<span class="c1">#</span> -<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> -<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> -<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> -<span class="c1"># See the License for the specific language governing permissions and</span> -<span class="c1"># limitations under the License.</span> +<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span> +<span class="c1"># or more contributor license agreements. See the NOTICE file</span> +<span class="c1"># distributed with this work for additional information</span> +<span class="c1"># regarding copyright ownership. The ASF licenses this file</span> +<span class="c1"># to you under the Apache License, Version 2.0 (the</span> +<span class="c1"># "License"); you may not use this file except in compliance</span> +<span class="c1"># with the License. You may obtain a copy of the License at</span> +<span class="c1"># </span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1"># </span> +<span class="c1"># Unless required by applicable law or agreed to in writing,</span> +<span class="c1"># software distributed under the License is distributed on an</span> +<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span> +<span class="c1"># KIND, either express or implied. See the License for the</span> +<span class="c1"># specific language governing permissions and limitations</span> +<span class="c1"># under the License.</span> <span class="kn">from</span> <span class="nn">future</span> <span class="k">import</span> <span class="n">standard_library</span> @@ -206,9 +213,9 @@ <span class="k">def</span> <span class="nf">get_framework_name</span><span class="p">():</span> - <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">):</span> + <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">):</span> <span class="k">return</span> <span class="n">DEFAULT_FRAMEWORK_NAME</span> - <span class="k">return</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">)</span> + <span class="k">return</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">)</span> <span class="c1"># AirflowMesosScheduler, implements Mesos Scheduler interface</span> @@ -232,11 +239,16 @@ <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span> <span class="o">=</span> <span class="n">task_mem</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span> <span class="o">=</span> <span class="mi">0</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span> <span class="o">=</span> <span class="p">{}</span> + <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DOCKER_IMAGE_SLAVE'</span><span class="p">):</span> + <span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span> + <span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DOCKER_IMAGE_SLAVE'</span> + <span class="p">)</span> <span class="k">def</span> <span class="nf">registered</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">frameworkId</span><span class="p">,</span> <span class="n">masterInfo</span><span class="p">):</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler registered to Mesos with framework ID </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span> - <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">)</span> <span class="ow">and</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span> + <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">)</span> <span class="ow">and</span> \ + <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span> <span class="c1"># Import here to work around a circular import error</span> <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span> @@ -322,6 +334,21 @@ <span class="n">command</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">cmd</span> <span class="n">task</span><span class="o">.</span><span class="n">command</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">command</span><span class="p">)</span> + <span class="c1"># If docker image for airflow is specified in config then pull that</span> + <span class="c1"># image before running the above airflow command</span> + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span><span class="p">:</span> + <span class="n">network</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DockerInfo</span><span class="o">.</span><span class="n">Network</span><span class="o">.</span><span class="n">Value</span><span class="p">(</span><span class="s1">'BRIDGE'</span><span class="p">)</span> + <span class="n">docker</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DockerInfo</span><span class="p">(</span> + <span class="n">image</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span><span class="p">,</span> + <span class="n">force_pull_image</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> + <span class="n">network</span><span class="o">=</span><span class="n">network</span> + <span class="p">)</span> + <span class="n">container</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="p">(</span> + <span class="nb">type</span><span class="o">=</span><span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DOCKER</span><span class="p">,</span> + <span class="n">docker</span><span class="o">=</span><span class="n">docker</span> + <span class="p">)</span> + <span class="n">task</span><span class="o">.</span><span class="n">container</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">container</span><span class="p">)</span> + <span class="n">tasks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">task</span><span class="p">)</span> <span class="n">remainingCpus</span> <span class="o">-=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span> @@ -371,28 +398,28 @@ <span class="n">framework</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">FrameworkInfo</span><span class="p">()</span> <span class="n">framework</span><span class="o">.</span><span class="n">user</span> <span class="o">=</span> <span class="s1">''</span> - <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">):</span> + <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">):</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting mesos master URL for mesos executor"</span><span class="p">)</span> <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"mesos.master not provided for mesos executor"</span><span class="p">)</span> - <span class="n">master</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">)</span> + <span class="n">master</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">)</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">get_framework_name</span><span class="p">()</span> - <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">):</span> + <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">):</span> <span class="n">task_cpu</span> <span class="o">=</span> <span class="mi">1</span> <span class="k">else</span><span class="p">:</span> - <span class="n">task_cpu</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">)</span> + <span class="n">task_cpu</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">)</span> - <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">):</span> + <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">):</span> <span class="n">task_memory</span> <span class="o">=</span> <span class="mi">256</span> <span class="k">else</span><span class="p">:</span> - <span class="n">task_memory</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">)</span> + <span class="n">task_memory</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">)</span> - <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">):</span> + <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">):</span> <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">True</span> - <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span> + <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span> <span class="c1"># Import here to work around a circular import error</span> <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span> @@ -404,7 +431,9 @@ <span class="c1"># Set the Framework ID to let the scheduler reconnect with running tasks.</span> <span class="n">framework</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">extra</span> - <span class="n">framework</span><span class="o">.</span><span class="n">failover_timeout</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">)</span> + <span class="n">framework</span><span class="o">.</span><span class="n">failover_timeout</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span> + <span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span> + <span class="p">)</span> <span class="k">else</span><span class="p">:</span> <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">False</span> @@ -415,17 +444,17 @@ <span class="n">implicit_acknowledgements</span> <span class="o">=</span> <span class="mi">1</span> - <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'AUTHENTICATE'</span><span class="p">):</span> - <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">):</span> + <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'AUTHENTICATE'</span><span class="p">):</span> + <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">):</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting authentication principal in the environment"</span><span class="p">)</span> <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"mesos.default_principal not provided in authenticated mode"</span><span class="p">)</span> - <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">):</span> + <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">):</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting authentication secret in the environment"</span><span class="p">)</span> <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"mesos.default_secret not provided in authenticated mode"</span><span class="p">)</span> <span class="n">credential</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Credential</span><span class="p">()</span> - <span class="n">credential</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">)</span> - <span class="n">credential</span><span class="o">.</span><span class="n">secret</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">)</span> + <span class="n">credential</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">)</span> + <span class="n">credential</span><span class="o">.</span><span class="n">secret</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">)</span> <span class="n">framework</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">credential</span><span class="o">.</span><span class="n">principal</span> @@ -446,7 +475,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span> <span class="o">=</span> <span class="n">driver</span> <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">start</span><span class="p">()</span> - <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> + <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">executor_config</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span> <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>