Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-airflow
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/9bc0d399 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/9bc0d399 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/9bc0d399 Branch: refs/heads/master Commit: 9bc0d399a6d303ff1068b880f1de76dfd5846c6f Parents: 004a347 6ac2963 Author: Chris Riccomini <criccom...@apache.org> Authored: Wed Sep 13 09:31:44 2017 -0700 Committer: Chris Riccomini <criccom...@apache.org> Committed: Wed Sep 13 09:31:44 2017 -0700 ---------------------------------------------------------------------- .rat-excludes | 10 + CHANGELOG.txt | 45 +++ INSTALL | 9 + README.md | 1 + UPDATING.md | 4 +- airflow/__init__.py | 10 +- airflow/api/__init__.py | 12 +- airflow/api/auth/backend/kerberos_auth.py | 14 +- airflow/api/common/experimental/get_task.py | 4 - .../common/experimental/get_task_instance.py | 4 - airflow/bin/airflow | 1 - airflow/bin/cli.py | 46 +-- airflow/configuration.py | 25 +- .../auth/backends/github_enterprise_auth.py | 5 +- airflow/contrib/auth/backends/google_auth.py | 11 +- airflow/contrib/auth/backends/kerberos_auth.py | 5 +- airflow/contrib/auth/backends/ldap_auth.py | 32 +- airflow/contrib/auth/backends/password_auth.py | 10 +- airflow/contrib/executors/mesos_executor.py | 51 +-- airflow/contrib/hooks/bigquery_hook.py | 66 ++-- airflow/contrib/hooks/cloudant_hook.py | 10 +- airflow/contrib/hooks/databricks_hook.py | 32 +- airflow/contrib/hooks/datadog_hook.py | 13 +- airflow/contrib/hooks/datastore_hook.py | 3 +- airflow/contrib/hooks/ftp_hook.py | 10 +- airflow/contrib/hooks/gcp_api_base_hook.py | 13 +- airflow/contrib/hooks/gcp_dataflow_hook.py | 29 +- airflow/contrib/hooks/gcp_dataproc_hook.py | 28 +- airflow/contrib/hooks/gcp_mlengine_hook.py | 48 ++- airflow/contrib/hooks/gcs_hook.py | 10 +- airflow/contrib/hooks/jira_hook.py | 9 +- airflow/contrib/hooks/qubole_hook.py | 22 +- airflow/contrib/hooks/redis_hook.py | 16 +- airflow/contrib/hooks/salesforce_hook.py | 31 +- airflow/contrib/hooks/spark_sql_hook.py | 10 +- airflow/contrib/hooks/spark_submit_hook.py | 24 +- airflow/contrib/hooks/sqoop_hook.py | 17 +- airflow/contrib/hooks/ssh_hook.py | 34 +- airflow/contrib/hooks/wasb_hook.py | 18 + airflow/contrib/operators/bigquery_operator.py | 4 +- .../operators/bigquery_table_delete_operator.py | 4 +- .../contrib/operators/bigquery_to_bigquery.py | 8 +- airflow/contrib/operators/bigquery_to_gcs.py | 4 +- .../contrib/operators/databricks_operator.py | 29 +- airflow/contrib/operators/dataproc_operator.py | 30 +- .../operators/datastore_export_operator.py | 5 +- .../operators/datastore_import_operator.py | 6 +- airflow/contrib/operators/ecs_operator.py | 24 +- .../contrib/operators/emr_add_steps_operator.py | 7 +- .../operators/emr_create_job_flow_operator.py | 10 +- .../emr_terminate_job_flow_operator.py | 7 +- airflow/contrib/operators/file_to_wasb.py | 14 +- airflow/contrib/operators/fs_operator.py | 4 +- .../contrib/operators/gcs_download_operator.py | 6 +- airflow/contrib/operators/gcs_to_bq.py | 18 +- airflow/contrib/operators/hipchat_operator.py | 3 +- airflow/contrib/operators/mlengine_operator.py | 30 +- .../operators/mlengine_prediction_summary.py | 2 - airflow/contrib/operators/mysql_to_gcs.py | 5 +- airflow/contrib/operators/sftp_operator.py | 7 +- .../contrib/operators/spark_submit_operator.py | 6 +- airflow/contrib/operators/ssh_operator.py | 1 - airflow/contrib/operators/vertica_operator.py | 5 +- airflow/contrib/operators/vertica_to_hive.py | 5 +- airflow/contrib/sensors/bigquery_sensor.py | 5 +- airflow/contrib/sensors/datadog_sensor.py | 5 +- airflow/contrib/sensors/emr_base_sensor.py | 9 +- airflow/contrib/sensors/emr_job_flow_sensor.py | 6 +- airflow/contrib/sensors/emr_step_sensor.py | 5 +- airflow/contrib/sensors/ftp_sensor.py | 4 +- airflow/contrib/sensors/gcs_sensor.py | 7 +- airflow/contrib/sensors/hdfs_sensors.py | 12 +- airflow/contrib/sensors/jira_sensor.py | 27 +- airflow/contrib/sensors/redis_key_sensor.py | 4 - airflow/contrib/sensors/wasb_sensor.py | 11 +- .../contrib/task_runner/cgroup_task_runner.py | 49 +-- airflow/executors/__init__.py | 8 +- airflow/executors/base_executor.py | 18 +- airflow/executors/celery_executor.py | 24 +- airflow/executors/dask_executor.py | 10 +- airflow/executors/local_executor.py | 11 +- airflow/executors/sequential_executor.py | 4 +- airflow/hooks/S3_hook.py | 52 +-- airflow/hooks/base_hook.py | 9 +- airflow/hooks/dbapi_hook.py | 33 +- airflow/hooks/druid_hook.py | 10 +- airflow/hooks/hive_hooks.py | 37 +- airflow/hooks/http_hook.py | 7 +- airflow/hooks/oracle_hook.py | 9 +- airflow/hooks/pig_hook.py | 6 +- airflow/hooks/presto_hook.py | 4 - airflow/hooks/webhdfs_hook.py | 17 +- airflow/hooks/zendesk_hook.py | 20 +- airflow/jobs.py | 352 +++++++++---------- ...d2ae31099d61_increase_text_size_for_mysql.py | 41 +++ airflow/models.py | 231 ++++++------ airflow/operators/bash_operator.py | 23 +- airflow/operators/check_operator.py | 24 +- airflow/operators/dagrun_operator.py | 6 +- airflow/operators/docker_operator.py | 12 +- airflow/operators/generic_transfer.py | 13 +- airflow/operators/hive_operator.py | 4 +- airflow/operators/hive_stats_operator.py | 10 +- airflow/operators/hive_to_druid.py | 18 +- airflow/operators/hive_to_mysql.py | 14 +- airflow/operators/hive_to_samba_operator.py | 6 +- airflow/operators/http_operator.py | 8 +- airflow/operators/jdbc_operator.py | 7 +- airflow/operators/latest_only_operator.py | 19 +- airflow/operators/mssql_operator.py | 5 +- airflow/operators/mssql_to_hive.py | 6 +- airflow/operators/mysql_operator.py | 5 +- airflow/operators/mysql_to_hive.py | 5 +- airflow/operators/oracle_operator.py | 5 +- airflow/operators/pig_operator.py | 4 +- airflow/operators/postgres_operator.py | 5 +- airflow/operators/presto_to_mysql.py | 12 +- airflow/operators/python_operator.py | 24 +- airflow/operators/redshift_to_s3_operator.py | 17 +- airflow/operators/s3_file_transform_operator.py | 30 +- airflow/operators/s3_to_hive_operator.py | 39 +- airflow/operators/sensors.py | 49 +-- airflow/operators/slack_operator.py | 9 +- airflow/operators/sqlite_operator.py | 5 +- airflow/plugins_manager.py | 11 +- airflow/security/kerberos.py | 25 +- airflow/settings.py | 17 +- airflow/task_runner/base_task_runner.py | 9 +- airflow/utils/dag_processing.py | 55 +-- airflow/utils/db.py | 15 +- airflow/utils/email.py | 8 +- airflow/utils/log/LoggingMixin.py | 45 +++ airflow/utils/log/file_task_handler.py | 34 +- airflow/utils/log/gcs_task_handler.py | 125 ++++++- airflow/utils/log/s3_task_handler.py | 97 ++++- airflow/utils/logging.py | 252 ------------- airflow/utils/timeout.py | 17 +- airflow/www/api/experimental/endpoints.py | 6 +- airflow/www/app.py | 10 +- airflow/www/views.py | 23 +- dev/airflow-pr | 11 +- docs/integration.rst | 96 ++++- setup.py | 15 +- tests/contrib/hooks/test_databricks_hook.py | 52 ++- tests/contrib/hooks/test_wasb_hook.py | 14 +- .../contrib/operators/test_dataproc_operator.py | 59 ++-- tests/core.py | 20 +- tests/operators/sensors.py | 53 +-- tests/utils/log/test_logging.py | 108 ++++++ tests/utils/test_logging.py | 103 ------ 150 files changed, 1906 insertions(+), 1791 deletions(-) ----------------------------------------------------------------------