[AIRFLOW-1433][AIRFLOW-85] New Airflow Webserver UI with RBAC support Closes #3015 from jgao54/rbac
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/05e1861e Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/05e1861e Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/05e1861e Branch: refs/heads/master Commit: 05e1861e24de42f9a2c649cd93041c5c744504e1 Parents: bd01004 Author: Joy Gao <joy...@apache.org> Authored: Fri Mar 23 09:18:48 2018 +0100 Committer: Bolke de Bruin <bo...@xs4all.nl> Committed: Fri Mar 23 09:18:48 2018 +0100 ---------------------------------------------------------------------- MANIFEST.in | 3 + UPDATING.md | 30 + airflow/bin/cli.py | 81 +- .../config_templates/airflow_local_settings.py | 10 + airflow/config_templates/default_airflow.cfg | 4 + airflow/config_templates/default_test.cfg | 1 + .../default_webserver_config.py | 84 + airflow/configuration.py | 10 + airflow/models.py | 60 +- airflow/settings.py | 2 +- airflow/utils/db.py | 19 +- airflow/www/views.py | 7 +- airflow/www_rbac/__init__.py | 13 + airflow/www_rbac/api/__init__.py | 14 + airflow/www_rbac/api/experimental/__init__.py | 14 + airflow/www_rbac/api/experimental/endpoints.py | 231 + airflow/www_rbac/app.py | 174 + airflow/www_rbac/blueprints.py | 30 + airflow/www_rbac/decorators.py | 88 + airflow/www_rbac/forms.py | 130 + airflow/www_rbac/security.py | 174 + airflow/www_rbac/static/airflow.gif | Bin 0 -> 622963 bytes airflow/www_rbac/static/bootstrap-theme.css | 6494 ++++++++ .../www_rbac/static/bootstrap-toggle.min.css | 28 + airflow/www_rbac/static/bootstrap-toggle.min.js | 9 + .../www_rbac/static/bootstrap3-typeahead.min.js | 21 + airflow/www_rbac/static/connection_form.js | 78 + airflow/www_rbac/static/d3.tip.v0.6.3.js | 302 + airflow/www_rbac/static/d3.v3.min.js | 5 + airflow/www_rbac/static/dagre-d3.js | 5007 ++++++ airflow/www_rbac/static/dagre-d3.min.js | 2 + airflow/www_rbac/static/dagre.css | 38 + .../www_rbac/static/dataTables.bootstrap.css | 333 + airflow/www_rbac/static/docs | 1 + airflow/www_rbac/static/favicon.ico | Bin 0 -> 1406 bytes airflow/www_rbac/static/gantt-chart-d3v2.js | 267 + airflow/www_rbac/static/gantt.css | 57 + airflow/www_rbac/static/graph.css | 72 + airflow/www_rbac/static/jqClock.min.js | 27 + airflow/www_rbac/static/jquery.dataTables.css | 495 + .../www_rbac/static/jquery.dataTables.min.js | 189 + airflow/www_rbac/static/loading.gif | Bin 0 -> 16671 bytes airflow/www_rbac/static/main.css | 267 + airflow/www_rbac/static/nv.d3.css | 788 + airflow/www_rbac/static/nv.d3.js | 14260 +++++++++++++++++ airflow/www_rbac/static/pin.svg | 129 + airflow/www_rbac/static/pin_100.jpg | Bin 0 -> 6780 bytes airflow/www_rbac/static/pin_100.png | Bin 0 -> 10977 bytes airflow/www_rbac/static/pin_25.png | Bin 0 -> 1442 bytes airflow/www_rbac/static/pin_30.png | Bin 0 -> 2015 bytes airflow/www_rbac/static/pin_35.png | Bin 0 -> 2171 bytes airflow/www_rbac/static/pin_40.png | Bin 0 -> 2685 bytes airflow/www_rbac/static/pin_large.jpg | Bin 0 -> 399613 bytes airflow/www_rbac/static/pin_large.png | Bin 0 -> 358276 bytes airflow/www_rbac/static/screenshots/gantt.png | Bin 0 -> 31140 bytes airflow/www_rbac/static/screenshots/graph.png | Bin 0 -> 38282 bytes airflow/www_rbac/static/screenshots/tree.png | Bin 0 -> 35259 bytes airflow/www_rbac/static/sort_asc.png | Bin 0 -> 160 bytes airflow/www_rbac/static/sort_both.png | Bin 0 -> 201 bytes airflow/www_rbac/static/sort_desc.png | Bin 0 -> 158 bytes airflow/www_rbac/static/tree.css | 96 + airflow/www_rbac/templates/airflow/chart.html | 57 + airflow/www_rbac/templates/airflow/circles.html | 144 + airflow/www_rbac/templates/airflow/code.html | 43 + airflow/www_rbac/templates/airflow/config.html | 69 + airflow/www_rbac/templates/airflow/confirm.html | 35 + .../www_rbac/templates/airflow/conn_create.html | 23 + .../www_rbac/templates/airflow/conn_edit.html | 23 + airflow/www_rbac/templates/airflow/dag.html | 430 + .../www_rbac/templates/airflow/dag_code.html | 58 + .../www_rbac/templates/airflow/dag_details.html | 72 + airflow/www_rbac/templates/airflow/dags.html | 468 + .../templates/airflow/duration_chart.html | 77 + airflow/www_rbac/templates/airflow/gantt.html | 67 + airflow/www_rbac/templates/airflow/graph.html | 369 + airflow/www_rbac/templates/airflow/master.html | 18 + .../www_rbac/templates/airflow/model_list.html | 93 + .../www_rbac/templates/airflow/noaccess.html | 24 + airflow/www_rbac/templates/airflow/task.html | 75 + .../templates/airflow/task_instance.html | 75 + airflow/www_rbac/templates/airflow/ti_code.html | 43 + airflow/www_rbac/templates/airflow/ti_log.html | 40 + .../www_rbac/templates/airflow/traceback.html | 33 + airflow/www_rbac/templates/airflow/tree.html | 381 + .../templates/airflow/variable_list.html | 30 + airflow/www_rbac/templates/airflow/version.html | 30 + airflow/www_rbac/templates/airflow/xcom.html | 37 + .../templates/appbuilder/baselayout.html | 84 + .../www_rbac/templates/appbuilder/index.html | 18 + .../www_rbac/templates/appbuilder/navbar.html | 47 + .../templates/appbuilder/navbar_menu.html | 57 + .../templates/appbuilder/navbar_right.html | 64 + airflow/www_rbac/utils.py | 350 + airflow/www_rbac/validators.py | 54 + airflow/www_rbac/views.py | 2056 +++ airflow/www_rbac/widgets.py | 19 + run_unit_tests.sh | 1 - scripts/ci/airflow_travis.cfg | 1 + setup.py | 3 +- tests/core.py | 4 +- tests/www_rbac/__init__.py | 13 + tests/www_rbac/api/__init__.py | 13 + tests/www_rbac/api/experimental/__init__.py | 13 + .../www_rbac/api/experimental/test_endpoints.py | 302 + .../api/experimental/test_kerberos_endpoints.py | 97 + .../2017-09-01T00.00.00/1.log | 1 + tests/www_rbac/test_security.py | 118 + tests/www_rbac/test_utils.py | 109 + tests/www_rbac/test_validators.py | 91 + tests/www_rbac/test_views.py | 405 + 110 files changed, 36839 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/MANIFEST.in ---------------------------------------------------------------------- diff --git a/MANIFEST.in b/MANIFEST.in index 69ccafe..2ee9f90 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -18,6 +18,9 @@ include CHANGELOG.txt include README.md graft airflow/www/templates graft airflow/www/static +graft airflow/www_rbac/static +graft airflow/www_rbac/templates +graft airflow/www_rbac/translations include airflow/alembic.ini graft scripts/systemd graft scripts/upstart http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/UPDATING.md ---------------------------------------------------------------------- diff --git a/UPDATING.md b/UPDATING.md index 21581bf..0c1b0d4 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -5,6 +5,36 @@ assists people when migrating to a new version. ## Airflow Master +### New Webserver UI with Role-Based Access Control + +Our current webserver UI uses the Flask-Admin extension. The new webserver UI uses the [Flask-AppBuilder (FAB)](https://github.com/dpgaspar/Flask-AppBuilder) extension. It has built-in authentication support and Role-Based Access Control (RBAC), which provides configurable roles and permissions for individual users. + +To turn on this feature, in your airflow.cfg file, under [webserver], set configuration variable `rbac = True`, and then run `airflow` command, which will generate the `webserver_config.py` file in your $AIRFLOW_HOME. + +#### Setting up Authentication + +FAB has built-in authentication support for DB, OAuth, OpenID, LDAP, and REMOTE_USER. The default auth type is `AUTH_DB`. + +For any other authentication type (OAuth, OpenID, LDAP, REMOTE_USER), see the [Authentication section of FAB docs](http://flask-appbuilder.readthedocs.io/en/latest/security.html#authentication-methods) for how to configure variables in webserver_config.py file. + +Once you modify your config file, run `airflow initdb` to generate new tables for RBAC support (these tables will have the prefix `ab_`). + +#### Creating an Admin Account + +Once you updated configuration settings and generated new tables, you need to create an admin account with `airflow create_user` command. + +#### Using your new UI + +Run `airflow webserver` as usual to start the new UI. This will bring you to a log in page, enter the admin username and password that were just created. + +There are five roles created for Airflow by default: Admin, User, Op, Viewer, and Public. To configure roles/permissions, go to the `Security` tab and click `List Roles` in the new UI. + +#### Breaking changes +- Users created and stored in the old users table will not be migrated automatically. You will need to reconfigure with one of FAB's built-in authentication support. +- Airflow dag home page is now `/home` (instead of `/admin`). +- All ModelViews in Flask-AppBuilder follow a different pattern from Flask-Admin. The `/admin` part of the url path will no longer exist. For example: `/admin/connection` becomes `/connection/list`, `/admin/connection/new` becomes `/connection/add`, `/admin/connection/edit` becomes `/connection/edit`, etc. +- Due to security concerns, the new webserver will no longer support the features in the `Data Profiling` menu of old UI, including `Ad Hoc Query`, `Charts`, and `Known Events`. + ### MySQL setting required We now rely on more strict ANSI SQL settings for MySQL in order to have sane defaults. Make sure http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/bin/cli.py ---------------------------------------------------------------------- diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py index 1801cc7..8d7d419 100755 --- a/airflow/bin/cli.py +++ b/airflow/bin/cli.py @@ -40,6 +40,7 @@ import traceback import time import psutil import re +import getpass from urllib.parse import urlunparse import airflow @@ -58,6 +59,9 @@ from airflow.utils.net import get_hostname from airflow.utils.log.logging_mixin import (LoggingMixin, redirect_stderr, redirect_stdout) from airflow.www.app import (cached_app, create_app) +from airflow.www_rbac.app import cached_app as cached_app_rbac +from airflow.www_rbac.app import create_app as create_app_rbac +from airflow.www_rbac.app import cached_appbuilder from sqlalchemy import func from sqlalchemy.orm import exc @@ -730,11 +734,11 @@ def webserver(args): print( "Starting the web server on port {0} and host {1}.".format( args.port, args.hostname)) - app = create_app(conf) + app = create_app_rbac(conf) if settings.RBAC else create_app(conf) app.run(debug=True, port=args.port, host=args.hostname, ssl_context=(ssl_cert, ssl_key) if ssl_cert and ssl_key else None) else: - app = cached_app(conf) + app = cached_app_rbac(conf) if settings.RBAC else cached_app(conf) pid, stdout, stderr, log_file = setup_locations( "webserver", args.pid, args.stdout, args.stderr, args.log_file) if args.daemon: @@ -760,7 +764,7 @@ def webserver(args): '-b', args.hostname + ':' + str(args.port), '-n', 'airflow-webserver', '-p', str(pid), - '-c', 'python:airflow.www.gunicorn_config' + '-c', 'python:airflow.www.gunicorn_config', ] if args.access_logfile: @@ -775,7 +779,8 @@ def webserver(args): if ssl_cert: run_args += ['--certfile', ssl_cert, '--keyfile', ssl_key] - run_args += ["airflow.www.app:cached_app()"] + webserver_module = 'www_rbac' if settings.RBAC else 'www' + run_args += ["airflow." + webserver_module + ".app:cached_app()"] gunicorn_master_proc = None @@ -934,7 +939,7 @@ def worker(args): def initdb(args): # noqa print("DB: " + repr(settings.engine.url)) - db_utils.initdb() + db_utils.initdb(settings.RBAC) print("Done.") @@ -943,7 +948,7 @@ def resetdb(args): if args.yes or input( "This will drop existing tables if they exist. " "Proceed? (y/n)").upper() == "Y": - db_utils.resetdb() + db_utils.resetdb(settings.RBAC) else: print("Bail.") @@ -1139,7 +1144,11 @@ def kerberos(args): # noqa import airflow.security.kerberos if args.daemon: - pid, stdout, stderr, log_file = setup_locations("kerberos", args.pid, args.stdout, args.stderr, args.log_file) + pid, stdout, stderr, log_file = setup_locations("kerberos", + args.pid, + args.stdout, + args.stderr, + args.log_file) stdout = open(stdout, 'w+') stderr = open(stderr, 'w+') @@ -1158,6 +1167,39 @@ def kerberos(args): # noqa airflow.security.kerberos.run() +def create_user(args): + fields = { + 'role': args.role, + 'username': args.username, + 'email': args.email, + 'firstname': args.firstname, + 'lastname': args.lastname, + } + empty_fields = [k for k, v in fields.items() if not v] + if empty_fields: + print('Missing arguments: {}.'.format(', '.join(empty_fields))) + sys.exit(0) + + appbuilder = cached_appbuilder() + role = appbuilder.sm.find_role(args.role) + if not role: + print('{} is not a valid role.'.format(args.role)) + sys.exit(0) + + password = getpass.getpass('Password:') + password_confirmation = getpass.getpass('Repeat for confirmation:') + if password != password_confirmation: + print('Passwords did not match!') + sys.exit(0) + + user = appbuilder.sm.add_user(args.username, args.firstname, args.lastname, + args.email, role, password) + if user: + print('{} user {} created.'.format(args.role, args.username)) + else: + print('Failed to create user.') + + Arg = namedtuple( 'Arg', ['flags', 'help', 'action', 'default', 'nargs', 'type', 'choices', 'metavar']) Arg.__new__.__defaults__ = (None, None, None, None, None, None, None) @@ -1523,6 +1565,27 @@ class CLIFactory(object): ('--conn_extra',), help='Connection `Extra` field, optional when adding a connection', type=str), + # create_user + 'role': Arg( + ('-r', '--role',), + help='Role of the user', + type=str), + 'firstname': Arg( + ('-f', '--firstname',), + help='First name of the admin user', + type=str), + 'lastname': Arg( + ('-l', '--lastname',), + help='Last name of the admin user', + type=str), + 'email': Arg( + ('-e', '--email',), + help='Email of the admin user', + type=str), + 'username': Arg( + ('-u', '--username',), + help='Username of the admin user', + type=str), } subparsers = ( { @@ -1660,6 +1723,10 @@ class CLIFactory(object): 'help': "List/Add/Delete connections", 'args': ('list_connections', 'add_connection', 'delete_connection', 'conn_id', 'conn_uri', 'conn_extra') + tuple(alternative_conn_specs), + }, { + 'func': create_user, + 'help': "Create an admin account", + 'args': ('role', 'username', 'email', 'firstname', 'lastname'), }, ) subparsers_dict = {sp['func'].__name__: sp for sp in subparsers} http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/config_templates/airflow_local_settings.py ---------------------------------------------------------------------- diff --git a/airflow/config_templates/airflow_local_settings.py b/airflow/config_templates/airflow_local_settings.py index 861c7a9..b72f999 100644 --- a/airflow/config_templates/airflow_local_settings.py +++ b/airflow/config_templates/airflow_local_settings.py @@ -22,6 +22,11 @@ from airflow import configuration as conf # settings.py and cli.py. Please see AIRFLOW-1455. LOG_LEVEL = conf.get('core', 'LOGGING_LEVEL').upper() + +# Flask appbuilder's info level log is very verbose, +# so it's set to 'WARN' by default. +FAB_LOG_LEVEL = 'WARN' + LOG_FORMAT = conf.get('core', 'LOG_FORMAT') BASE_LOG_FOLDER = conf.get('core', 'BASE_LOG_FOLDER') @@ -76,6 +81,11 @@ DEFAULT_LOGGING_CONFIG = { 'level': LOG_LEVEL, 'propagate': False, }, + 'flask_appbuilder': { + 'handler': ['console'], + 'level': FAB_LOG_LEVEL, + 'propagate': True, + } }, 'root': { 'handlers': ['console'], http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/config_templates/default_airflow.cfg ---------------------------------------------------------------------- diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index 8f82208..cc3e89f 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -266,6 +266,10 @@ hide_paused_dags_by_default = False # Consistent page size across all listing views in the UI page_size = 100 +# Use FAB-based webserver with RBAC feature +rbac = False + + [email] email_backend = airflow.utils.email.send_email_smtp http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/config_templates/default_test.cfg ---------------------------------------------------------------------- diff --git a/airflow/config_templates/default_test.cfg b/airflow/config_templates/default_test.cfg index 9016c2b..2f7cbb8 100644 --- a/airflow/config_templates/default_test.cfg +++ b/airflow/config_templates/default_test.cfg @@ -64,6 +64,7 @@ dag_default_view = tree log_fetch_timeout_sec = 5 hide_paused_dags_by_default = False page_size = 100 +rbac = False [email] email_backend = airflow.utils.email.send_email_smtp http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/config_templates/default_webserver_config.py ---------------------------------------------------------------------- diff --git a/airflow/config_templates/default_webserver_config.py b/airflow/config_templates/default_webserver_config.py new file mode 100644 index 0000000..953e650 --- /dev/null +++ b/airflow/config_templates/default_webserver_config.py @@ -0,0 +1,84 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from airflow import configuration as conf +from flask_appbuilder.security.manager import AUTH_DB +# from flask_appbuilder.security.manager import AUTH_LDAP +# from flask_appbuilder.security.manager import AUTH_OAUTH +# from flask_appbuilder.security.manager import AUTH_OID +# from flask_appbuilder.security.manager import AUTH_REMOTE_USER +basedir = os.path.abspath(os.path.dirname(__file__)) + +# The SQLAlchemy connection string. +SQLALCHEMY_DATABASE_URI = conf.get('core', 'SQL_ALCHEMY_CONN') + +# Flask-WTF flag for CSRF +CSRF_ENABLED = True + +# ---------------------------------------------------- +# AUTHENTICATION CONFIG +# ---------------------------------------------------- +# For details on how to set up each of the following authentication, see +# http://flask-appbuilder.readthedocs.io/en/latest/security.html# authentication-methods +# for details. + +# The authentication type +# AUTH_OID : Is for OpenID +# AUTH_DB : Is for database +# AUTH_LDAP : Is for LDAP +# AUTH_REMOTE_USER : Is for using REMOTE_USER from web server +# AUTH_OAUTH : Is for OAuth +AUTH_TYPE = AUTH_DB + +# Uncomment to setup Full admin role name +# AUTH_ROLE_ADMIN = 'Admin' + +# Uncomment to setup Public role name, no authentication needed +# AUTH_ROLE_PUBLIC = 'Public' + +# Will allow user self registration +# AUTH_USER_REGISTRATION = True + +# The default user self registration role +# AUTH_USER_REGISTRATION_ROLE = "Public" + +# When using OAuth Auth, uncomment to setup provider(s) info +# Google OAuth example: +# OAUTH_PROVIDERS = [{ +# 'name':'google', +# 'whitelist': ['@YOU_COMPANY_DOMAIN'], # optional +# 'token_key':'access_token', +# 'icon':'fa-google', +# 'remote_app': { +# 'base_url':'https://www.googleapis.com/oauth2/v2/', +# 'request_token_params':{ +# 'scope': 'email profile' +# }, +# 'access_token_url':'https://accounts.google.com/o/oauth2/token', +# 'authorize_url':'https://accounts.google.com/o/oauth2/auth', +# 'request_token_url': None, +# 'consumer_key': CONSUMER_KEY, +# 'consumer_secret': SECRET_KEY, +# } +# }] + +# When using LDAP Auth, setup the ldap server +# AUTH_LDAP_SERVER = "ldap://ldapserver.new" + +# When using OpenID Auth, uncomment to setup OpenID providers. +# example for OpenID authentication +# OPENID_PROVIDERS = [ +# { 'name': 'Yahoo', 'url': 'https://me.yahoo.com' }, +# { 'name': 'AOL', 'url': 'http://openid.aol.com/<username>' }, +# { 'name': 'Flickr', 'url': 'http://www.flickr.com/<username>' }, +# { 'name': 'MyOpenID', 'url': 'https://www.myopenid.com' }] http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/configuration.py ---------------------------------------------------------------------- diff --git a/airflow/configuration.py b/airflow/configuration.py index 30fa7fe..a14ca7d 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -422,6 +422,16 @@ log.info("Reading the config from %s", AIRFLOW_CONFIG) conf = AirflowConfigParser() conf.read(AIRFLOW_CONFIG) +if conf.getboolean('webserver', 'rbac'): + with open(os.path.join(_templates_dir, 'default_webserver_config.py')) as f: + DEFAULT_WEBSERVER_CONFIG = f.read() + + WEBSERVER_CONFIG = AIRFLOW_HOME + '/webserver_config.py' + + if not os.path.isfile(WEBSERVER_CONFIG): + log.info('Creating new FAB webserver config file in: %s', WEBSERVER_CONFIG) + with open(WEBSERVER_CONFIG, 'w') as f: + f.write(DEFAULT_WEBSERVER_CONFIG) def load_test_config(): """ http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/models.py ---------------------------------------------------------------------- diff --git a/airflow/models.py b/airflow/models.py index aa10ad5..baf101f 100755 --- a/airflow/models.py +++ b/airflow/models.py @@ -1047,26 +1047,43 @@ class TaskInstance(Base, LoggingMixin): def log_url(self): iso = quote(self.execution_date.isoformat()) BASE_URL = configuration.get('webserver', 'BASE_URL') - return BASE_URL + ( - "/admin/airflow/log" - "?dag_id={self.dag_id}" - "&task_id={self.task_id}" - "&execution_date={iso}" - ).format(**locals()) + if settings.RBAC: + return BASE_URL + ( + "/log/list/" + "?_flt_3_dag_id={self.dag_id}" + "&_flt_3_task_id={self.task_id}" + "&_flt_3_execution_date={iso}" + ).format(**locals()) + else: + return BASE_URL + ( + "/admin/airflow/log" + "?dag_id={self.dag_id}" + "&task_id={self.task_id}" + "&execution_date={iso}" + ).format(**locals()) @property def mark_success_url(self): iso = quote(self.execution_date.isoformat()) BASE_URL = configuration.get('webserver', 'BASE_URL') - return BASE_URL + ( - "/admin/airflow/action" - "?action=success" - "&task_id={self.task_id}" - "&dag_id={self.dag_id}" - "&execution_date={iso}" - "&upstream=false" - "&downstream=false" - ).format(**locals()) + if settings.RBAC: + return BASE_URL + ( + "/success" + "?task_id={self.task_id}" + "&dag_id={self.dag_id}" + "&execution_date={iso}" + "&upstream=false" + "&downstream=false" + ).format(**locals()) + else: + return BASE_URL + ( + "/admin/airflow/success" + "?task_id={self.task_id}" + "&dag_id={self.dag_id}" + "&execution_date={iso}" + "&upstream=false" + "&downstream=false" + ).format(**locals()) @provide_session def current_state(self, session=None): @@ -4197,19 +4214,20 @@ class Variable(Base, LoggingMixin): return '{} : {}'.format(self.key, self._val) def get_val(self): + log = LoggingMixin().log if self._val and self.is_encrypted: try: fernet = get_fernet() except: - raise AirflowException( - "Can't decrypt _val for key={}, FERNET_KEY configuration \ - missing".format(self.key)) + log.error("Can't decrypt _val for key={}, FERNET_KEY " + "configuration missing".format(self.key)) + return None try: return fernet.decrypt(bytes(self._val, 'utf-8')).decode() except: - raise AirflowException( - "Can't decrypt _val for key={}, invalid token or value" - .format(self.key)) + log.error("Can't decrypt _val for key={}, invalid token " + "or value".format(self.key)) + return None else: return self._val http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/settings.py ---------------------------------------------------------------------- diff --git a/airflow/settings.py b/airflow/settings.py index 929663b..06748ce 100644 --- a/airflow/settings.py +++ b/airflow/settings.py @@ -32,6 +32,7 @@ from airflow.utils.sqlalchemy import setup_event_handlers log = logging.getLogger(__name__) +RBAC = conf.getboolean('webserver', 'rbac') TIMEZONE = pendulum.timezone('UTC') try: @@ -84,7 +85,6 @@ ___ ___ | / _ / _ __/ _ / / /_/ /_ |/ |/ / _/_/ |_/_/ /_/ /_/ /_/ \____/____/|__/ """ -BASE_LOG_URL = '/admin/airflow/log' LOGGING_LEVEL = logging.INFO # the prefix to append to gunicorn worker processes after init http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/utils/db.py ---------------------------------------------------------------------- diff --git a/airflow/utils/db.py b/airflow/utils/db.py index 6c7f3c0..ed8aa4b 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -80,7 +80,7 @@ def merge_conn(conn, session=None): session.commit() -def initdb(): +def initdb(rbac): session = settings.Session() from airflow import models @@ -303,6 +303,11 @@ def initdb(): session.add(chart) session.commit() + if rbac: + from flask_appbuilder.security.sqla import models + from flask_appbuilder.models.sqla import Base + Base.metadata.create_all(settings.engine) + def upgradedb(): # alembic adds significant import time, so we import it lazily @@ -320,11 +325,12 @@ def upgradedb(): command.upgrade(config, 'heads') -def resetdb(): +def resetdb(rbac): ''' Clear out the database ''' from airflow import models + # alembic adds significant import time, so we import it lazily from alembic.migration import MigrationContext @@ -334,4 +340,11 @@ def resetdb(): mc = MigrationContext.configure(settings.engine) if mc._version.exists(settings.engine): mc._version.drop(settings.engine) - initdb() + + if rbac: + # drop rbac security tables + from flask_appbuilder.security.sqla import models + from flask_appbuilder.models.sqla import Base + Base.metadata.drop_all(settings.engine) + + initdb(rbac) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/www/views.py ---------------------------------------------------------------------- diff --git a/airflow/www/views.py b/airflow/www/views.py index 2f56175..83e567a 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -2303,9 +2303,10 @@ class VariableView(wwwutils.DataProfilingMixin, AirflowModelView): def hidden_field_formatter(view, context, model, name): if wwwutils.should_hide_value_for_key(model.key): return Markup('*' * 8) - try: - return getattr(model, name) - except AirflowException: + val = getattr(model, name) + if val: + return val + else: return Markup('<span class="label label-danger">Invalid</span>') form_columns = ( http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/www_rbac/__init__.py ---------------------------------------------------------------------- diff --git a/airflow/www_rbac/__init__.py b/airflow/www_rbac/__init__.py new file mode 100644 index 0000000..9d7677a --- /dev/null +++ b/airflow/www_rbac/__init__.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/www_rbac/api/__init__.py ---------------------------------------------------------------------- diff --git a/airflow/www_rbac/api/__init__.py b/airflow/www_rbac/api/__init__.py new file mode 100644 index 0000000..759b563 --- /dev/null +++ b/airflow/www_rbac/api/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/www_rbac/api/experimental/__init__.py ---------------------------------------------------------------------- diff --git a/airflow/www_rbac/api/experimental/__init__.py b/airflow/www_rbac/api/experimental/__init__.py new file mode 100644 index 0000000..759b563 --- /dev/null +++ b/airflow/www_rbac/api/experimental/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/www_rbac/api/experimental/endpoints.py ---------------------------------------------------------------------- diff --git a/airflow/www_rbac/api/experimental/endpoints.py b/airflow/www_rbac/api/experimental/endpoints.py new file mode 100644 index 0000000..31cd958 --- /dev/null +++ b/airflow/www_rbac/api/experimental/endpoints.py @@ -0,0 +1,231 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import airflow.api + +from airflow.api.common.experimental import pool as pool_api +from airflow.api.common.experimental import trigger_dag as trigger +from airflow.api.common.experimental.get_task import get_task +from airflow.api.common.experimental.get_task_instance import get_task_instance +from airflow.exceptions import AirflowException +from airflow.utils.log.logging_mixin import LoggingMixin +from airflow.utils import timezone +from airflow.www_rbac.app import csrf + +from flask import g, Blueprint, jsonify, request, url_for + +_log = LoggingMixin().log + +requires_authentication = airflow.api.api_auth.requires_authentication + +api_experimental = Blueprint('api_experimental', __name__) + + +@csrf.exempt +@api_experimental.route('/dags/<string:dag_id>/dag_runs', methods=['POST']) +@requires_authentication +def trigger_dag(dag_id): + """ + Trigger a new dag run for a Dag with an execution date of now unless + specified in the data. + """ + data = request.get_json(force=True) + + run_id = None + if 'run_id' in data: + run_id = data['run_id'] + + conf = None + if 'conf' in data: + conf = data['conf'] + + execution_date = None + if 'execution_date' in data and data['execution_date'] is not None: + execution_date = data['execution_date'] + + # Convert string datetime into actual datetime + try: + execution_date = timezone.parse(execution_date) + except ValueError: + error_message = ( + 'Given execution date, {}, could not be identified ' + 'as a date. Example date format: 2015-11-16T14:34:15+00:00' + .format(execution_date)) + _log.info(error_message) + response = jsonify({'error': error_message}) + response.status_code = 400 + + return response + + try: + dr = trigger.trigger_dag(dag_id, run_id, conf, execution_date) + except AirflowException as err: + _log.error(err) + response = jsonify(error="{}".format(err)) + response.status_code = 404 + return response + + if getattr(g, 'user', None): + _log.info("User {} created {}".format(g.user, dr)) + + response = jsonify(message="Created {}".format(dr)) + return response + + +@api_experimental.route('/test', methods=['GET']) +@requires_authentication +def test(): + return jsonify(status='OK') + + +@api_experimental.route('/dags/<string:dag_id>/tasks/<string:task_id>', methods=['GET']) +@requires_authentication +def task_info(dag_id, task_id): + """Returns a JSON with a task's public instance variables. """ + try: + info = get_task(dag_id, task_id) + except AirflowException as err: + _log.info(err) + response = jsonify(error="{}".format(err)) + response.status_code = 404 + return response + + # JSONify and return. + fields = {k: str(v) + for k, v in vars(info).items() + if not k.startswith('_')} + return jsonify(fields) + + +@api_experimental.route( + '/dags/<string:dag_id>/dag_runs/<string:execution_date>/tasks/<string:task_id>', + methods=['GET']) +@requires_authentication +def task_instance_info(dag_id, execution_date, task_id): + """ + Returns a JSON with a task instance's public instance variables. + The format for the exec_date is expected to be + "YYYY-mm-DDTHH:MM:SS", for example: "2016-11-16T11:34:15". This will + of course need to have been encoded for URL in the request. + """ + + # Convert string datetime into actual datetime + try: + execution_date = timezone.parse(execution_date) + except ValueError: + error_message = ( + 'Given execution date, {}, could not be identified ' + 'as a date. Example date format: 2015-11-16T14:34:15+00:00' + .format(execution_date)) + _log.info(error_message) + response = jsonify({'error': error_message}) + response.status_code = 400 + + return response + + try: + info = get_task_instance(dag_id, task_id, execution_date) + except AirflowException as err: + _log.info(err) + response = jsonify(error="{}".format(err)) + response.status_code = 404 + return response + + # JSONify and return. + fields = {k: str(v) + for k, v in vars(info).items() + if not k.startswith('_')} + return jsonify(fields) + + +@api_experimental.route('/latest_runs', methods=['GET']) +@requires_authentication +def latest_dag_runs(): + """Returns the latest DagRun for each DAG formatted for the UI. """ + from airflow.models import DagRun + dagruns = DagRun.get_latest_runs() + payload = [] + for dagrun in dagruns: + if dagrun.execution_date: + payload.append({ + 'dag_id': dagrun.dag_id, + 'execution_date': dagrun.execution_date.isoformat(), + 'start_date': ((dagrun.start_date or '') and + dagrun.start_date.isoformat()), + 'dag_run_url': url_for('Airflow.graph', dag_id=dagrun.dag_id, + execution_date=dagrun.execution_date) + }) + return jsonify(items=payload) # old flask versions dont support jsonifying arrays + + +@api_experimental.route('/pools/<string:name>', methods=['GET']) +@requires_authentication +def get_pool(name): + """Get pool by a given name.""" + try: + pool = pool_api.get_pool(name=name) + except AirflowException as e: + _log.error(e) + response = jsonify(error="{}".format(e)) + response.status_code = getattr(e, 'status', 500) + return response + else: + return jsonify(pool.to_json()) + + +@api_experimental.route('/pools', methods=['GET']) +@requires_authentication +def get_pools(): + """Get all pools.""" + try: + pools = pool_api.get_pools() + except AirflowException as e: + _log.error(e) + response = jsonify(error="{}".format(e)) + response.status_code = getattr(e, 'status', 500) + return response + else: + return jsonify([p.to_json() for p in pools]) + + +@csrf.exempt +@api_experimental.route('/pools', methods=['POST']) +@requires_authentication +def create_pool(): + """Create a pool.""" + params = request.get_json(force=True) + try: + pool = pool_api.create_pool(**params) + except AirflowException as e: + _log.error(e) + response = jsonify(error="{}".format(e)) + response.status_code = getattr(e, 'status', 500) + return response + else: + return jsonify(pool.to_json()) + + +@csrf.exempt +@api_experimental.route('/pools/<string:name>', methods=['DELETE']) +@requires_authentication +def delete_pool(name): + """Delete pool.""" + try: + pool = pool_api.delete_pool(name=name) + except AirflowException as e: + _log.error(e) + response = jsonify(error="{}".format(e)) + response.status_code = getattr(e, 'status', 500) + return response + else: + return jsonify(pool.to_json()) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/www_rbac/app.py ---------------------------------------------------------------------- diff --git a/airflow/www_rbac/app.py b/airflow/www_rbac/app.py new file mode 100644 index 0000000..9f4e142 --- /dev/null +++ b/airflow/www_rbac/app.py @@ -0,0 +1,174 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import socket +import six + +from flask import Flask +from flask_appbuilder import AppBuilder, SQLA +from flask_caching import Cache +from flask_wtf.csrf import CSRFProtect +from six.moves.urllib.parse import urlparse +from werkzeug.wsgi import DispatcherMiddleware + +from airflow import settings +from airflow import configuration as conf +from airflow.logging_config import configure_logging + + +app = None +appbuilder = None +csrf = CSRFProtect() + + +def create_app(config=None, testing=False, app_name="Airflow"): + global app, appbuilder + app = Flask(__name__) + app.secret_key = conf.get('webserver', 'SECRET_KEY') + + airflow_home_path = conf.get('core', 'AIRFLOW_HOME') + webserver_config_path = airflow_home_path + '/webserver_config.py' + app.config.from_pyfile(webserver_config_path, silent=True) + app.config['APP_NAME'] = app_name + app.config['TESTING'] = testing + + csrf.init_app(app) + + db = SQLA(app) + + from airflow import api + api.load_auth() + api.api_auth.init_app(app) + + cache = Cache(app=app, config={'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp'}) # noqa + + from airflow.www_rbac.blueprints import routes + app.register_blueprint(routes) + + configure_logging() + + with app.app_context(): + appbuilder = AppBuilder( + app, + db.session, + security_manager_class=app.config.get('SECURITY_MANAGER_CLASS'), + base_template='appbuilder/baselayout.html') + + def init_views(appbuilder): + from airflow.www_rbac import views + appbuilder.add_view_no_menu(views.Airflow()) + appbuilder.add_view_no_menu(views.DagModelView()) + appbuilder.add_view_no_menu(views.ConfigurationView()) + appbuilder.add_view_no_menu(views.VersionView()) + appbuilder.add_view(views.DagRunModelView, + "DAG Runs", + category="Browse", + category_icon="fa-globe") + appbuilder.add_view(views.JobModelView, + "Jobs", + category="Browse") + appbuilder.add_view(views.LogModelView, + "Logs", + category="Browse") + appbuilder.add_view(views.SlaMissModelView, + "SLA Misses", + category="Browse") + appbuilder.add_view(views.TaskInstanceModelView, + "Task Instances", + category="Browse") + appbuilder.add_link("Configurations", + href='/configuration', + category="Admin", + category_icon="fa-user") + appbuilder.add_view(views.ConnectionModelView, + "Connections", + category="Admin") + appbuilder.add_view(views.PoolModelView, + "Pools", + category="Admin") + appbuilder.add_view(views.VariableModelView, + "Variables", + category="Admin") + appbuilder.add_view(views.XComModelView, + "XComs", + category="Admin") + appbuilder.add_link("Documentation", + href='https://airflow.apache.org/', + category="Docs", + category_icon="fa-cube") + appbuilder.add_link("Github", + href='https://github.com/apache/incubator-airflow', + category="Docs") + appbuilder.add_link('Version', + href='/version', + category='About', + category_icon='fa-th') + + # Garbage collect old permissions/views after they have been modified. + # Otherwise, when the name of a view or menu is changed, the framework + # will add the new Views and Menus names to the backend, but will not + # delete the old ones. + appbuilder.security_cleanup() + + init_views(appbuilder) + + from airflow.www_rbac.security import init_roles + init_roles(appbuilder) + + from airflow.www_rbac.api.experimental import endpoints as e + # required for testing purposes otherwise the module retains + # a link to the default_auth + if app.config['TESTING']: + if six.PY2: + reload(e) # noqa + else: + import importlib + importlib.reload(e) + + app.register_blueprint(e.api_experimental, url_prefix='/api/experimental') + + @app.context_processor + def jinja_globals(): + return { + 'hostname': socket.getfqdn(), + } + + @app.teardown_appcontext + def shutdown_session(exception=None): + settings.Session.remove() + + return app, appbuilder + + +def root_app(env, resp): + resp(b'404 Not Found', [(b'Content-Type', b'text/plain')]) + return [b'Apache Airflow is not at this location'] + + +def cached_app(config=None, testing=False): + global app, appbuilder + if not app or not appbuilder: + base_url = urlparse(conf.get('webserver', 'base_url'))[2] + if not base_url or base_url == '/': + base_url = "" + + app, _ = create_app(config, testing) + app = DispatcherMiddleware(root_app, {base_url: app}) + return app + + +def cached_appbuilder(config=None, testing=False): + global appbuilder + cached_app(config, testing) + return appbuilder http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/www_rbac/blueprints.py ---------------------------------------------------------------------- diff --git a/airflow/www_rbac/blueprints.py b/airflow/www_rbac/blueprints.py new file mode 100644 index 0000000..43ae838 --- /dev/null +++ b/airflow/www_rbac/blueprints.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from flask import Markup, Blueprint, redirect +import markdown + +routes = Blueprint('routes', __name__) + + +@routes.route('/') +def index(): + return redirect('/home') + + +@routes.route('/health') +def health(): + """ We can add an array of tests here to check the server's health """ + content = Markup(markdown.markdown("The server is healthy!")) + return content http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/www_rbac/decorators.py ---------------------------------------------------------------------- diff --git a/airflow/www_rbac/decorators.py b/airflow/www_rbac/decorators.py new file mode 100644 index 0000000..22da021 --- /dev/null +++ b/airflow/www_rbac/decorators.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gzip +import functools +import pendulum +from io import BytesIO as IO +from flask import after_this_request, request, g +from airflow import models, settings + + +def action_logging(f): + ''' + Decorator to log user actions + ''' + @functools.wraps(f) + def wrapper(*args, **kwargs): + session = settings.Session() + if g.user.is_anonymous(): + user = 'anonymous' + else: + user = g.user.username + + log = models.Log( + event=f.__name__, + task_instance=None, + owner=user, + extra=str(list(request.args.items())), + task_id=request.args.get('task_id'), + dag_id=request.args.get('dag_id')) + + if 'execution_date' in request.args: + log.execution_date = pendulum.parse( + request.args.get('execution_date')) + + session.add(log) + session.commit() + + return f(*args, **kwargs) + + return wrapper + + +def gzipped(f): + ''' + Decorator to make a view compressed + ''' + @functools.wraps(f) + def view_func(*args, **kwargs): + @after_this_request + def zipper(response): + accept_encoding = request.headers.get('Accept-Encoding', '') + + if 'gzip' not in accept_encoding.lower(): + return response + + response.direct_passthrough = False + + if (response.status_code < 200 or response.status_code >= 300 or + 'Content-Encoding' in response.headers): + return response + gzip_buffer = IO() + gzip_file = gzip.GzipFile(mode='wb', + fileobj=gzip_buffer) + gzip_file.write(response.data) + gzip_file.close() + + response.data = gzip_buffer.getvalue() + response.headers['Content-Encoding'] = 'gzip' + response.headers['Vary'] = 'Accept-Encoding' + response.headers['Content-Length'] = len(response.data) + + return response + + return f(*args, **kwargs) + + return view_func http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/www_rbac/forms.py ---------------------------------------------------------------------- diff --git a/airflow/www_rbac/forms.py b/airflow/www_rbac/forms.py new file mode 100644 index 0000000..2faef3b --- /dev/null +++ b/airflow/www_rbac/forms.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from airflow import models +from airflow.utils import timezone + +from flask_appbuilder.forms import DynamicForm +from flask_appbuilder.fieldwidgets import (BS3TextFieldWidget, BS3TextAreaFieldWidget, + BS3PasswordFieldWidget, Select2Widget, + DateTimePickerWidget) +from flask_babel import lazy_gettext +from flask_wtf import Form + +from wtforms import validators +from wtforms.fields import (IntegerField, SelectField, TextAreaField, PasswordField, + StringField, DateTimeField, BooleanField) + + +class DateTimeForm(Form): + # Date filter form needed for gantt and graph view + execution_date = DateTimeField( + "Execution date", widget=DateTimePickerWidget()) + + +class DateTimeWithNumRunsForm(Form): + # Date time and number of runs form for tree view, task duration + # and landing times + base_date = DateTimeField( + "Anchor date", widget=DateTimePickerWidget(), default=timezone.utcnow()) + num_runs = SelectField("Number of runs", default=25, choices=( + (5, "5"), + (25, "25"), + (50, "50"), + (100, "100"), + (365, "365"), + )) + + +class DagRunForm(DynamicForm): + dag_id = StringField( + lazy_gettext('Dag Id'), + validators=[validators.DataRequired()], + widget=BS3TextFieldWidget()) + start_date = DateTimeField( + lazy_gettext('Start Date'), + widget=DateTimePickerWidget()) + end_date = DateTimeField( + lazy_gettext('End Date'), + widget=DateTimePickerWidget()) + run_id = StringField( + lazy_gettext('Run Id'), + widget=BS3TextFieldWidget()) + state = SelectField( + lazy_gettext('State'), + choices=(('success', 'success'), ('running', 'running'), ('failed', 'failed'),), + widget=Select2Widget()) + execution_date = DateTimeField( + lazy_gettext('Execution Date'), + widget=DateTimePickerWidget()) + external_trigger = BooleanField( + lazy_gettext('External Trigger')) + + +class ConnectionForm(DynamicForm): + conn_id = StringField( + lazy_gettext('Conn Id'), + widget=BS3TextFieldWidget()) + conn_type = SelectField( + lazy_gettext('Conn Type'), + choices=(models.Connection._types), + widget=Select2Widget()) + host = StringField( + lazy_gettext('Host'), + widget=BS3TextFieldWidget()) + schema = StringField( + lazy_gettext('Schema'), + widget=BS3TextFieldWidget()) + login = StringField( + lazy_gettext('Login'), + widget=BS3TextFieldWidget()) + password = PasswordField( + lazy_gettext('Password'), + widget=BS3PasswordFieldWidget()) + port = IntegerField( + lazy_gettext('Port'), + validators=[validators.Optional()], + widget=BS3TextFieldWidget()) + extra = TextAreaField( + lazy_gettext('Extra'), + widget=BS3TextAreaFieldWidget()) + + # Used to customized the form, the forms elements get rendered + # and results are stored in the extra field as json. All of these + # need to be prefixed with extra__ and then the conn_type ___ as in + # extra__{conn_type}__name. You can also hide form elements and rename + # others from the connection_form.js file + extra__jdbc__drv_path = StringField( + lazy_gettext('Driver Path'), + widget=BS3TextFieldWidget()) + extra__jdbc__drv_clsname = StringField( + lazy_gettext('Driver Class'), + widget=BS3TextFieldWidget()) + extra__google_cloud_platform__project = StringField( + lazy_gettext('Project Id'), + widget=BS3TextFieldWidget()) + extra__google_cloud_platform__key_path = StringField( + lazy_gettext('Keyfile Path'), + widget=BS3TextFieldWidget()) + extra__google_cloud_platform__keyfile_dict = PasswordField( + lazy_gettext('Keyfile JSON'), + widget=BS3PasswordFieldWidget()) + extra__google_cloud_platform__scope = StringField( + lazy_gettext('Scopes (comma separated)'), + widget=BS3TextFieldWidget()) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/www_rbac/security.py ---------------------------------------------------------------------- diff --git a/airflow/www_rbac/security.py b/airflow/www_rbac/security.py new file mode 100644 index 0000000..49806c1 --- /dev/null +++ b/airflow/www_rbac/security.py @@ -0,0 +1,174 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from flask_appbuilder.security.sqla import models as sqla_models + +########################################################################### +# VIEW MENUS +########################################################################### +viewer_vms = [ + 'Airflow', + 'DagModelView', + 'Browse', + 'DAG Runs', + 'DagRunModelView', + 'Task Instances', + 'TaskInstanceModelView', + 'SLA Misses', + 'SlaMissModelView', + 'Jobs', + 'JobModelView', + 'Logs', + 'LogModelView', + 'Docs', + 'Documentation', + 'Github', + 'About', + 'Version', + 'VersionView', +] + +user_vms = viewer_vms + +op_vms = [ + 'Admin', + 'Configurations', + 'ConfigurationView', + 'Connections', + 'ConnectionModelView', + 'Pools', + 'PoolModelView', + 'Variables', + 'VariableModelView', + 'XComs', + 'XComModelView', +] + +########################################################################### +# PERMISSIONS +########################################################################### + +viewer_perms = [ + 'menu_access', + 'can_index', + 'can_list', + 'can_show', + 'can_chart', + 'can_dag_stats', + 'can_dag_details', + 'can_task_stats', + 'can_code', + 'can_log', + 'can_tries', + 'can_graph', + 'can_tree', + 'can_task', + 'can_task_instances', + 'can_xcom', + 'can_gantt', + 'can_landing_times', + 'can_duration', + 'can_blocked', + 'can_rendered', + 'can_pickle_info', + 'can_version', +] + +user_perms = [ + 'can_dagrun_clear', + 'can_run', + 'can_trigger', + 'can_add', + 'can_edit', + 'can_delete', + 'can_paused', + 'can_refresh', + 'can_success', + 'muldelete', + 'set_failed', + 'set_running', + 'set_success', + 'clear', +] + +op_perms = [ + 'can_conf', + 'can_varimport', +] + +########################################################################### +# DEFAULT ROLE CONFIGURATIONS +########################################################################### + +ROLE_CONFIGS = [ + { + 'role': 'Viewer', + 'perms': viewer_perms, + 'vms': viewer_vms, + }, + { + 'role': 'User', + 'perms': viewer_perms + user_perms, + 'vms': viewer_vms + user_vms, + }, + { + 'role': 'Op', + 'perms': viewer_perms + user_perms + op_perms, + 'vms': viewer_vms + user_vms + op_vms, + }, +] + + +def init_role(sm, role_name, role_vms, role_perms): + sm_session = sm.get_session + pvms = sm_session.query(sqla_models.PermissionView).all() + pvms = [p for p in pvms if p.permission and p.view_menu] + + valid_perms = [p.permission.name for p in pvms] + valid_vms = [p.view_menu.name for p in pvms] + invalid_perms = [p for p in role_perms if p not in valid_perms] + if invalid_perms: + raise Exception('The following permissions are not valid: {}' + .format(invalid_perms)) + invalid_vms = [v for v in role_vms if v not in valid_vms] + if invalid_vms: + raise Exception('The following view menus are not valid: {}' + .format(invalid_vms)) + + role = sm.add_role(role_name) + role_pvms = [] + for pvm in pvms: + if pvm.view_menu.name in role_vms and pvm.permission.name in role_perms: + role_pvms.append(pvm) + role_pvms = list(set(role_pvms)) + role.permissions = role_pvms + sm_session.merge(role) + sm_session.commit() + + +def init_roles(appbuilder): + for config in ROLE_CONFIGS: + name = config['role'] + vms = config['vms'] + perms = config['perms'] + init_role(appbuilder.sm, name, vms, perms) + + +def is_view_only(user, appbuilder): + if user.is_anonymous(): + anonymous_role = appbuilder.sm.auth_role_public + return anonymous_role == 'Viewer' + + user_roles = user.roles + return len(user_roles) == 1 and user_roles[0].name == 'Viewer' http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/05e1861e/airflow/www_rbac/static/airflow.gif ---------------------------------------------------------------------- diff --git a/airflow/www_rbac/static/airflow.gif b/airflow/www_rbac/static/airflow.gif new file mode 100644 index 0000000..1889b86 Binary files /dev/null and b/airflow/www_rbac/static/airflow.gif differ