Repository: incubator-airflow Updated Branches: refs/heads/master 2f930b671 -> 45c0c5479
[AIRFLOW-2429] Fix hook, macros folder flake8 error Closes #3420 from feng-tao/flake8_p4 Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/45c0c547 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/45c0c547 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/45c0c547 Branch: refs/heads/master Commit: 45c0c54792a2517d6694c90489e93aedef424195 Parents: 2f930b6 Author: Tao feng <tf...@lyft.com> Authored: Mon May 28 16:23:44 2018 +0200 Committer: Fokko Driesprong <fokkodriespr...@godatadriven.com> Committed: Mon May 28 16:23:44 2018 +0200 ---------------------------------------------------------------------- airflow/hooks/S3_hook.py | 23 ++++++++++--------- airflow/hooks/__init__.py | 8 +++---- airflow/hooks/base_hook.py | 4 ++-- airflow/hooks/dbapi_hook.py | 6 ++--- airflow/hooks/docker_hook.py | 15 +++++++------ airflow/hooks/hdfs_hook.py | 11 ++++----- airflow/hooks/hive_hooks.py | 15 ++++++------- airflow/hooks/jdbc_hook.py | 4 ++-- airflow/hooks/mssql_hook.py | 4 ++-- airflow/hooks/mysql_hook.py | 10 ++++----- airflow/hooks/oracle_hook.py | 46 ++++++++++++++++++++++++-------------- airflow/hooks/pig_hook.py | 4 ++-- airflow/hooks/presto_hook.py | 11 ++++----- airflow/hooks/slack_hook.py | 7 +++--- airflow/hooks/sqlite_hook.py | 4 ++-- airflow/hooks/webhdfs_hook.py | 18 ++++++++------- airflow/hooks/zendesk_hook.py | 6 ++--- airflow/macros/__init__.py | 13 ++++++----- 18 files changed, 114 insertions(+), 95 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/45c0c547/airflow/hooks/S3_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/S3_hook.py b/airflow/hooks/S3_hook.py index edde6ea..3d72275 100644 --- a/airflow/hooks/S3_hook.py +++ b/airflow/hooks/S3_hook.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -25,6 +25,7 @@ from urllib.parse import urlparse import re import fnmatch + class S3Hook(AwsHook): """ Interact with AWS S3, using the boto3 library. @@ -276,16 +277,16 @@ class S3Hook(AwsHook): if not replace and self.check_for_key(key, bucket_name): raise ValueError("The key {key} already exists.".format(key=key)) - extra_args={} + extra_args = {} if encrypt: extra_args['ServerSideEncryption'] = "AES256" client = self.get_conn() client.upload_file(filename, bucket_name, key, ExtraArgs=extra_args) - def load_string(self, + def load_string(self, string_data, - key, + key, bucket_name=None, replace=False, encrypt=False, @@ -294,7 +295,7 @@ class S3Hook(AwsHook): Loads a string to S3 This is provided as a convenience to drop a string in S3. It uses the - boto infrastructure to ship a file to s3. + boto infrastructure to ship a file to s3. :param string_data: string to set as content for the key. :type string_data: str @@ -342,15 +343,15 @@ class S3Hook(AwsHook): """ if not bucket_name: (bucket_name, key) = self.parse_s3_url(key) - + if not replace and self.check_for_key(key, bucket_name): raise ValueError("The key {key} already exists.".format(key=key)) - - extra_args={} + + extra_args = {} if encrypt: extra_args['ServerSideEncryption'] = "AES256" - + filelike_buffer = BytesIO(bytes_data) - + client = self.get_conn() client.upload_fileobj(filelike_buffer, bucket_name, key, ExtraArgs=extra_args) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/45c0c547/airflow/hooks/__init__.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/__init__.py b/airflow/hooks/__init__.py index d863b56..1f1c40f 100644 --- a/airflow/hooks/__init__.py +++ b/airflow/hooks/__init__.py @@ -7,18 +7,18 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# +import os as _os import sys @@ -66,7 +66,7 @@ _hooks = { 'slack_hook': ['SlackHook'], } -import os as _os + if not _os.environ.get('AIRFLOW_USE_NEW_IMPORTS', False): from airflow.utils.helpers import AirflowImporter airflow_importer = AirflowImporter(sys.modules[__name__], _hooks) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/45c0c547/airflow/hooks/base_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/base_hook.py b/airflow/hooks/base_hook.py index 16b19e0..103fa62 100644 --- a/airflow/hooks/base_hook.py +++ b/airflow/hooks/base_hook.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/45c0c547/airflow/hooks/dbapi_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/dbapi_hook.py b/airflow/hooks/dbapi_hook.py index 70439e8..30a9b32 100644 --- a/airflow/hooks/dbapi_hook.py +++ b/airflow/hooks/dbapi_hook.py @@ -218,10 +218,10 @@ class DbApiHook(BaseHook): with closing(conn.cursor()) as cur: for i, row in enumerate(rows, 1): - l = [] + lst = [] for cell in row: - l.append(self._serialize_cell(cell, conn)) - values = tuple(l) + lst.append(self._serialize_cell(cell, conn)) + values = tuple(lst) placeholders = ["%s", ] * len(values) sql = "INSERT INTO {0} {1} VALUES ({2})".format( table, http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/45c0c547/airflow/hooks/docker_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/docker_hook.py b/airflow/hooks/docker_hook.py index 5375c72..d528af6 100644 --- a/airflow/hooks/docker_hook.py +++ b/airflow/hooks/docker_hook.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -24,6 +24,7 @@ from airflow.exceptions import AirflowException from airflow.hooks.base_hook import BaseHook from airflow.utils.log.logging_mixin import LoggingMixin + class DockerHook(BaseHook, LoggingMixin): """ Interact with a private Docker registry. @@ -33,11 +34,11 @@ class DockerHook(BaseHook, LoggingMixin): :type docker_conn_id: str """ def __init__(self, - docker_conn_id='docker_default', - base_url=None, - version=None, - tls=None - ): + docker_conn_id='docker_default', + base_url=None, + version=None, + tls=None + ): if not base_url: raise AirflowException('No Docker base URL provided') if not version: http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/45c0c547/airflow/hooks/hdfs_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/hdfs_hook.py b/airflow/hooks/hdfs_hook.py index a87fa94..3c9136b 100644 --- a/airflow/hooks/hdfs_hook.py +++ b/airflow/hooks/hdfs_hook.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -18,16 +18,17 @@ # under the License. from six import PY2 -from airflow.hooks.base_hook import BaseHook + from airflow import configuration +from airflow.exceptions import AirflowException +from airflow.hooks.base_hook import BaseHook + snakebite_imported = False if PY2: from snakebite.client import Client, HAClient, Namenode, AutoConfigClient snakebite_imported = True -from airflow.exceptions import AirflowException - class HDFSHookException(AirflowException): pass http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/45c0c547/airflow/hooks/hive_hooks.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/hive_hooks.py b/airflow/hooks/hive_hooks.py index e2e3111..7f0f068 100644 --- a/airflow/hooks/hive_hooks.py +++ b/airflow/hooks/hive_hooks.py @@ -16,14 +16,12 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# from __future__ import print_function, unicode_literals from six.moves import zip from past.builtins import basestring, unicode import unicodecsv as csv -import itertools import re import six import subprocess @@ -273,9 +271,9 @@ class HiveCliHook(BaseHook): self.log.info(message) error_loc = re.search('(\d+):(\d+)', message) if error_loc and error_loc.group(1).isdigit(): - l = int(error_loc.group(1)) - begin = max(l-2, 0) - end = min(l+3, len(query.split('\n'))) + lst = int(error_loc.group(1)) + begin = max(lst - 2, 0) + end = min(lst + 3, len(query.split('\n'))) context = '\n'.join(query.split('\n')[begin:end]) self.log.info("Context :\n %s", context) else: @@ -725,9 +723,9 @@ class HiveMetastoreHook(BaseHook): False """ try: - t = self.get_table(table_name, db) + self.get_table(table_name, db) return True - except Exception as e: + except Exception: return False @@ -752,7 +750,8 @@ class HiveServer2Hook(BaseHook): # impyla uses GSSAPI instead of KERBEROS as a auth_mechanism identifier if auth_mechanism == 'KERBEROS': self.log.warning( - "Detected deprecated 'KERBEROS' for authMechanism for %s. Please use 'GSSAPI' instead", + "Detected deprecated 'KERBEROS' for " + "authMechanism for %s. Please use 'GSSAPI' instead", self.hiveserver2_conn_id ) auth_mechanism = 'GSSAPI' http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/45c0c547/airflow/hooks/jdbc_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/jdbc_hook.py b/airflow/hooks/jdbc_hook.py index 8f0cd67..25d1c37 100644 --- a/airflow/hooks/jdbc_hook.py +++ b/airflow/hooks/jdbc_hook.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/45c0c547/airflow/hooks/mssql_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/mssql_hook.py b/airflow/hooks/mssql_hook.py index 88f8eee..7ba3489 100644 --- a/airflow/hooks/mssql_hook.py +++ b/airflow/hooks/mssql_hook.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/45c0c547/airflow/hooks/mysql_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/mysql_hook.py b/airflow/hooks/mysql_hook.py index f52b60c..bd41733 100644 --- a/airflow/hooks/mysql_hook.py +++ b/airflow/hooks/mysql_hook.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -69,7 +69,7 @@ class MySqlHook(DbApiHook): conn_config["cursorclass"] = MySQLdb.cursors.DictCursor elif (conn.extra_dejson["cursor"]).lower() == 'ssdictcursor': conn_config["cursorclass"] = MySQLdb.cursors.SSDictCursor - local_infile = conn.extra_dejson.get('local_infile',False) + local_infile = conn.extra_dejson.get('local_infile', False) if conn.extra_dejson.get('ssl', False): conn_config['ssl'] = conn.extra_dejson['ssl'] if local_infile: @@ -104,8 +104,8 @@ class MySqlHook(DbApiHook): @staticmethod def _serialize_cell(cell, conn): """ - MySQLdb converts an argument to a literal when passing those seperately to execute. - Hence, this method does nothing. + MySQLdb converts an argument to a literal + when passing those seperately to execute. Hence, this method does nothing. :param cell: The cell to insert into the table :type cell: object http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/45c0c547/airflow/hooks/oracle_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/oracle_hook.py b/airflow/hooks/oracle_hook.py index 2b98e5c..39e447d 100644 --- a/airflow/hooks/oracle_hook.py +++ b/airflow/hooks/oracle_hook.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -37,12 +37,15 @@ class OracleHook(DbApiHook): def get_conn(self): """ Returns a oracle connection object - Optional parameters for using a custom DSN connection (instead of using a server alias from tnsnames.ora) - The dsn (data source name) is the TNS entry (from the Oracle names server or tnsnames.ora file) + Optional parameters for using a custom DSN connection + (instead of using a server alias from tnsnames.ora) + The dsn (data source name) is the TNS entry + (from the Oracle names server or tnsnames.ora file) or is a string like the one returned from makedsn(). :param dsn: the host address for the Oracle server - :param service_name: the db_unique_name of the database that you are connecting to (CONNECT_DATA part of TNS) + :param service_name: the db_unique_name of the database + that you are connecting to (CONNECT_DATA part of TNS) You can set these parameters in the extra fields of your connection as in ``{ "dsn":"some.host.address" , "service_name":"some.service.name" }`` """ @@ -72,7 +75,8 @@ class OracleHook(DbApiHook): the whole set of inserts is treated as one transaction Changes from standard DbApiHook implementation: - Oracle SQL queries in cx_Oracle can not be terminated with a semicolon (';') - - Replace NaN values with NULL using numpy.nan_to_num (not using is_nan() because of input types error for strings) + - Replace NaN values with NULL using numpy.nan_to_num (not using is_nan() + because of input types error for strings) - Coerce datetime cells to Oracle DATETIME format during insert """ if target_fields: @@ -88,22 +92,28 @@ class OracleHook(DbApiHook): i = 0 for row in rows: i += 1 - l = [] + lst = [] for cell in row: if isinstance(cell, basestring): - l.append("'" + str(cell).replace("'", "''") + "'") + lst.append("'" + str(cell).replace("'", "''") + "'") elif cell is None: - l.append('NULL') - elif type(cell) == float and numpy.isnan(cell): # coerce numpy NaN to NULL - l.append('NULL') + lst.append('NULL') + elif type(cell) == float and \ + numpy.isnan(cell): # coerce numpy NaN to NULL + lst.append('NULL') elif isinstance(cell, numpy.datetime64): - l.append("'" + str(cell) + "'") + lst.append("'" + str(cell) + "'") elif isinstance(cell, datetime): - l.append("to_date('" + cell.strftime('%Y-%m-%d %H:%M:%S') + "','YYYY-MM-DD HH24:MI:SS')") + lst.append("to_date('" + + cell.strftime('%Y-%m-%d %H:%M:%S') + + "','YYYY-MM-DD HH24:MI:SS')") else: - l.append(str(cell)) - values = tuple(l) - sql = 'INSERT /*+ APPEND */ INTO {0} {1} VALUES ({2})'.format(table, target_fields, ','.join(values)) + lst.append(str(cell)) + values = tuple(lst) + sql = 'INSERT /*+ APPEND */ ' \ + 'INTO {0} {1} VALUES ({2})'.format(table, + target_fields, + ','.join(values)) cur.execute(sql) if i % commit_every == 0: conn.commit() @@ -114,7 +124,9 @@ class OracleHook(DbApiHook): self.log.info('Done loading. Loaded a total of {i} rows'.format(**locals())) def bulk_insert_rows(self, table, rows, target_fields=None, commit_every=5000): - """A performant bulk insert for cx_Oracle that uses prepared statements via `executemany()`. + """ + A performant bulk insert for cx_Oracle + that uses prepared statements via `executemany()`. For best performance, pass in `rows` as an iterator. """ conn = self.get_conn() http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/45c0c547/airflow/hooks/pig_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/pig_hook.py b/airflow/hooks/pig_hook.py index 5c73dab..a3836b1 100644 --- a/airflow/hooks/pig_hook.py +++ b/airflow/hooks/pig_hook.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/45c0c547/airflow/hooks/presto_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/presto_hook.py b/airflow/hooks/presto_hook.py index 8920448..d6b5293 100644 --- a/airflow/hooks/presto_hook.py +++ b/airflow/hooks/presto_hook.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -24,6 +24,7 @@ from pyhive.exc import DatabaseError from airflow.hooks.dbapi_hook import DbApiHook + class PrestoException(Exception): pass @@ -59,9 +60,9 @@ class PrestoHook(DbApiHook): """ Parses some DatabaseError to provide a better error message """ - if (hasattr(e, 'message') - and 'errorName' in e.message - and 'message' in e.message): + if (hasattr(e, 'message') and + 'errorName' in e.message and + 'message' in e.message): return ('{name}: {message}'.format( name=e.message['errorName'], message=e.message['message'])) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/45c0c547/airflow/hooks/slack_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/slack_hook.py b/airflow/hooks/slack_hook.py index 05f2016..17006fc 100644 --- a/airflow/hooks/slack_hook.py +++ b/airflow/hooks/slack_hook.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -50,7 +50,8 @@ class SlackHook(BaseHook): raise AirflowException('Missing token(password) in Slack connection') return conn.password else: - raise AirflowException('Cannot get token: No valid Slack token nor slack_conn_id supplied.') + raise AirflowException('Cannot get token: ' + 'No valid Slack token nor slack_conn_id supplied.') def call(self, method, api_params): sc = SlackClient(self.token) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/45c0c547/airflow/hooks/sqlite_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/sqlite_hook.py b/airflow/hooks/sqlite_hook.py index 7f6c655..997a923 100644 --- a/airflow/hooks/sqlite_hook.py +++ b/airflow/hooks/sqlite_hook.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/45c0c547/airflow/hooks/webhdfs_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/webhdfs_hook.py b/airflow/hooks/webhdfs_hook.py index 6445e2b..c4dbe8b 100644 --- a/airflow/hooks/webhdfs_hook.py +++ b/airflow/hooks/webhdfs_hook.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -17,13 +17,14 @@ # specific language governing permissions and limitations # under the License. -from airflow.hooks.base_hook import BaseHook -from airflow import configuration - from hdfs import InsecureClient, HdfsError +from airflow import configuration +from airflow.exceptions import AirflowException +from airflow.hooks.base_hook import BaseHook from airflow.utils.log.logging_mixin import LoggingMixin + _kerberos_security_mode = configuration.conf.get("core", "security") == "kerberos" if _kerberos_security_mode: try: @@ -32,7 +33,6 @@ if _kerberos_security_mode: log = LoggingMixin().log log.error("Could not load the Kerberos extension for the WebHDFSHook.") raise -from airflow.exceptions import AirflowException class AirflowWebHDFSHookException(AirflowException): @@ -66,10 +66,12 @@ class WebHDFSHook(BaseHook): return client except HdfsError as e: self.log.debug( - "Read operation on namenode {nn.host} failed with error: {e}".format(**locals()) + "Read operation on namenode {nn.host} " + "failed with error: {e}".format(**locals()) ) nn_hosts = [c.host for c in nn_connections] - no_nn_error = "Read operations failed on the namenodes below:\n{}".format("\n".join(nn_hosts)) + no_nn_error = "Read operations failed " \ + "on the namenodes below:\n{}".format("\n".join(nn_hosts)) raise AirflowWebHDFSHookException(no_nn_error) def check_for_path(self, hdfs_path): http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/45c0c547/airflow/hooks/zendesk_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/zendesk_hook.py b/airflow/hooks/zendesk_hook.py index a8aee23..3cf8353 100644 --- a/airflow/hooks/zendesk_hook.py +++ b/airflow/hooks/zendesk_hook.py @@ -7,16 +7,16 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# + import time from zdesk import Zendesk, RateLimitError, ZendeskError from airflow.hooks.base_hook import BaseHook http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/45c0c547/airflow/macros/__init__.py ---------------------------------------------------------------------- diff --git a/airflow/macros/__init__.py b/airflow/macros/__init__.py index 80a4e4c..3c33acd 100644 --- a/airflow/macros/__init__.py +++ b/airflow/macros/__init__.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -19,10 +19,11 @@ from __future__ import absolute_import from datetime import datetime, timedelta -import dateutil -import time -from . import hive -import uuid +import dateutil # noqa +from random import random # noqa +import time # noqa +from . import hive # noqa +import uuid # noqa def ds_add(ds, days):
