This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new a7310f9c91 Refactor regex in providers (#33898)
a7310f9c91 is described below
commit a7310f9c9127cf87a71e0bfa141c066d6a0bc82b
Author: Miroslav Šedivý <[email protected]>
AuthorDate: Tue Sep 5 14:39:43 2023 +0000
Refactor regex in providers (#33898)
* Refactor regex in providers
* Satisfy Mypy's optional check
---------
Co-authored-by: Tzu-ping Chung <[email protected]>
---
airflow/providers/amazon/aws/hooks/chime.py | 2 +-
airflow/providers/amazon/aws/hooks/s3.py | 6 ++---
airflow/providers/amazon/aws/hooks/sagemaker.py | 3 +--
airflow/providers/amazon/aws/sensors/s3.py | 2 +-
airflow/providers/amazon/aws/utils/__init__.py | 6 +++--
airflow/providers/apache/hive/hooks/hive.py | 2 +-
airflow/providers/apache/livy/hooks/livy.py | 4 ++--
.../providers/apache/spark/hooks/spark_submit.py | 12 +++++-----
airflow/providers/common/sql/operators/sql.py | 2 +-
airflow/providers/discord/hooks/discord_webhook.py | 2 +-
airflow/providers/ftp/sensors/ftp.py | 5 ++--
airflow/providers/google/cloud/hooks/cloud_sql.py | 3 +--
airflow/providers/google/cloud/hooks/dataflow.py | 2 +-
.../providers/google/cloud/operators/dataproc.py | 28 +++++++++++-----------
.../google/cloud/secrets/secret_manager.py | 4 ++--
.../google/cloud/utils/mlengine_operator_utils.py | 2 +-
.../providers/microsoft/azure/secrets/key_vault.py | 4 ++--
17 files changed, 45 insertions(+), 44 deletions(-)
diff --git a/airflow/providers/amazon/aws/hooks/chime.py
b/airflow/providers/amazon/aws/hooks/chime.py
index f12defdeb8..a209a3a18a 100644
--- a/airflow/providers/amazon/aws/hooks/chime.py
+++ b/airflow/providers/amazon/aws/hooks/chime.py
@@ -70,7 +70,7 @@ class ChimeWebhookHook(HttpHook):
url = conn.schema + "://" + conn.host
endpoint = url + token
# Check to make sure the endpoint matches what Chime expects
- if not re.match(r"^[a-zA-Z0-9_-]+\?token=[a-zA-Z0-9_-]+$", token):
+ if not re.fullmatch(r"[a-zA-Z0-9_-]+\?token=[a-zA-Z0-9_-]+", token):
raise AirflowException(
"Expected Chime webhook token in the form of
'{webhook.id}?token={webhook.token}'."
)
diff --git a/airflow/providers/amazon/aws/hooks/s3.py
b/airflow/providers/amazon/aws/hooks/s3.py
index 5a19fc8c85..c3eb5fc52f 100644
--- a/airflow/providers/amazon/aws/hooks/s3.py
+++ b/airflow/providers/amazon/aws/hooks/s3.py
@@ -468,7 +468,7 @@ class S3Hook(AwsBaseHook):
:param bucket_name: the name of the bucket
:param key: the path to the key
"""
- prefix = re.split(r"[\[\*\?]", key, 1)[0]
+ prefix = re.split(r"[\[*?]", key, 1)[0]
delimiter = ""
paginator = client.get_paginator("list_objects_v2")
response = paginator.paginate(Bucket=bucket_name, Prefix=prefix,
Delimiter=delimiter)
@@ -570,7 +570,7 @@ class S3Hook(AwsBaseHook):
for key in bucket_keys:
prefix = key
if wildcard_match:
- prefix = re.split(r"[\[\*\?]", key, 1)[0]
+ prefix = re.split(r"[\[*?]", key, 1)[0]
paginator = client.get_paginator("list_objects_v2")
response = paginator.paginate(Bucket=bucket, Prefix=prefix,
Delimiter=delimiter)
@@ -1015,7 +1015,7 @@ class S3Hook(AwsBaseHook):
:param delimiter: the delimiter marks key hierarchy
:return: the key object from the bucket or None if none has been found.
"""
- prefix = re.split(r"[\[\*\?]", wildcard_key, 1)[0]
+ prefix = re.split(r"[\[*?]", wildcard_key, 1)[0]
key_list = self.list_keys(bucket_name, prefix=prefix,
delimiter=delimiter)
key_matches = [k for k in key_list if fnmatch.fnmatch(k, wildcard_key)]
if key_matches:
diff --git a/airflow/providers/amazon/aws/hooks/sagemaker.py
b/airflow/providers/amazon/aws/hooks/sagemaker.py
index 0dfcbca0f3..2911ddd86f 100644
--- a/airflow/providers/amazon/aws/hooks/sagemaker.py
+++ b/airflow/providers/amazon/aws/hooks/sagemaker.py
@@ -979,8 +979,7 @@ class SageMakerHook(AwsBaseHook):
found_name: str,
job_name_suffix: str | None = None,
) -> bool:
- pattern = re.compile(f"^{processing_job_name}({job_name_suffix})?$")
- return pattern.fullmatch(found_name) is not None
+ return re.fullmatch(f"{processing_job_name}({job_name_suffix})?",
found_name) is not None
def count_processing_jobs_by_name(
self,
diff --git a/airflow/providers/amazon/aws/sensors/s3.py
b/airflow/providers/amazon/aws/sensors/s3.py
index 0afa2cf704..d517709e27 100644
--- a/airflow/providers/amazon/aws/sensors/s3.py
+++ b/airflow/providers/amazon/aws/sensors/s3.py
@@ -113,7 +113,7 @@ class S3KeySensor(BaseSensorOperator):
}]
"""
if self.wildcard_match:
- prefix = re.split(r"[\[\*\?]", key, 1)[0]
+ prefix = re.split(r"[\[*?]", key, 1)[0]
keys = self.hook.get_file_metadata(prefix, bucket_name)
key_matches = [k for k in keys if fnmatch.fnmatch(k["Key"], key)]
if not key_matches:
diff --git a/airflow/providers/amazon/aws/utils/__init__.py
b/airflow/providers/amazon/aws/utils/__init__.py
index 0a26f3bba5..8495e1587a 100644
--- a/airflow/providers/amazon/aws/utils/__init__.py
+++ b/airflow/providers/amazon/aws/utils/__init__.py
@@ -66,8 +66,10 @@ def datetime_to_epoch_us(date_time: datetime) -> int:
def get_airflow_version() -> tuple[int, ...]:
- val = re.sub(r"(\d+\.\d+\.\d+).*", lambda x: x.group(1), version)
- return tuple(int(x) for x in val.split("."))
+ match = re.match(r"(\d+)\.(\d+)\.(\d+)", version)
+ if match is None: # Not theoratically possible.
+ raise RuntimeError(f"Broken Airflow version: {version}")
+ return tuple(int(x) for x in match.groups())
class _StringCompareEnum(Enum):
diff --git a/airflow/providers/apache/hive/hooks/hive.py
b/airflow/providers/apache/hive/hooks/hive.py
index e46530d35a..9e9e893733 100644
--- a/airflow/providers/apache/hive/hooks/hive.py
+++ b/airflow/providers/apache/hive/hooks/hive.py
@@ -315,7 +315,7 @@ class HiveCliHook(BaseHook):
message = e.args[0].splitlines()[-2]
self.log.info(message)
error_loc = re.search(r"(\d+):(\d+)", message)
- if error_loc and error_loc.group(1).isdigit():
+ if error_loc:
lst = int(error_loc.group(1))
begin = max(lst - 2, 0)
end = min(lst + 3, len(query.splitlines()))
diff --git a/airflow/providers/apache/livy/hooks/livy.py
b/airflow/providers/apache/livy/hooks/livy.py
index 96b1582d49..ff2f9faac3 100644
--- a/airflow/providers/apache/livy/hooks/livy.py
+++ b/airflow/providers/apache/livy/hooks/livy.py
@@ -419,7 +419,7 @@ class LivyHook(HttpHook, LoggingMixin):
:param size: size value
:return: true if valid format
"""
- if size and not (isinstance(size, str) and re.match(r"^\d+[kmgt]b?$",
size, re.IGNORECASE)):
+ if size and not (isinstance(size, str) and
re.fullmatch(r"\d+[kmgt]b?", size, re.IGNORECASE)):
raise ValueError(f"Invalid java size format for string'{size}'")
return True
@@ -800,7 +800,7 @@ class LivyAsyncHook(HttpAsyncHook, LoggingMixin):
:param size: size value
:return: true if valid format
"""
- if size and not (isinstance(size, str) and re.match(r"^\d+[kmgt]b?$",
size, re.IGNORECASE)):
+ if size and not (isinstance(size, str) and
re.fullmatch(r"\d+[kmgt]b?", size, re.IGNORECASE)):
raise ValueError(f"Invalid java size format for string'{size}'")
return True
diff --git a/airflow/providers/apache/spark/hooks/spark_submit.py
b/airflow/providers/apache/spark/hooks/spark_submit.py
index 97b1f4ebc9..f95a9d0341 100644
--- a/airflow/providers/apache/spark/hooks/spark_submit.py
+++ b/airflow/providers/apache/spark/hooks/spark_submit.py
@@ -461,9 +461,9 @@ class SparkSubmitHook(BaseHook, LoggingMixin):
# If we run yarn cluster mode, we want to extract the application
id from
# the logs so we can kill the application when we stop it
unexpectedly
if self._is_yarn and self._connection["deploy_mode"] == "cluster":
- match = re.search("(application[0-9_]+)", line)
+ match = re.search("application[0-9_]+", line)
if match:
- self._yarn_application_id = match.groups()[0]
+ self._yarn_application_id = match.group(0)
self.log.info("Identified spark driver id: %s",
self._yarn_application_id)
# If we run Kubernetes cluster mode, we want to extract the driver
pod id
@@ -471,21 +471,21 @@ class SparkSubmitHook(BaseHook, LoggingMixin):
elif self._is_kubernetes:
match = re.search(r"\s*pod name: ((.+?)-([a-z0-9]+)-driver)",
line)
if match:
- self._kubernetes_driver_pod = match.groups()[0]
+ self._kubernetes_driver_pod = match.group(1)
self.log.info("Identified spark driver pod: %s",
self._kubernetes_driver_pod)
# Store the Spark Exit code
match_exit_code = re.search(r"\s*[eE]xit code: (\d+)", line)
if match_exit_code:
- self._spark_exit_code = int(match_exit_code.groups()[0])
+ self._spark_exit_code = int(match_exit_code.group(1))
# if we run in standalone cluster mode and we want to track the
driver status
# we need to extract the driver id from the logs. This allows us
to poll for
# the status using the driver id. Also, we can kill the driver
when needed.
elif self._should_track_driver_status and not self._driver_id:
- match_driver_id = re.search(r"(driver-[0-9\-]+)", line)
+ match_driver_id = re.search(r"driver-[0-9\-]+", line)
if match_driver_id:
- self._driver_id = match_driver_id.groups()[0]
+ self._driver_id = match_driver_id.group(0)
self.log.info("identified spark driver id: %s",
self._driver_id)
self.log.info(line)
diff --git a/airflow/providers/common/sql/operators/sql.py
b/airflow/providers/common/sql/operators/sql.py
index 82499ad3c4..2dba7b12cb 100644
--- a/airflow/providers/common/sql/operators/sql.py
+++ b/airflow/providers/common/sql/operators/sql.py
@@ -84,7 +84,7 @@ keep those methods to avoid 8.4.0 version from failing.
"""
-_PROVIDERS_MATCHER = re.compile(r"airflow\.providers\.(.*)\.hooks.*")
+_PROVIDERS_MATCHER = re.compile(r"airflow\.providers\.(.*?)\.hooks.*")
_MIN_SUPPORTED_PROVIDERS_VERSION = {
"amazon": "4.1.0",
diff --git a/airflow/providers/discord/hooks/discord_webhook.py
b/airflow/providers/discord/hooks/discord_webhook.py
index 76fb9ac52f..6bc0734121 100644
--- a/airflow/providers/discord/hooks/discord_webhook.py
+++ b/airflow/providers/discord/hooks/discord_webhook.py
@@ -95,7 +95,7 @@ class DiscordWebhookHook(HttpHook):
)
# make sure endpoint matches the expected Discord webhook format
- if not re.match("^webhooks/[0-9]+/[a-zA-Z0-9_-]+$", endpoint):
+ if not re.fullmatch("webhooks/[0-9]+/[a-zA-Z0-9_-]+", endpoint):
raise AirflowException(
'Expected Discord webhook endpoint in the form of
"webhooks/{webhook.id}/{webhook.token}".'
)
diff --git a/airflow/providers/ftp/sensors/ftp.py
b/airflow/providers/ftp/sensors/ftp.py
index 4ec7cf7516..7dddbf521a 100644
--- a/airflow/providers/ftp/sensors/ftp.py
+++ b/airflow/providers/ftp/sensors/ftp.py
@@ -44,7 +44,7 @@ class FTPSensor(BaseSensorOperator):
"""Errors that are transient in nature, and where action can be retried"""
transient_errors = [421, 425, 426, 434, 450, 451, 452]
- error_code_pattern = re.compile(r"([\d]+)")
+ error_code_pattern = re.compile(r"\d+")
def __init__(
self, *, path: str, ftp_conn_id: str = "ftp_default",
fail_on_transient_errors: bool = True, **kwargs
@@ -64,9 +64,10 @@ class FTPSensor(BaseSensorOperator):
try:
matches = self.error_code_pattern.match(str(e))
code = int(matches.group(0))
- return code
except ValueError:
return e
+ else:
+ return code
def poke(self, context: Context) -> bool:
with self._create_hook() as hook:
diff --git a/airflow/providers/google/cloud/hooks/cloud_sql.py
b/airflow/providers/google/cloud/hooks/cloud_sql.py
index b84e05892c..0b8bece36f 100644
--- a/airflow/providers/google/cloud/hooks/cloud_sql.py
+++ b/airflow/providers/google/cloud/hooks/cloud_sql.py
@@ -670,8 +670,7 @@ class CloudSqlProxyRunner(LoggingMixin):
command_to_run.extend(["--version"])
command_to_run.extend(self._get_credential_parameters())
result = subprocess.check_output(command_to_run).decode("utf-8")
- pattern = re.compile("^.*[V|v]ersion ([^;]*);.*$")
- matched = pattern.match(result)
+ matched = re.search("[Vv]ersion (.*?);", result)
if matched:
return matched.group(1)
else:
diff --git a/airflow/providers/google/cloud/hooks/dataflow.py
b/airflow/providers/google/cloud/hooks/dataflow.py
index 4d155af80e..819a674b82 100644
--- a/airflow/providers/google/cloud/hooks/dataflow.py
+++ b/airflow/providers/google/cloud/hooks/dataflow.py
@@ -867,7 +867,7 @@ class DataflowHook(GoogleBaseHook):
"""Builds Dataflow job name."""
base_job_name = str(job_name).replace("_", "-")
- if not re.match(r"^[a-z]([-a-z0-9]*[a-z0-9])?$", base_job_name):
+ if not re.fullmatch(r"[a-z]([-a-z0-9]*[a-z0-9])?", base_job_name):
raise ValueError(
f"Invalid job_name ({base_job_name}); the name must consist of
only the characters "
f"[-a-z0-9], starting with a letter and ending with a letter
or number "
diff --git a/airflow/providers/google/cloud/operators/dataproc.py
b/airflow/providers/google/cloud/operators/dataproc.py
index 6a8283659b..dde5baf472 100644
--- a/airflow/providers/google/cloud/operators/dataproc.py
+++ b/airflow/providers/google/cloud/operators/dataproc.py
@@ -245,12 +245,13 @@ class ClusterGenerator:
return PreemptibilityType(preemptibility.upper())
def _get_init_action_timeout(self) -> dict:
- match = re.match(r"^(\d+)([sm])$", self.init_action_timeout)
+ match = re.fullmatch(r"(\d+)([sm])", self.init_action_timeout)
if match:
- val = float(match.group(1))
- if match.group(2) == "s":
- return {"seconds": int(val)}
- elif match.group(2) == "m":
+ val = int(match.group(1))
+ unit = match.group(2)
+ if unit == "s":
+ return {"seconds": val}
+ elif unit == "m":
return {"seconds": int(timedelta(minutes=val).total_seconds())}
raise AirflowException(
@@ -811,18 +812,17 @@ class
DataprocScaleClusterOperator(GoogleCloudBaseOperator):
return None
timeout = None
- match = re.match(r"^(\d+)([smdh])$",
self.graceful_decommission_timeout)
+ match = re.fullmatch(r"(\d+)([smdh])",
self.graceful_decommission_timeout)
if match:
- if match.group(2) == "s":
- timeout = int(match.group(1))
- elif match.group(2) == "m":
- val = float(match.group(1))
+ val = int(match.group(1))
+ unit = match.group(2)
+ if unit == "s":
+ timeout = val
+ elif unit == "m":
timeout = int(timedelta(minutes=val).total_seconds())
- elif match.group(2) == "h":
- val = float(match.group(1))
+ elif unit == "h":
timeout = int(timedelta(hours=val).total_seconds())
- elif match.group(2) == "d":
- val = float(match.group(1))
+ elif unit == "d":
timeout = int(timedelta(days=val).total_seconds())
if not timeout:
diff --git a/airflow/providers/google/cloud/secrets/secret_manager.py
b/airflow/providers/google/cloud/secrets/secret_manager.py
index 7269a17ef4..fd8b8e33e2 100644
--- a/airflow/providers/google/cloud/secrets/secret_manager.py
+++ b/airflow/providers/google/cloud/secrets/secret_manager.py
@@ -36,8 +36,8 @@ SECRET_ID_PATTERN = r"^[a-zA-Z0-9-_]*$"
def _parse_version(val):
- val = re.sub(r"(\d+\.\d+\.\d+).*", lambda x: x.group(1), val)
- return tuple(int(x) for x in val.split("."))
+ match = re.search(r"(\d+)\.(\d+)\.(\d+)", val)
+ return tuple(int(x) for x in match.groups())
class CloudSecretManagerBackend(BaseSecretsBackend, LoggingMixin):
diff --git a/airflow/providers/google/cloud/utils/mlengine_operator_utils.py
b/airflow/providers/google/cloud/utils/mlengine_operator_utils.py
index d58e9190d6..feffbd2f35 100644
--- a/airflow/providers/google/cloud/utils/mlengine_operator_utils.py
+++ b/airflow/providers/google/cloud/utils/mlengine_operator_utils.py
@@ -193,7 +193,7 @@ def create_evaluate_ops(
# Verify that task_prefix doesn't have any special characters except hyphen
# '-', which is the only allowed non-alphanumeric character by Dataflow.
- if not re.match(r"^[a-zA-Z][-A-Za-z0-9]*$", task_prefix):
+ if not re.fullmatch(r"[a-zA-Z][-A-Za-z0-9]*", task_prefix):
raise AirflowException(
"Malformed task_id for DataFlowPythonOperator (only alphanumeric "
"and hyphens are allowed but got: " + task_prefix
diff --git a/airflow/providers/microsoft/azure/secrets/key_vault.py
b/airflow/providers/microsoft/azure/secrets/key_vault.py
index 25ee2880ba..af8089287d 100644
--- a/airflow/providers/microsoft/azure/secrets/key_vault.py
+++ b/airflow/providers/microsoft/azure/secrets/key_vault.py
@@ -33,8 +33,8 @@ from airflow.version import version as airflow_version
def _parse_version(val):
- val = re.sub(r"(\d+\.\d+\.\d+).*", lambda x: x.group(1), val)
- return tuple(int(x) for x in val.split("."))
+ match = re.search(r"(\d+)\.(\d+)\.(\d+)", val)
+ return tuple(int(x) for x in match.groups())
class AzureKeyVaultBackend(BaseSecretsBackend, LoggingMixin):