This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 7612f4ba68b Implement concurrent directory transfer in SFTPOperator
(#47533)
7612f4ba68b is described below
commit 7612f4ba68b7a6d767107f35accb9910776590ad
Author: Dawnpool <[email protected]>
AuthorDate: Wed Apr 9 08:56:01 2025 +0900
Implement concurrent directory transfer in SFTPOperator (#47533)
* Add concurrent directory transfer functions
* Add test code
* Change default cpu count
* Add concurrency docstring
---
.../sftp/src/airflow/providers/sftp/hooks/sftp.py | 123 +++++++++++++++++++++
.../src/airflow/providers/sftp/operators/sftp.py | 30 ++++-
providers/sftp/tests/unit/sftp/hooks/test_sftp.py | 17 +++
.../sftp/tests/unit/sftp/operators/test_sftp.py | 32 ++++++
4 files changed, 198 insertions(+), 4 deletions(-)
diff --git a/providers/sftp/src/airflow/providers/sftp/hooks/sftp.py
b/providers/sftp/src/airflow/providers/sftp/hooks/sftp.py
index c1ac71194d9..fe2f1b59f15 100644
--- a/providers/sftp/src/airflow/providers/sftp/hooks/sftp.py
+++ b/providers/sftp/src/airflow/providers/sftp/hooks/sftp.py
@@ -19,6 +19,7 @@
from __future__ import annotations
+import concurrent.futures
import datetime
import os
import stat
@@ -339,6 +340,63 @@ class SFTPHook(SSHHook):
new_local_path = os.path.join(local_full_path,
os.path.relpath(file_path, remote_full_path))
self.retrieve_file(file_path, new_local_path, prefetch)
+ def retrieve_directory_concurrently(
+ self, remote_full_path: str, local_full_path: str, workers: int =
os.cpu_count() or 2
+ ) -> None:
+ """
+ Transfer the remote directory to a local location concurrently.
+
+ If local_full_path is a string path, the directory will be put
+ at that location.
+
+ :param remote_full_path: full path to the remote directory
+ :param local_full_path: full path to the local directory
+ :param prefetch: controls whether prefetch is performed (default: True)
+ :param workers: number of workers to use for concurrent transfer
(default: number of CPUs or 2 if undetermined)
+ """
+
+ def retrieve_file_chunk(
+ conn: SFTPClient, local_file_chunk: list[str], remote_file_chunk:
list[str], prefetch: bool = True
+ ):
+ for local_file, remote_file in zip(local_file_chunk,
remote_file_chunk):
+ conn.get(remote_file, local_file, prefetch=prefetch)
+
+ with self.get_managed_conn():
+ if Path(local_full_path).exists():
+ raise AirflowException(f"{local_full_path} already exists")
+ Path(local_full_path).mkdir(parents=True)
+ new_local_file_paths, remote_file_paths = [], []
+ files, dirs, _ = self.get_tree_map(remote_full_path)
+ for dir_path in dirs:
+ new_local_path = os.path.join(local_full_path,
os.path.relpath(dir_path, remote_full_path))
+ Path(new_local_path).mkdir(parents=True, exist_ok=True)
+ for file in files:
+ remote_file_paths.append(file)
+ new_local_file_paths.append(
+ os.path.join(local_full_path, os.path.relpath(file,
remote_full_path))
+ )
+ remote_file_chunks = [remote_file_paths[i::workers] for i in
range(workers)]
+ local_file_chunks = [new_local_file_paths[i::workers] for i in
range(workers)]
+ self.log.info("Opening %s new SFTP connections", workers)
+ conns = [SFTPHook(ssh_conn_id=self.ssh_conn_id).get_conn() for _ in
range(workers)]
+ try:
+ self.log.info("Retrieving files concurrently with %s threads",
workers)
+ with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as
executor:
+ futures = [
+ executor.submit(
+ retrieve_file_chunk,
+ conns[i],
+ local_file_chunks[i],
+ remote_file_chunks[i],
+ )
+ for i in range(workers)
+ ]
+ for future in concurrent.futures.as_completed(futures):
+ future.result()
+ finally:
+ for conn in conns:
+ conn.close()
+
def store_directory(self, remote_full_path: str, local_full_path: str,
confirm: bool = True) -> None:
"""
Transfer a local directory to the remote location.
@@ -367,6 +425,71 @@ class SFTPHook(SSHHook):
)
self.store_file(new_remote_path, file_path, confirm)
+ def store_directory_concurrently(
+ self,
+ remote_full_path: str,
+ local_full_path: str,
+ confirm: bool = True,
+ workers: int = os.cpu_count() or 2,
+ ) -> None:
+ """
+ Transfer a local directory to the remote location concurrently.
+
+ If local_full_path is a string path, the directory will be read
+ from that location.
+
+ :param remote_full_path: full path to the remote directory
+ :param local_full_path: full path to the local directory
+ :param confirm: whether to confirm the file size after transfer
(default: True)
+ :param workers: number of workers to use for concurrent transfer
(default: number of CPUs or 2 if undetermined)
+ """
+
+ def store_file_chunk(
+ conn: SFTPClient, local_file_chunk: list[str], remote_file_chunk:
list[str], confirm: bool
+ ):
+ for local_file, remote_file in zip(local_file_chunk,
remote_file_chunk):
+ conn.put(local_file, remote_file, confirm=confirm)
+
+ with self.get_managed_conn():
+ if self.path_exists(remote_full_path):
+ raise AirflowException(f"{remote_full_path} already exists")
+ self.create_directory(remote_full_path)
+
+ local_file_paths, new_remote_file_paths = [], []
+ for root, dirs, files in os.walk(local_full_path):
+ for dir_name in dirs:
+ dir_path = os.path.join(root, dir_name)
+ new_remote_path = os.path.join(
+ remote_full_path, os.path.relpath(dir_path,
local_full_path)
+ )
+ self.create_directory(new_remote_path)
+ for file_name in files:
+ file_path = os.path.join(root, file_name)
+ new_remote_path = os.path.join(
+ remote_full_path, os.path.relpath(file_path,
local_full_path)
+ )
+ local_file_paths.append(file_path)
+ new_remote_file_paths.append(new_remote_path)
+
+ remote_file_chunks = [new_remote_file_paths[i::workers] for i in
range(workers)]
+ local_file_chunks = [local_file_paths[i::workers] for i in
range(workers)]
+ self.log.info("Opening %s new SFTP connections", workers)
+ conns = [SFTPHook(ssh_conn_id=self.ssh_conn_id).get_conn() for _ in
range(workers)]
+ try:
+ self.log.info("Storing files concurrently with %s threads",
workers)
+ with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as
executor:
+ futures = [
+ executor.submit(
+ store_file_chunk, conns[i], local_file_chunks[i],
remote_file_chunks[i], confirm
+ )
+ for i in range(workers)
+ ]
+ for future in concurrent.futures.as_completed(futures):
+ future.result()
+ finally:
+ for conn in conns:
+ conn.close()
+
def get_mod_time(self, path: str) -> str:
"""
Get an entry's modification time.
diff --git a/providers/sftp/src/airflow/providers/sftp/operators/sftp.py
b/providers/sftp/src/airflow/providers/sftp/operators/sftp.py
index 99365e0932d..dfa7577c728 100644
--- a/providers/sftp/src/airflow/providers/sftp/operators/sftp.py
+++ b/providers/sftp/src/airflow/providers/sftp/operators/sftp.py
@@ -74,6 +74,8 @@ class SFTPOperator(BaseOperator):
create_intermediate_dirs=True,
dag=dag,
)
+ :param concurrency: Number of threads when transferring directories. Each
thread opens a new SFTP connection.
+ This parameter is used only when transferring directories, not
individual files. (Default is 1)
"""
@@ -90,6 +92,7 @@ class SFTPOperator(BaseOperator):
operation: str = SFTPOperation.PUT,
confirm: bool = True,
create_intermediate_dirs: bool = False,
+ concurrency: int = 1,
**kwargs,
) -> None:
super().__init__(**kwargs)
@@ -101,6 +104,7 @@ class SFTPOperator(BaseOperator):
self.create_intermediate_dirs = create_intermediate_dirs
self.local_filepath = local_filepath
self.remote_filepath = remote_filepath
+ self.concurrency = concurrency
def execute(self, context: Any) -> str | list[str] | None:
if self.local_filepath is None:
@@ -132,6 +136,9 @@ class SFTPOperator(BaseOperator):
f"expected {SFTPOperation.GET} or {SFTPOperation.PUT} or
{SFTPOperation.DELETE}."
)
+ if self.concurrency < 1:
+ raise ValueError(f"concurrency should be greater than 0, got
{self.concurrency}")
+
file_msg = None
try:
if self.ssh_conn_id:
@@ -161,7 +168,14 @@ class SFTPOperator(BaseOperator):
file_msg = f"from {_remote_filepath} to
{_local_filepath}"
self.log.info("Starting to transfer %s", file_msg)
if self.sftp_hook.isdir(_remote_filepath):
-
self.sftp_hook.retrieve_directory(_remote_filepath, _local_filepath)
+ if self.concurrency > 1:
+ self.sftp_hook.retrieve_directory_concurrently(
+ _remote_filepath,
+ _local_filepath,
+ workers=self.concurrency,
+ )
+ elif self.concurrency == 1:
+
self.sftp_hook.retrieve_directory(_remote_filepath, _local_filepath)
else:
self.sftp_hook.retrieve_file(_remote_filepath,
_local_filepath)
elif self.operation.lower() == SFTPOperation.PUT:
@@ -171,9 +185,17 @@ class SFTPOperator(BaseOperator):
file_msg = f"from {_local_filepath} to
{_remote_filepath}"
self.log.info("Starting to transfer file %s", file_msg)
if os.path.isdir(_local_filepath):
- self.sftp_hook.store_directory(
- _remote_filepath, _local_filepath,
confirm=self.confirm
- )
+ if self.concurrency > 1:
+ self.sftp_hook.store_directory_concurrently(
+ _remote_filepath,
+ _local_filepath,
+ confirm=self.confirm,
+ workers=self.concurrency,
+ )
+ elif self.concurrency == 1:
+ self.sftp_hook.store_directory(
+ _remote_filepath, _local_filepath,
confirm=self.confirm
+ )
else:
self.sftp_hook.store_file(_remote_filepath,
_local_filepath, confirm=self.confirm)
elif self.operation.lower() == SFTPOperation.DELETE:
diff --git a/providers/sftp/tests/unit/sftp/hooks/test_sftp.py
b/providers/sftp/tests/unit/sftp/hooks/test_sftp.py
index 38239546da8..cab012921f2 100644
--- a/providers/sftp/tests/unit/sftp/hooks/test_sftp.py
+++ b/providers/sftp/tests/unit/sftp/hooks/test_sftp.py
@@ -530,6 +530,23 @@ class TestSFTPHook:
)
assert retrieved_dir_name in os.listdir(os.path.join(self.temp_dir,
TMP_DIR_FOR_TESTS))
+ def test_store_and_retrieve_directory_concurrently(self):
+ stored_dir_name = "stored_dir"
+ self.hook.store_directory_concurrently(
+ remote_full_path=os.path.join(self.temp_dir, TMP_DIR_FOR_TESTS,
stored_dir_name),
+ local_full_path=os.path.join(self.temp_dir, TMP_DIR_FOR_TESTS,
SUB_DIR),
+ )
+ output = self.hook.list_directory(
+ path=os.path.join(self.temp_dir, TMP_DIR_FOR_TESTS,
stored_dir_name)
+ )
+ assert output == [TMP_FILE_FOR_TESTS]
+ retrieved_dir_name = "retrieved_dir"
+ self.hook.retrieve_directory_concurrently(
+ remote_full_path=os.path.join(self.temp_dir, TMP_DIR_FOR_TESTS,
stored_dir_name),
+ local_full_path=os.path.join(self.temp_dir, TMP_DIR_FOR_TESTS,
retrieved_dir_name),
+ )
+ assert retrieved_dir_name in os.listdir(os.path.join(self.temp_dir,
TMP_DIR_FOR_TESTS))
+
@patch("paramiko.SSHClient")
@patch("paramiko.ProxyCommand")
def test_sftp_hook_with_proxy_command(self, mock_proxy_command,
mock_ssh_client):
diff --git a/providers/sftp/tests/unit/sftp/operators/test_sftp.py
b/providers/sftp/tests/unit/sftp/operators/test_sftp.py
index a376fefda7d..b7e94c376e1 100644
--- a/providers/sftp/tests/unit/sftp/operators/test_sftp.py
+++ b/providers/sftp/tests/unit/sftp/operators/test_sftp.py
@@ -429,6 +429,22 @@ class TestSFTPOperator:
args, _ = mock_get.call_args_list[0]
assert args == (remote_dirpath, local_dirpath)
+
@mock.patch("airflow.providers.sftp.operators.sftp.SFTPHook.retrieve_directory_concurrently")
+ def test_str_dirpaths_get_concurrently(self, mock_get):
+ local_dirpath = "/tmp_local"
+ remote_dirpath = "/tmp"
+ SFTPOperator(
+ task_id="test_str_to_list",
+ sftp_hook=self.sftp_hook,
+ local_filepath=local_dirpath,
+ remote_filepath=remote_dirpath,
+ operation=SFTPOperation.GET,
+ concurrency=2,
+ ).execute(None)
+ assert mock_get.call_count == 1
+ args, _ = mock_get.call_args_list[0]
+ assert args == (remote_dirpath, local_dirpath)
+
@mock.patch("airflow.providers.sftp.operators.sftp.SFTPHook.store_file")
def test_str_filepaths_put(self, mock_get):
local_filepath = "/tmp/test"
@@ -477,6 +493,22 @@ class TestSFTPOperator:
args, _ = mock_get.call_args_list[0]
assert args == (remote_dirpath, local_dirpath)
+
@mock.patch("airflow.providers.sftp.operators.sftp.SFTPHook.store_directory_concurrently")
+ def test_str_dirpaths_put_concurrently(self, mock_get):
+ local_dirpath = "/tmp"
+ remote_dirpath = "/tmp_remote"
+ SFTPOperator(
+ task_id="test_str_dirpaths_put",
+ sftp_hook=self.sftp_hook,
+ local_filepath=local_dirpath,
+ remote_filepath=remote_dirpath,
+ operation=SFTPOperation.PUT,
+ concurrency=2,
+ ).execute(None)
+ assert mock_get.call_count == 1
+ args, _ = mock_get.call_args_list[0]
+ assert args == (remote_dirpath, local_dirpath)
+
@mock.patch("airflow.providers.sftp.operators.sftp.SFTPHook.retrieve_file")
def test_return_str_when_local_filepath_was_str(self, mock_get):
local_filepath = "/tmp/ltest1"