This is an automated email from the ASF dual-hosted git repository. machristie pushed a commit to branch mft-integration in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit c80aa136889125717d299920082cd27192e5731f Author: Marcus Christie <[email protected]> AuthorDate: Thu Apr 15 14:43:05 2021 -0400 AIRAVATA-3420 Implementing get_download_url, moving download view to sdk --- airavata_django_portal_sdk/urls.py | 16 ++++-- .../user_storage/__init__.py | 2 + airavata_django_portal_sdk/user_storage/api.py | 25 +++++++++- .../user_storage/backends/base.py | 18 ++++--- .../user_storage/backends/mft_provider.py | 40 ++++++++++++--- airavata_django_portal_sdk/views.py | 57 ++++++++++++++++++++++ 6 files changed, 142 insertions(+), 16 deletions(-) diff --git a/airavata_django_portal_sdk/urls.py b/airavata_django_portal_sdk/urls.py index 9469b0f..5b70a91 100644 --- a/airavata_django_portal_sdk/urls.py +++ b/airavata_django_portal_sdk/urls.py @@ -1,9 +1,19 @@ +import warnings from urllib.parse import urlencode -from django.urls import reverse +from django.urls import path, reverse + +from . import views def get_download_url(data_product_uri): - """Get URL for downloading data product identified by data_product_uri.""" - return (reverse("django_airavata_api:download_file") + "?" + + """(Deprecated) Get URL for downloading data product identified by data_product_uri.""" + warnings.warn("Use user_storage.get_download_url instead.", DeprecationWarning) + return (reverse("airavata_django_portal_sdk:download_file") + "?" + urlencode({"data-product-uri": data_product_uri})) + + +app_name = 'airavata_django_portal_sdk' +urlpatterns = [ + path('download', views.download_file, name='download_file'), +] diff --git a/airavata_django_portal_sdk/user_storage/__init__.py b/airavata_django_portal_sdk/user_storage/__init__.py index 21a22b4..283b2b9 100644 --- a/airavata_django_portal_sdk/user_storage/__init__.py +++ b/airavata_django_portal_sdk/user_storage/__init__.py @@ -7,6 +7,7 @@ from .api import ( dir_exists, exists, experiment_dir_exists, + get_download_url, get_experiment_dir, get_file, get_file_metadata, @@ -33,6 +34,7 @@ __all__ = [ 'dir_exists', 'exists', 'experiment_dir_exists', + 'get_download_url', 'get_experiment_dir', 'get_file', 'get_file_metadata', diff --git a/airavata_django_portal_sdk/user_storage/api.py b/airavata_django_portal_sdk/user_storage/api.py index 1d69836..93e36e0 100644 --- a/airavata_django_portal_sdk/user_storage/api.py +++ b/airavata_django_portal_sdk/user_storage/api.py @@ -7,7 +7,7 @@ import mimetypes import os import warnings from http import HTTPStatus -from urllib.parse import quote, unquote, urlparse +from urllib.parse import quote, unquote, urlencode, urlparse import requests from airavata.model.data.replica.ttypes import ( @@ -21,6 +21,8 @@ from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from ..util import convert_iso8601_to_datetime +from django.urls import reverse +from airavata_django_portal_sdk.user_storage.backends.base import ProvidesDownloadUrl logger = logging.getLogger(__name__) @@ -181,6 +183,25 @@ def move_input_file(request, data_product=None, path=None, data_product_uri=None return move(request, data_product=data_product, path=path, data_product_uri=data_product_uri, storage_resource_id=storage_resource_id) +def get_download_url(request, data_product=None, data_product_uri=None): + if data_product is None: + data_product = _get_data_product(request, data_product_uri) + if _is_remote_api(): + raise NotImplementedError() + else: + storage_resource_id, path = _get_replica_resource_id_and_filepath(data_product) + backend = get_user_storage_provider(request, + owner_username=data_product.ownerName, + storage_resource_id=storage_resource_id) + if isinstance(backend, ProvidesDownloadUrl): + return backend.get_download_url(path) + else: + # if backend doesn't provide a download url, then use default one + # that uses backend to read the file + return (reverse("airavata_django_portal_sdk:download_file") + "?" + + urlencode({"data-product-uri": data_product.productUri})) + + def open_file(request, data_product=None, data_product_uri=None): """ Return file object for replica if it exists in user storage. One of @@ -435,6 +456,8 @@ def listdir(request, path, storage_resource_id=None): mime_type = data_product.productMetadata['mime-type'] file['data-product-uri'] = data_product_uri file['mime_type'] = mime_type + # TODO: remove this, there's no need for hidden files + file['hidden'] = False return directories, files diff --git a/airavata_django_portal_sdk/user_storage/backends/base.py b/airavata_django_portal_sdk/user_storage/backends/base.py index 9936765..95601fc 100644 --- a/airavata_django_portal_sdk/user_storage/backends/base.py +++ b/airavata_django_portal_sdk/user_storage/backends/base.py @@ -1,4 +1,16 @@ +class ProvidesDownloadUrl: + """Mixin for UserStorageProvider that provides download url.""" + def get_download_url(self, resource_path): + raise NotImplementedError() + + +class ProvidesUploadUrl: + """Mixin for UserStorageProvider that provides upload url.""" + def get_upload_url(self, resource_path): + raise NotImplementedError() + + class UserStorageProvider: def __init__(self, authz_token, resource_id, context=None, **kwargs): self.authz_token = authz_token @@ -13,15 +25,9 @@ class UserStorageProvider: """ raise NotImplementedError() - def get_upload_url(self, resource_path): - raise NotImplementedError() - def open(self, resource_path): raise NotImplementedError() - def get_download_url(self, resource_path): - raise NotImplementedError() - def exists(self, resource_path): raise NotImplementedError() diff --git a/airavata_django_portal_sdk/user_storage/backends/mft_provider.py b/airavata_django_portal_sdk/user_storage/backends/mft_provider.py index 488e0b0..cc53e7e 100644 --- a/airavata_django_portal_sdk/user_storage/backends/mft_provider.py +++ b/airavata_django_portal_sdk/user_storage/backends/mft_provider.py @@ -1,16 +1,18 @@ +import io import logging import os from datetime import datetime import grpc +import requests from . import CredCommon_pb2, MFTApi_pb2, MFTApi_pb2_grpc -from .base import UserStorageProvider +from .base import ProvidesDownloadUrl, UserStorageProvider logger = logging.getLogger(__name__) -class MFTUserStorageProvider(UserStorageProvider): +class MFTUserStorageProvider(UserStorageProvider, ProvidesDownloadUrl): def __init__(self, authz_token, resource_id, context=None, resource_token=None, mft_api_endpoint=None, mft_api_secure=False, resource_per_gateway=False, **kwargs): super().__init__(authz_token, resource_id, context=context, **kwargs) @@ -100,9 +102,6 @@ class MFTUserStorageProvider(UserStorageProvider): "resource_path": d.resourcePath, "created_time": created_time, "size": size, - # TODO how to handle hidden directories or directories for - # staging input file uploads - "hidden": False } ) files_data = [] @@ -127,7 +126,6 @@ class MFTUserStorageProvider(UserStorageProvider): "resource_path": f.resourcePath, "created_time": created_time, "size": size, - "hidden": False, } ) return directories_data, files_data @@ -182,6 +180,36 @@ class MFTUserStorageProvider(UserStorageProvider): logger.warning(f"Could not get metadata for {child_path} on {self.resource_id}") return False + def get_download_url(self, resource_path): + with grpc.insecure_channel(self.mft_api_endpoint) as channel: + child_path = self._get_child_path(resource_path) + stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel) + download_request = MFTApi_pb2.HttpDownloadApiRequest( + # sourceResourceId=self.resource_id, + # FIXME: just hacking in something to force it to work + sourceResourceId="remote-ssh-resource", + # sourcePath=child_path, + sourceToken=self.resource_token, + sourceType="SCP", + targetAgent="agent0", + mftAuthorizationToken=self.auth_token, + ) + try: + response = stub.submitHttpDownload(download_request) + logger.debug(f"Download request for {self.resource_id}:{child_path}. Response = {response}") + return response.url + except Exception as e: + logger.error(f"submitHttpDownload request {download_request} failed.") + raise Exception(f"Failed to get download url for {resource_path}") from e + + def open(self, resource_path): + download_url = self.get_download_url(resource_path) + r = requests.get(download_url) + r.raise_for_status() + file = io.BytesIO(r.content) + file.name = os.path.basename(resource_path) + return file + def _get_child_path(self, resource_path): """Convert possibly relative child path into absolute path.""" if not resource_path.startswith("/"): diff --git a/airavata_django_portal_sdk/views.py b/airavata_django_portal_sdk/views.py new file mode 100644 index 0000000..688ca84 --- /dev/null +++ b/airavata_django_portal_sdk/views.py @@ -0,0 +1,57 @@ +import logging +import os +from urllib.parse import urlparse + +from django.contrib.auth.decorators import login_required +from django.core.exceptions import ObjectDoesNotExist +from django.http import FileResponse, Http404 +from django.shortcuts import redirect +from django.urls import reverse + +from airavata_django_portal_sdk import user_storage + +logger = logging.getLogger(__name__) + + +# TODO: moving this view out of REST API means losing access token based authentication +@login_required +def download_file(request): + + data_product_uri = request.GET.get('data-product-uri', '') + download_url = user_storage.get_download_url(request, data_product_uri=data_product_uri) + # If the download_url resolves to this view, then handle it directly + if urlparse(download_url).path == reverse('airavata_django_portal_sdk:download_file'): + return _internal_download_file(request) + else: + return redirect(download_url) + + +def _internal_download_file(request): + data_product_uri = request.GET.get('data-product-uri', '') + force_download = 'download' in request.GET + data_product = None + try: + data_product = request.airavata_client.getDataProduct( + request.authz_token, data_product_uri) + mime_type = "application/octet-stream" # default mime-type + if (data_product.productMetadata and + 'mime-type' in data_product.productMetadata): + mime_type = data_product.productMetadata['mime-type'] + # 'mime-type' url parameter overrides + mime_type = request.GET.get('mime-type', mime_type) + except Exception as e: + logger.warning("Failed to load DataProduct for {}" + .format(data_product_uri), exc_info=True) + raise Http404("data product does not exist") from e + try: + data_file = user_storage.open_file(request, data_product) + response = FileResponse(data_file, content_type=mime_type) + file_name = os.path.basename(data_file.name) + if mime_type == 'application/octet-stream' or force_download: + response['Content-Disposition'] = ('attachment; filename="{}"' + .format(file_name)) + else: + response['Content-Disposition'] = f'inline; filename="{file_name}"' + return response + except ObjectDoesNotExist as e: + raise Http404(str(e)) from e
