This is an automated email from the ASF dual-hosted git repository. machristie pushed a commit to branch mft-integration in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit d7858965e93ecc968a50f0546cab8a3e31cc3625 Author: Marcus Christie <[email protected]> AuthorDate: Wed Apr 14 18:03:12 2021 -0400 AIRAVATA-3420 Implement browsing methods of MFTUserStorageProvider --- airavata_django_portal_sdk/user_storage/api.py | 27 +- .../user_storage/backends/CredCommon_pb2.py | 306 ++++++++++++++++++ .../user_storage/backends/CredCommon_pb2_grpc.py | 4 + .../{ => user_storage/backends}/MFTApi_pb2.py | 189 +++++------- .../{ => user_storage/backends}/MFTApi_pb2_grpc.py | 0 .../user_storage/backends/__init__.py | 3 +- .../user_storage/backends/base.py | 4 + .../backends/django_filesystem_provider.py | 11 +- .../user_storage/backends/mft_provider.py | 341 ++++++++++++--------- 9 files changed, 608 insertions(+), 277 deletions(-) diff --git a/airavata_django_portal_sdk/user_storage/api.py b/airavata_django_portal_sdk/user_storage/api.py index 4a21498..1d69836 100644 --- a/airavata_django_portal_sdk/user_storage/api.py +++ b/airavata_django_portal_sdk/user_storage/api.py @@ -51,7 +51,8 @@ def get_user_storage_provider(request, owner_username=None, storage_resource_id= options = dict(directory=settings.GATEWAY_DATA_STORE_DIR) logger.warning("Please add the USER_STORAGES setting. Using legacy GATEWAY_DATA_STORE_RESOURCE_ID and GATEWAY_DATA_STORE_DIR settings.") else: - for conf in settings.USER_STORAGES: + for key in settings.USER_STORAGES: + conf = settings.USER_STORAGES[key] if conf['STORAGE_RESOURCE_ID'] == storage_resource_id: module_class_name = conf['BACKEND'] options = conf.get('OPTIONS', {}) @@ -241,7 +242,7 @@ def dir_exists(request, path, storage_resource_id=None): return resp.json()['isDir'] else: backend = get_user_storage_provider(request, storage_resource_id=storage_resource_id) - return backend.exists(path) + return backend.is_dir(path) def user_file_exists(request, path, storage_resource_id=None): @@ -256,7 +257,7 @@ def user_file_exists(request, path, storage_resource_id=None): resp.raise_for_status() return resp.json()['files'][0]['dataProductURI'] backend = get_user_storage_provider(request, storage_resource_id=storage_resource_id) - if backend.exists(path) and backend.is_file(path): + if backend.is_file(path): _, files = backend.get_metadata(path) full_path = files[0]['resource_path'] data_product_uri = _get_data_product_uri(request, full_path, backend.resource_id) @@ -343,7 +344,7 @@ def get_file_metadata(request, path, storage_resource_id=None): return file backend = get_user_storage_provider(request, storage_resource_id=storage_resource_id) - if backend.exists(path) and backend.is_file(path): + if backend.is_file(path): _, files = backend.get_metadata(path) file = files[0] data_product_uri = _get_data_product_uri(request, file['resource_path'], @@ -418,6 +419,9 @@ def listdir(request, path, storage_resource_id=None): backend = get_user_storage_provider(request, storage_resource_id=storage_resource_id) directories, files = backend.get_metadata(path) + # Mark the TMP_INPUT_FILE_UPLOAD_DIR directory as hidden in the UI + for directory in directories: + directory['hidden'] = directory['path'] == TMP_INPUT_FILE_UPLOAD_DIR # for each file, lookup or register a data product and enrich the file # metadata with data-product-uri and mime-type for file in files: @@ -668,13 +672,14 @@ def _determine_content_type(full_path, content_type=None): # Try to guess the content-type from file extension guessed_type, encoding = mimetypes.guess_type(full_path) result = guessed_type - if result is None or result == "application/octet-stream": - # Check if file is Unicode text by trying to read some of it - try: - open(full_path, "r").read(1024) - result = "text/plain" - except UnicodeDecodeError: - logger.debug(f"Failed to read as Unicode text: {full_path}") + # TODO: implement change to use UserStorageProvider.open to read from file + # if result is None or result == "application/octet-stream": + # # Check if file is Unicode text by trying to read some of it + # try: + # open(full_path, "r").read(1024) + # result = "text/plain" + # except UnicodeDecodeError: + # logger.debug(f"Failed to read as Unicode text: {full_path}") return result diff --git a/airavata_django_portal_sdk/user_storage/backends/CredCommon_pb2.py b/airavata_django_portal_sdk/user_storage/backends/CredCommon_pb2.py new file mode 100644 index 0000000..3c6a6c3 --- /dev/null +++ b/airavata_django_portal_sdk/user_storage/backends/CredCommon_pb2.py @@ -0,0 +1,306 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: CredCommon.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='CredCommon.proto', + package='org.apache.airavata.mft.common', + syntax='proto3', + serialized_options=b'P\001', + create_key=_descriptor._internal_create_key, + serialized_pb=b'\n\x10\x43redCommon.proto\x12\x1eorg.apache.airavata.mft.common\"\x1e\n\rUserTokenAuth\x12\r\n\x05token\x18\x01 \x01(\t\"@\n\tAgentAuth\x12\r\n\x05token\x18\x01 \x01(\t\x12\x0f\n\x07\x61gentId\x18\x02 \x01(\t\x12\x13\n\x0b\x61gentSecret\x18\x03 \x01(\t\"\xcb\x01\n\x0c\x44\x65legateAuth\x12\x0e\n\x06userId\x18\x01 \x01(\t\x12\x10\n\x08\x63lientId\x18\x02 \x01(\t\x12\x14\n\x0c\x63lientSecret\x18\x03 \x01(\t\x12P\n\nproperties\x18\x04 \x03(\x0b\x32<.org.apache.airavata.mft [...] +) + + + + +_USERTOKENAUTH = _descriptor.Descriptor( + name='UserTokenAuth', + full_name='org.apache.airavata.mft.common.UserTokenAuth', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='token', full_name='org.apache.airavata.mft.common.UserTokenAuth.token', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=52, + serialized_end=82, +) + + +_AGENTAUTH = _descriptor.Descriptor( + name='AgentAuth', + full_name='org.apache.airavata.mft.common.AgentAuth', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='token', full_name='org.apache.airavata.mft.common.AgentAuth.token', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='agentId', full_name='org.apache.airavata.mft.common.AgentAuth.agentId', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='agentSecret', full_name='org.apache.airavata.mft.common.AgentAuth.agentSecret', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=84, + serialized_end=148, +) + + +_DELEGATEAUTH_PROPERTIESENTRY = _descriptor.Descriptor( + name='PropertiesEntry', + full_name='org.apache.airavata.mft.common.DelegateAuth.PropertiesEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='org.apache.airavata.mft.common.DelegateAuth.PropertiesEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='value', full_name='org.apache.airavata.mft.common.DelegateAuth.PropertiesEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=b'8\001', + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=305, + serialized_end=354, +) + +_DELEGATEAUTH = _descriptor.Descriptor( + name='DelegateAuth', + full_name='org.apache.airavata.mft.common.DelegateAuth', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='userId', full_name='org.apache.airavata.mft.common.DelegateAuth.userId', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='clientId', full_name='org.apache.airavata.mft.common.DelegateAuth.clientId', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='clientSecret', full_name='org.apache.airavata.mft.common.DelegateAuth.clientSecret', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='properties', full_name='org.apache.airavata.mft.common.DelegateAuth.properties', index=3, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_DELEGATEAUTH_PROPERTIESENTRY, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=151, + serialized_end=354, +) + + +_AUTHTOKEN = _descriptor.Descriptor( + name='AuthToken', + full_name='org.apache.airavata.mft.common.AuthToken', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='userTokenAuth', full_name='org.apache.airavata.mft.common.AuthToken.userTokenAuth', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='agentAuth', full_name='org.apache.airavata.mft.common.AuthToken.agentAuth', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='delegateAuth', full_name='org.apache.airavata.mft.common.AuthToken.delegateAuth', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='authMechanism', full_name='org.apache.airavata.mft.common.AuthToken.authMechanism', + index=0, containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[]), + ], + serialized_start=357, + serialized_end=591, +) + +_DELEGATEAUTH_PROPERTIESENTRY.containing_type = _DELEGATEAUTH +_DELEGATEAUTH.fields_by_name['properties'].message_type = _DELEGATEAUTH_PROPERTIESENTRY +_AUTHTOKEN.fields_by_name['userTokenAuth'].message_type = _USERTOKENAUTH +_AUTHTOKEN.fields_by_name['agentAuth'].message_type = _AGENTAUTH +_AUTHTOKEN.fields_by_name['delegateAuth'].message_type = _DELEGATEAUTH +_AUTHTOKEN.oneofs_by_name['authMechanism'].fields.append( + _AUTHTOKEN.fields_by_name['userTokenAuth']) +_AUTHTOKEN.fields_by_name['userTokenAuth'].containing_oneof = _AUTHTOKEN.oneofs_by_name['authMechanism'] +_AUTHTOKEN.oneofs_by_name['authMechanism'].fields.append( + _AUTHTOKEN.fields_by_name['agentAuth']) +_AUTHTOKEN.fields_by_name['agentAuth'].containing_oneof = _AUTHTOKEN.oneofs_by_name['authMechanism'] +_AUTHTOKEN.oneofs_by_name['authMechanism'].fields.append( + _AUTHTOKEN.fields_by_name['delegateAuth']) +_AUTHTOKEN.fields_by_name['delegateAuth'].containing_oneof = _AUTHTOKEN.oneofs_by_name['authMechanism'] +DESCRIPTOR.message_types_by_name['UserTokenAuth'] = _USERTOKENAUTH +DESCRIPTOR.message_types_by_name['AgentAuth'] = _AGENTAUTH +DESCRIPTOR.message_types_by_name['DelegateAuth'] = _DELEGATEAUTH +DESCRIPTOR.message_types_by_name['AuthToken'] = _AUTHTOKEN +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +UserTokenAuth = _reflection.GeneratedProtocolMessageType('UserTokenAuth', (_message.Message,), { + 'DESCRIPTOR' : _USERTOKENAUTH, + '__module__' : 'CredCommon_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.common.UserTokenAuth) + }) +_sym_db.RegisterMessage(UserTokenAuth) + +AgentAuth = _reflection.GeneratedProtocolMessageType('AgentAuth', (_message.Message,), { + 'DESCRIPTOR' : _AGENTAUTH, + '__module__' : 'CredCommon_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.common.AgentAuth) + }) +_sym_db.RegisterMessage(AgentAuth) + +DelegateAuth = _reflection.GeneratedProtocolMessageType('DelegateAuth', (_message.Message,), { + + 'PropertiesEntry' : _reflection.GeneratedProtocolMessageType('PropertiesEntry', (_message.Message,), { + 'DESCRIPTOR' : _DELEGATEAUTH_PROPERTIESENTRY, + '__module__' : 'CredCommon_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.common.DelegateAuth.PropertiesEntry) + }) + , + 'DESCRIPTOR' : _DELEGATEAUTH, + '__module__' : 'CredCommon_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.common.DelegateAuth) + }) +_sym_db.RegisterMessage(DelegateAuth) +_sym_db.RegisterMessage(DelegateAuth.PropertiesEntry) + +AuthToken = _reflection.GeneratedProtocolMessageType('AuthToken', (_message.Message,), { + 'DESCRIPTOR' : _AUTHTOKEN, + '__module__' : 'CredCommon_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.common.AuthToken) + }) +_sym_db.RegisterMessage(AuthToken) + + +DESCRIPTOR._options = None +_DELEGATEAUTH_PROPERTIESENTRY._options = None +# @@protoc_insertion_point(module_scope) diff --git a/airavata_django_portal_sdk/user_storage/backends/CredCommon_pb2_grpc.py b/airavata_django_portal_sdk/user_storage/backends/CredCommon_pb2_grpc.py new file mode 100644 index 0000000..2daafff --- /dev/null +++ b/airavata_django_portal_sdk/user_storage/backends/CredCommon_pb2_grpc.py @@ -0,0 +1,4 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + diff --git a/airavata_django_portal_sdk/MFTApi_pb2.py b/airavata_django_portal_sdk/user_storage/backends/MFTApi_pb2.py similarity index 79% rename from airavata_django_portal_sdk/MFTApi_pb2.py rename to airavata_django_portal_sdk/user_storage/backends/MFTApi_pb2.py index 1e89796..2301467 100644 --- a/airavata_django_portal_sdk/MFTApi_pb2.py +++ b/airavata_django_portal_sdk/user_storage/backends/MFTApi_pb2.py @@ -13,6 +13,7 @@ _sym_db = _symbol_database.Default() from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 +from . import CredCommon_pb2 as CredCommon__pb2 DESCRIPTOR = _descriptor.FileDescriptor( @@ -21,9 +22,9 @@ DESCRIPTOR = _descriptor.FileDescriptor( syntax='proto3', serialized_options=b'P\001', create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\x0cMFTApi.proto\x12#org.apache.airavata.mft.api.service\x1a\x1cgoogle/api/annotations.proto\x1a\x1bgoogle/protobuf/empty.proto\"\x9f\x04\n\x12TransferApiRequest\x12\x17\n\x0fsourceStorageId\x18\x01 \x01(\t\x12\x12\n\nsourcePath\x18\x02 \x01(\t\x12\x12\n\nsourceType\x18\x03 \x01(\t\x12\x13\n\x0bsourceToken\x18\x04 \x01(\t\x12\x1d\n\x15sourceResourceBackend\x18\x05 \x01(\t\x12\x1f\n\x17sourceCredentialBackend\x18\x06 \x01(\t\x12\x1c\n\x14\x64\x65stinationStorageId\x18\ [...] + serialized_pb=b'\n\x0cMFTApi.proto\x12#org.apache.airavata.mft.api.service\x1a\x1cgoogle/api/annotations.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x10\x43redCommon.proto\"\xa3\x03\n\x12TransferApiRequest\x12\x18\n\x10sourceResourceId\x18\x01 \x01(\t\x12\x12\n\nsourceType\x18\x02 \x01(\t\x12\x13\n\x0bsourceToken\x18\x03 \x01(\t\x12\x1d\n\x15\x64\x65stinationResourceId\x18\x07 \x01(\t\x12\x17\n\x0f\x64\x65stinationType\x18\t \x01(\t\x12\x18\n\x10\x64\x65stinationToken\x18\n \x01(\t\x1 [...] , - dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,]) + dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,CredCommon__pb2.DESCRIPTOR,]) @@ -62,8 +63,8 @@ _TRANSFERAPIREQUEST_TARGETAGENTSENTRY = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=605, - serialized_end=656, + serialized_start=499, + serialized_end=550, ) _TRANSFERAPIREQUEST = _descriptor.Descriptor( @@ -75,107 +76,65 @@ _TRANSFERAPIREQUEST = _descriptor.Descriptor( create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( - name='sourceStorageId', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceStorageId', index=0, + name='sourceResourceId', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceResourceId', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='sourcePath', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourcePath', index=1, + name='sourceType', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceType', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='sourceType', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceType', index=2, + name='sourceToken', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceToken', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='sourceToken', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceToken', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='sourceResourceBackend', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceResourceBackend', index=4, - number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='sourceCredentialBackend', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceCredentialBackend', index=5, - number=6, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='destinationStorageId', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destinationStorageId', index=6, + name='destinationResourceId', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destinationResourceId', index=3, number=7, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='destinationPath', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destinationPath', index=7, - number=8, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='destinationType', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destinationType', index=8, + name='destinationType', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destinationType', index=4, number=9, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='destinationToken', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destinationToken', index=9, + name='destinationToken', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destinationToken', index=5, number=10, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='destResourceBackend', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destResourceBackend', index=10, - number=11, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='destCredentialBackend', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destCredentialBackend', index=11, - number=12, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='affinityTransfer', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.affinityTransfer', index=12, + name='affinityTransfer', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.affinityTransfer', index=6, number=13, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='targetAgents', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.targetAgents', index=13, + name='targetAgents', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.targetAgents', index=7, number=14, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.mftAuthorizationToken', index=14, - number=15, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), + name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.mftAuthorizationToken', index=8, + number=15, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), @@ -191,8 +150,8 @@ _TRANSFERAPIREQUEST = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=113, - serialized_end=656, + serialized_start=131, + serialized_end=550, ) @@ -223,8 +182,8 @@ _TRANSFERAPIRESPONSE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=658, - serialized_end=699, + serialized_start=552, + serialized_end=593, ) @@ -237,44 +196,37 @@ _HTTPUPLOADAPIREQUEST = _descriptor.Descriptor( create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( - name='destinationStoreId', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.destinationStoreId', index=0, + name='destinationResourceId', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.destinationResourceId', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='destinationPath', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.destinationPath', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='destinationToken', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.destinationToken', index=2, + name='destinationToken', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.destinationToken', index=1, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='destinationType', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.destinationType', index=3, + name='destinationType', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.destinationType', index=2, number=4, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='targetAgent', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.targetAgent', index=4, + name='targetAgent', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.targetAgent', index=3, number=5, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.mftAuthorizationToken', index=5, - number=6, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), + name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.mftAuthorizationToken', index=4, + number=6, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), @@ -290,8 +242,8 @@ _HTTPUPLOADAPIREQUEST = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=702, - serialized_end=880, + serialized_start=596, + serialized_end=795, ) @@ -329,8 +281,8 @@ _HTTPUPLOADAPIRESPONSE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=882, - serialized_end=939, + serialized_start=797, + serialized_end=854, ) @@ -343,44 +295,37 @@ _HTTPDOWNLOADAPIREQUEST = _descriptor.Descriptor( create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( - name='sourceStoreId', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.sourceStoreId', index=0, + name='sourceResourceId', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.sourceResourceId', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='sourcePath', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.sourcePath', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='sourceToken', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.sourceToken', index=2, + name='sourceToken', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.sourceToken', index=1, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='sourceType', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.sourceType', index=3, + name='sourceType', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.sourceType', index=2, number=4, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='targetAgent', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.targetAgent', index=4, + name='targetAgent', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.targetAgent', index=3, number=5, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.mftAuthorizationToken', index=5, - number=6, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), + name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.mftAuthorizationToken', index=4, + number=6, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), @@ -396,8 +341,8 @@ _HTTPDOWNLOADAPIREQUEST = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=942, - serialized_end=1102, + serialized_start=857, + serialized_end=1043, ) @@ -435,8 +380,8 @@ _HTTPDOWNLOADAPIRESPONSE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=1104, - serialized_end=1163, + serialized_start=1045, + serialized_end=1104, ) @@ -457,8 +402,8 @@ _TRANSFERSTATEAPIREQUEST = _descriptor.Descriptor( serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.TransferStateApiRequest.mftAuthorizationToken', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), @@ -474,8 +419,8 @@ _TRANSFERSTATEAPIREQUEST = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=1165, - serialized_end=1241, + serialized_start=1106, + serialized_end=1225, ) @@ -527,8 +472,8 @@ _TRANSFERSTATEAPIRESPONSE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=1243, - serialized_end=1349, + serialized_start=1227, + serialized_end=1333, ) @@ -577,8 +522,8 @@ _RESOURCEAVAILABILITYREQUEST = _descriptor.Descriptor( serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.ResourceAvailabilityRequest.mftAuthorizationToken', index=5, - number=6, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), + number=6, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), @@ -594,8 +539,8 @@ _RESOURCEAVAILABILITYREQUEST = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=1352, - serialized_end=1537, + serialized_start=1336, + serialized_end=1564, ) @@ -626,8 +571,8 @@ _RESOURCEAVAILABILITYRESPONSE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=1539, - serialized_end=1588, + serialized_start=1566, + serialized_end=1615, ) @@ -707,8 +652,8 @@ _FILEMETADATARESPONSE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=1591, - serialized_end=1790, + serialized_start=1618, + serialized_end=1817, ) @@ -795,8 +740,8 @@ _DIRECTORYMETADATARESPONSE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=1793, - serialized_end=2143, + serialized_start=1820, + serialized_end=2170, ) @@ -859,8 +804,8 @@ _FETCHRESOURCEMETADATAREQUEST = _descriptor.Descriptor( serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.FetchResourceMetadataRequest.mftAuthorizationToken', index=7, - number=9, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), + number=9, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), @@ -876,14 +821,20 @@ _FETCHRESOURCEMETADATAREQUEST = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=2146, - serialized_end=2374, + serialized_start=2173, + serialized_end=2444, ) _TRANSFERAPIREQUEST_TARGETAGENTSENTRY.containing_type = _TRANSFERAPIREQUEST _TRANSFERAPIREQUEST.fields_by_name['targetAgents'].message_type = _TRANSFERAPIREQUEST_TARGETAGENTSENTRY +_TRANSFERAPIREQUEST.fields_by_name['mftAuthorizationToken'].message_type = CredCommon__pb2._AUTHTOKEN +_HTTPUPLOADAPIREQUEST.fields_by_name['mftAuthorizationToken'].message_type = CredCommon__pb2._AUTHTOKEN +_HTTPDOWNLOADAPIREQUEST.fields_by_name['mftAuthorizationToken'].message_type = CredCommon__pb2._AUTHTOKEN +_TRANSFERSTATEAPIREQUEST.fields_by_name['mftAuthorizationToken'].message_type = CredCommon__pb2._AUTHTOKEN +_RESOURCEAVAILABILITYREQUEST.fields_by_name['mftAuthorizationToken'].message_type = CredCommon__pb2._AUTHTOKEN _DIRECTORYMETADATARESPONSE.fields_by_name['directories'].message_type = _DIRECTORYMETADATARESPONSE _DIRECTORYMETADATARESPONSE.fields_by_name['files'].message_type = _FILEMETADATARESPONSE +_FETCHRESOURCEMETADATAREQUEST.fields_by_name['mftAuthorizationToken'].message_type = CredCommon__pb2._AUTHTOKEN DESCRIPTOR.message_types_by_name['TransferApiRequest'] = _TRANSFERAPIREQUEST DESCRIPTOR.message_types_by_name['TransferApiResponse'] = _TRANSFERAPIRESPONSE DESCRIPTOR.message_types_by_name['HttpUploadApiRequest'] = _HTTPUPLOADAPIREQUEST @@ -1009,8 +960,8 @@ _MFTAPISERVICE = _descriptor.ServiceDescriptor( index=0, serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_start=2377, - serialized_end=3838, + serialized_start=2447, + serialized_end=3908, methods=[ _descriptor.MethodDescriptor( name='submitTransfer', diff --git a/airavata_django_portal_sdk/MFTApi_pb2_grpc.py b/airavata_django_portal_sdk/user_storage/backends/MFTApi_pb2_grpc.py similarity index 100% rename from airavata_django_portal_sdk/MFTApi_pb2_grpc.py rename to airavata_django_portal_sdk/user_storage/backends/MFTApi_pb2_grpc.py diff --git a/airavata_django_portal_sdk/user_storage/backends/__init__.py b/airavata_django_portal_sdk/user_storage/backends/__init__.py index eefe980..48846df 100644 --- a/airavata_django_portal_sdk/user_storage/backends/__init__.py +++ b/airavata_django_portal_sdk/user_storage/backends/__init__.py @@ -1,3 +1,4 @@ from .django_filesystem_provider import DjangoFileSystemProvider +from .mft_provider import MFTUserStorageProvider -__all__ = ['DjangoFileSystemProvider'] +__all__ = ['DjangoFileSystemProvider', 'MFTUserStorageProvider'] diff --git a/airavata_django_portal_sdk/user_storage/backends/base.py b/airavata_django_portal_sdk/user_storage/backends/base.py index 409a794..9936765 100644 --- a/airavata_django_portal_sdk/user_storage/backends/base.py +++ b/airavata_django_portal_sdk/user_storage/backends/base.py @@ -62,3 +62,7 @@ class UserStorageProvider: @property def username(self): return self.authz_token.claimsMap['userName'] + + @property + def access_token(self): + return self.authz_token.accessToken diff --git a/airavata_django_portal_sdk/user_storage/backends/django_filesystem_provider.py b/airavata_django_portal_sdk/user_storage/backends/django_filesystem_provider.py index 78812bd..97e2f9e 100644 --- a/airavata_django_portal_sdk/user_storage/backends/django_filesystem_provider.py +++ b/airavata_django_portal_sdk/user_storage/backends/django_filesystem_provider.py @@ -6,12 +6,9 @@ from django.core.exceptions import ObjectDoesNotExist, SuspiciousFileOperation from django.core.files.storage import FileSystemStorage from .base import UserStorageProvider -from django.core.files import File logger = logging.getLogger(__name__) -TMP_INPUT_FILE_UPLOAD_DIR = "tmp" - class DjangoFileSystemProvider(UserStorageProvider): def __init__(self, authz_token, resource_id, context=None, directory=None, storage_resource_id=None, **kwargs): @@ -58,9 +55,9 @@ class DjangoFileSystemProvider(UserStorageProvider): { "name": d, "path": datastore.rel_path(dpath), + "resource_path": datastore.rel_path(dpath), "created_time": created_time, "size": size, - "hidden": dpath == TMP_INPUT_FILE_UPLOAD_DIR, } ) files_data = [] @@ -77,10 +74,9 @@ class DjangoFileSystemProvider(UserStorageProvider): { "name": f, "path": datastore.rel_path(full_path), - "resource_path": full_path, + "resource_path": datastore.rel_path(full_path), "created_time": created_time, "size": size, - "hidden": False, } ) return directories_data, files_data @@ -93,10 +89,9 @@ class DjangoFileSystemProvider(UserStorageProvider): { "name": os.path.basename(resource_path), "path": datastore.rel_path(full_path), - "resource_path": full_path, + "resource_path": datastore.rel_path(full_path), "created_time": created_time, "size": size, - "hidden": False, } ] else: diff --git a/airavata_django_portal_sdk/user_storage/backends/mft_provider.py b/airavata_django_portal_sdk/user_storage/backends/mft_provider.py index 6678572..488e0b0 100644 --- a/airavata_django_portal_sdk/user_storage/backends/mft_provider.py +++ b/airavata_django_portal_sdk/user_storage/backends/mft_provider.py @@ -1,152 +1,217 @@ +import logging +import os +from datetime import datetime + +import grpc + +from . import CredCommon_pb2, MFTApi_pb2, MFTApi_pb2_grpc from .base import UserStorageProvider +logger = logging.getLogger(__name__) + class MFTUserStorageProvider(UserStorageProvider): + def __init__(self, authz_token, resource_id, context=None, resource_token=None, mft_api_endpoint=None, mft_api_secure=False, resource_per_gateway=False, **kwargs): + super().__init__(authz_token, resource_id, context=context, **kwargs) + self.resource_token = resource_token + self.mft_api_endpoint = mft_api_endpoint + self.mft_api_secure = mft_api_secure + self.resource_per_gateway = resource_per_gateway + def exists(self, resource_path): - return super().exists(resource_path) -# with grpc.insecure_channel('localhost:7004') as channel: -# # remove trailing slash and figure out parent path -# # FIXME remove the hard coded /tmp path -# parent_path, child_path = os.path.split(f"/tmp/{path}".rstrip("/")) -# logger.debug(f"parent_path={parent_path}, child_path={child_path}") -# stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel) -# # Get metadata for parent directory and see if child_path exists -# request = MFTApi_pb2.FetchResourceMetadataRequest( -# resourceId="remote-ssh-dir-resource", -# resourceType="SCP", -# resourceToken="local-ssh-cred", -# resourceBackend="FILE", -# resourceCredentialBackend="FILE", -# targetAgentId="agent0", -# childPath=parent_path, -# mftAuthorizationToken="user token") -# response = stub.getDirectoryResourceMetadata(request) -# # if not child_path, then return True since the response was -# # successful and we just need to confirm the existence of the root dir -# if child_path == '': -# return True -# return child_path in map(lambda f: f.friendlyName, response.directories) + with grpc.insecure_channel(self.mft_api_endpoint) as channel: + child_path = self._get_child_path(resource_path) + # TODO: is this still needed? + # parent_path, child_path = os.path.split(f"/tmp/{resource_path}".rstrip("/")) + # get metadata for the parent path + if child_path is not None: + parent_path, child_path = os.path.split(child_path) + else: + parent_path = None + stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel) + # Get metadata for parent directory and see if child_path exists + request = MFTApi_pb2.FetchResourceMetadataRequest( + # resourceId="remote-ssh-dir-resource", + resourceId=self.resource_id, + resourceType="SCP", + # resourceToken="local-ssh-cred", + resourceToken=self.resource_token, + resourceBackend="FILE", + resourceCredentialBackend="FILE", + targetAgentId="agent0", + childPath=parent_path, + mftAuthorizationToken=self.auth_token, + ) + try: + response = stub.getDirectoryResourceMetadata(request) + except Exception: + # Could not find the parent path, so apparently doesn't exist + logger.warning(f"Could not get metadata for {parent_path} on {self.resource_id}") + return False + # if not child_path, then return True since the response was + # successful and we just need to confirm the existence of the root dir + if child_path is None: + return True + return child_path in map(lambda f: f.friendlyName, list(response.directories) + list(response.files)) def get_metadata(self, resource_path): - return super().get_metadata(resource_path) -# def listdir(self, request, path): -# # TODO setup resourceId, etc from __init__ arguments -# channel = grpc.insecure_channel('localhost:7004') -# stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel) -# request = MFTApi_pb2.FetchResourceMetadataRequest( -# resourceId="remote-ssh-dir-resource", -# resourceType="SCP", -# resourceToken="local-ssh-cred", -# resourceBackend="FILE", -# resourceCredentialBackend="FILE", -# targetAgentId="agent0", -# childPath=f"/tmp/{path}", -# mftAuthorizationToken="user token") -# response = stub.getDirectoryResourceMetadata(request) -# directories_data = [] -# for d in response.directories: - -# dpath = os.path.join(path, d.friendlyName) -# created_time = datetime.fromtimestamp(d.createdTime) -# # TODO MFT API doesn't report size -# size = 0 -# directories_data.append( -# { -# "name": d.friendlyName, -# "path": dpath, -# "created_time": created_time, -# "size": size, -# # TODO how to handle hidden directories or directories for -# # staging input file uploads -# "hidden": False -# } -# ) -# files_data = [] -# for f in response.files: -# user_rel_path = os.path.join(path, f.friendlyName) -# # TODO do we need to check for broken symlinks? -# created_time = datetime.fromtimestamp(f.createdTime) -# # TODO get the size as well -# size = 0 -# # full_path = datastore.path(request.user.username, user_rel_path) -# # TODO how do we register these as data products, do we need to? -# # data_product_uri = _get_data_product_uri(request, full_path) + with grpc.insecure_channel(self.mft_api_endpoint) as channel: + child_path = self._get_child_path(resource_path) + stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel) + request = MFTApi_pb2.FetchResourceMetadataRequest( + # resourceId="remote-ssh-dir-resource", + resourceId=self.resource_id, + resourceType="SCP", + # resourceToken="local-ssh-cred", + resourceToken=self.resource_token, + resourceBackend="FILE", + resourceCredentialBackend="FILE", + targetAgentId="agent0", + childPath=child_path, + mftAuthorizationToken=self.auth_token) + try: + logger.debug(f"getDirectoryResourceMetadata({request})") + response = stub.getDirectoryResourceMetadata(request) + logger.debug(f"getDirectoryResourceMetadata response={response}") + directories = response.directories + files = response.files + except Exception: + # if getting metadata for directory fails, try as file + # FIXME is there a better way to determine if directory or file? + logger.debug(f"getFileResourceMetadata({request})") + response = stub.getFileResourceMetadata(request) + logger.debug(f"getFileResourceMetadata response={response}") + directories = [] + files = [response] + directories_data = [] + for d in directories: -# # data_product = request.airavata_client.getDataProduct( -# # request.authz_token, data_product_uri) -# # mime_type = None -# # if 'mime-type' in data_product.productMetadata: -# # mime_type = data_product.productMetadata['mime-type'] -# files_data.append( -# { -# "name": f.friendlyName, -# "path": user_rel_path, -# "data-product-uri": None, -# "created_time": created_time, -# "mime_type": None, -# "size": size, -# "hidden": False, -# } -# ) -# return directories_data, files_data + dpath = os.path.join(resource_path, d.friendlyName) + created_time = datetime.fromtimestamp(d.createdTime) + # TODO MFT API doesn't report size + size = 0 + directories_data.append( + { + "name": d.friendlyName, + # path is the relative path, or at least, relative to given resource_path + "path": dpath, + # resource_path is the id or full path to the resource + "resource_path": d.resourcePath, + "created_time": created_time, + "size": size, + # TODO how to handle hidden directories or directories for + # staging input file uploads + "hidden": False + } + ) + files_data = [] + for f in files: + user_rel_path = os.path.join(resource_path, f.friendlyName) + # TODO do we need to check for broken symlinks? + created_time = datetime.fromtimestamp(f.createdTime) + size = f.resourceSize + # full_path = datastore.path(request.user.username, user_rel_path) + # TODO how do we register these as data products, do we need to? + # data_product_uri = _get_data_product_uri(request, full_path) -# def get_file(self, request, path): -# # FIXME remove hard coded /tmp path -# path = f"/tmp/{path}".rstrip("/") -# file_metadata = self._get_file(path) -# if file_metadata is not None: -# user_rel_path = os.path.join(path, file_metadata.friendlyName) -# created_time = datetime.fromtimestamp(file_metadata.createdTime) -# # TODO get the size as well -# size = 0 + # data_product = request.airavata_client.getDataProduct( + # request.authz_token, data_product_uri) + # mime_type = None + # if 'mime-type' in data_product.productMetadata: + # mime_type = data_product.productMetadata['mime-type'] + files_data.append( + { + "name": f.friendlyName, + "path": user_rel_path, + "resource_path": f.resourcePath, + "created_time": created_time, + "size": size, + "hidden": False, + } + ) + return directories_data, files_data -# return { -# "name": file_metadata.friendlyName, -# "path": user_rel_path, -# "data-product-uri": None, -# "created_time": created_time, -# "mime_type": None, -# "size": size, -# "hidden": False, -# } -# else: -# raise ObjectDoesNotExist("User storage file path does not exist") + def is_file(self, resource_path): + with grpc.insecure_channel(self.mft_api_endpoint) as channel: + child_path = self._get_child_path(resource_path) + stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel) + # Get metadata for parent directory and see if child_path exists + request = MFTApi_pb2.FetchResourceMetadataRequest( + # resourceId="remote-ssh-dir-resource", + resourceId=self.resource_id, + resourceType="SCP", + # resourceToken="local-ssh-cred", + resourceToken=self.resource_token, + resourceBackend="FILE", + resourceCredentialBackend="FILE", + targetAgentId="agent0", + childPath=child_path, + mftAuthorizationToken=self.auth_token, + ) + try: + stub.getFileResourceMetadata(request) + return True + except Exception: + # assume that is doesn't exist, or isn't a file + logger.warning(f"Could not get metadata for {child_path} on {self.resource_id}") + return False -# def _get_file(self, path): -# with grpc.insecure_channel('localhost:7004') as channel: -# stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel) -# # Get metadata for parent directory and see if child_path exists -# request = MFTApi_pb2.FetchResourceMetadataRequest( -# resourceId="remote-ssh-dir-resource", -# resourceType="SCP", -# resourceToken="local-ssh-cred", -# resourceBackend="FILE", -# resourceCredentialBackend="FILE", -# targetAgentId="agent0", -# childPath=path, -# mftAuthorizationToken="user token") -# try: -# # TODO is there a better way to check if file exists than catching exception? -# return stub.getFileResourceMetadata(request) -# except Exception: -# logger.exception(f"_get_file({path})") -# return None + def is_dir(self, resource_path): + with grpc.insecure_channel(self.mft_api_endpoint) as channel: + child_path = self._get_child_path(resource_path) + stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel) + # Get metadata for parent directory and see if child_path exists + request = MFTApi_pb2.FetchResourceMetadataRequest( + # resourceId="remote-ssh-dir-resource", + resourceId=self.resource_id, + resourceType="SCP", + # resourceToken="local-ssh-cred", + resourceToken=self.resource_token, + resourceBackend="FILE", + resourceCredentialBackend="FILE", + targetAgentId="agent0", + childPath=child_path, + mftAuthorizationToken=self.auth_token, + ) + try: + stub.getDirectoryResourceMetadata(request) + return True + except Exception: + # assume that it doesn't exist or isn't a file + logger.warning(f"Could not get metadata for {child_path} on {self.resource_id}") + return False -# def _get_download_url(self, path): + def _get_child_path(self, resource_path): + """Convert possibly relative child path into absolute path.""" + if not resource_path.startswith("/"): + # resource_path is relative, need to construct an absolute path + if self.resource_per_gateway: + resource_path = os.path.join(self.username, resource_path).rstrip("/") + # If there is no child path, just return none + if resource_path == '': + return None + logger.debug(f"figuring out resourcePath of {self.resource_id} ...") + with grpc.insecure_channel(self.mft_api_endpoint) as channel: + stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel) + request = MFTApi_pb2.FetchResourceMetadataRequest( + # resourceId="remote-ssh-dir-resource", + resourceId=self.resource_id, + resourceType="SCP", + # resourceToken="local-ssh-cred", + resourceToken=self.resource_token, + resourceBackend="FILE", + resourceCredentialBackend="FILE", + targetAgentId="agent0", + mftAuthorizationToken=self.auth_token) + response = stub.getDirectoryResourceMetadata(request) + logger.debug(f"metadata of {self.resource_id} is {response}") + return os.path.join(response.resourcePath, resource_path) + else: + # resource_path appears to be absolute path + return resource_path -# with grpc.insecure_channel('localhost:7004') as channel: -# stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel) -# download_request = MFTApi_pb2.HttpDownloadApiRequest(sourceStoreId="remote-ssh-storage", -# sourcePath="/tmp/a.txt", -# sourceToken="local-ssh-cred", -# sourceType="SCP", -# targetAgent="agent0", -# mftAuthorizationToken="") -# try: -# # TODO is there a better way to check if file exists than catching exception? -# # response stub.submitHttpDownload(request) -# pass -# except Exception: -# logger.exception(f"_get_file({path})") -# return None + @property + def auth_token(self): + """Instance of CredCommon.AuthToken wrapping user's access token.""" + return CredCommon_pb2.AuthToken(userTokenAuth=CredCommon_pb2.UserTokenAuth(token=self.access_token))
