This is an automated email from the ASF dual-hosted git repository. machristie pushed a commit to branch mft-integration in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit c687b45420a98ee9ef995408f876b60c1e8cdb4b Author: Marcus Christie <[email protected]> AuthorDate: Tue Mar 23 15:28:25 2021 -0400 WIP: UserStorageProvider abstraction --- airavata_django_portal_sdk/MFTApi_pb2.py | 1100 ++++++++++++++++++++ airavata_django_portal_sdk/MFTApi_pb2_grpc.py | 297 ++++++ airavata_django_portal_sdk/user_storage.py | 357 +++++-- .../user_storage_provider.py | 160 +++ requirements.txt | 9 +- 5 files changed, 1842 insertions(+), 81 deletions(-) diff --git a/airavata_django_portal_sdk/MFTApi_pb2.py b/airavata_django_portal_sdk/MFTApi_pb2.py new file mode 100644 index 0000000..1e89796 --- /dev/null +++ b/airavata_django_portal_sdk/MFTApi_pb2.py @@ -0,0 +1,1100 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: MFTApi.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='MFTApi.proto', + package='org.apache.airavata.mft.api.service', + syntax='proto3', + serialized_options=b'P\001', + create_key=_descriptor._internal_create_key, + serialized_pb=b'\n\x0cMFTApi.proto\x12#org.apache.airavata.mft.api.service\x1a\x1cgoogle/api/annotations.proto\x1a\x1bgoogle/protobuf/empty.proto\"\x9f\x04\n\x12TransferApiRequest\x12\x17\n\x0fsourceStorageId\x18\x01 \x01(\t\x12\x12\n\nsourcePath\x18\x02 \x01(\t\x12\x12\n\nsourceType\x18\x03 \x01(\t\x12\x13\n\x0bsourceToken\x18\x04 \x01(\t\x12\x1d\n\x15sourceResourceBackend\x18\x05 \x01(\t\x12\x1f\n\x17sourceCredentialBackend\x18\x06 \x01(\t\x12\x1c\n\x14\x64\x65stinationStorageId\x18\ [...] + , + dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,]) + + + + +_TRANSFERAPIREQUEST_TARGETAGENTSENTRY = _descriptor.Descriptor( + name='TargetAgentsEntry', + full_name='org.apache.airavata.mft.api.service.TransferApiRequest.TargetAgentsEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.TargetAgentsEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='value', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.TargetAgentsEntry.value', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=b'8\001', + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=605, + serialized_end=656, +) + +_TRANSFERAPIREQUEST = _descriptor.Descriptor( + name='TransferApiRequest', + full_name='org.apache.airavata.mft.api.service.TransferApiRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='sourceStorageId', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceStorageId', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='sourcePath', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourcePath', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='sourceType', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceType', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='sourceToken', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceToken', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='sourceResourceBackend', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceResourceBackend', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='sourceCredentialBackend', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceCredentialBackend', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='destinationStorageId', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destinationStorageId', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='destinationPath', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destinationPath', index=7, + number=8, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='destinationType', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destinationType', index=8, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='destinationToken', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destinationToken', index=9, + number=10, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='destResourceBackend', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destResourceBackend', index=10, + number=11, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='destCredentialBackend', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destCredentialBackend', index=11, + number=12, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='affinityTransfer', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.affinityTransfer', index=12, + number=13, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='targetAgents', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.targetAgents', index=13, + number=14, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.mftAuthorizationToken', index=14, + number=15, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_TRANSFERAPIREQUEST_TARGETAGENTSENTRY, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=113, + serialized_end=656, +) + + +_TRANSFERAPIRESPONSE = _descriptor.Descriptor( + name='TransferApiResponse', + full_name='org.apache.airavata.mft.api.service.TransferApiResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='transferId', full_name='org.apache.airavata.mft.api.service.TransferApiResponse.transferId', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=658, + serialized_end=699, +) + + +_HTTPUPLOADAPIREQUEST = _descriptor.Descriptor( + name='HttpUploadApiRequest', + full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='destinationStoreId', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.destinationStoreId', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='destinationPath', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.destinationPath', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='destinationToken', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.destinationToken', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='destinationType', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.destinationType', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='targetAgent', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.targetAgent', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.mftAuthorizationToken', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=702, + serialized_end=880, +) + + +_HTTPUPLOADAPIRESPONSE = _descriptor.Descriptor( + name='HttpUploadApiResponse', + full_name='org.apache.airavata.mft.api.service.HttpUploadApiResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='url', full_name='org.apache.airavata.mft.api.service.HttpUploadApiResponse.url', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='targetAgent', full_name='org.apache.airavata.mft.api.service.HttpUploadApiResponse.targetAgent', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=882, + serialized_end=939, +) + + +_HTTPDOWNLOADAPIREQUEST = _descriptor.Descriptor( + name='HttpDownloadApiRequest', + full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='sourceStoreId', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.sourceStoreId', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='sourcePath', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.sourcePath', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='sourceToken', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.sourceToken', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='sourceType', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.sourceType', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='targetAgent', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.targetAgent', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.mftAuthorizationToken', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=942, + serialized_end=1102, +) + + +_HTTPDOWNLOADAPIRESPONSE = _descriptor.Descriptor( + name='HttpDownloadApiResponse', + full_name='org.apache.airavata.mft.api.service.HttpDownloadApiResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='url', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiResponse.url', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='targetAgent', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiResponse.targetAgent', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1104, + serialized_end=1163, +) + + +_TRANSFERSTATEAPIREQUEST = _descriptor.Descriptor( + name='TransferStateApiRequest', + full_name='org.apache.airavata.mft.api.service.TransferStateApiRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='transferId', full_name='org.apache.airavata.mft.api.service.TransferStateApiRequest.transferId', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.TransferStateApiRequest.mftAuthorizationToken', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1165, + serialized_end=1241, +) + + +_TRANSFERSTATEAPIRESPONSE = _descriptor.Descriptor( + name='TransferStateApiResponse', + full_name='org.apache.airavata.mft.api.service.TransferStateApiResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='state', full_name='org.apache.airavata.mft.api.service.TransferStateApiResponse.state', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='updateTimeMils', full_name='org.apache.airavata.mft.api.service.TransferStateApiResponse.updateTimeMils', index=1, + number=2, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='percentage', full_name='org.apache.airavata.mft.api.service.TransferStateApiResponse.percentage', index=2, + number=3, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='description', full_name='org.apache.airavata.mft.api.service.TransferStateApiResponse.description', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1243, + serialized_end=1349, +) + + +_RESOURCEAVAILABILITYREQUEST = _descriptor.Descriptor( + name='ResourceAvailabilityRequest', + full_name='org.apache.airavata.mft.api.service.ResourceAvailabilityRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='resourceId', full_name='org.apache.airavata.mft.api.service.ResourceAvailabilityRequest.resourceId', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='resourceType', full_name='org.apache.airavata.mft.api.service.ResourceAvailabilityRequest.resourceType', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='resourceToken', full_name='org.apache.airavata.mft.api.service.ResourceAvailabilityRequest.resourceToken', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='resourceBackend', full_name='org.apache.airavata.mft.api.service.ResourceAvailabilityRequest.resourceBackend', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='resourceCredentialBackend', full_name='org.apache.airavata.mft.api.service.ResourceAvailabilityRequest.resourceCredentialBackend', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.ResourceAvailabilityRequest.mftAuthorizationToken', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1352, + serialized_end=1537, +) + + +_RESOURCEAVAILABILITYRESPONSE = _descriptor.Descriptor( + name='ResourceAvailabilityResponse', + full_name='org.apache.airavata.mft.api.service.ResourceAvailabilityResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='available', full_name='org.apache.airavata.mft.api.service.ResourceAvailabilityResponse.available', index=0, + number=1, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1539, + serialized_end=1588, +) + + +_FILEMETADATARESPONSE = _descriptor.Descriptor( + name='FileMetadataResponse', + full_name='org.apache.airavata.mft.api.service.FileMetadataResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='friendlyName', full_name='org.apache.airavata.mft.api.service.FileMetadataResponse.friendlyName', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='resourceSize', full_name='org.apache.airavata.mft.api.service.FileMetadataResponse.resourceSize', index=1, + number=2, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='createdTime', full_name='org.apache.airavata.mft.api.service.FileMetadataResponse.createdTime', index=2, + number=3, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='updateTime', full_name='org.apache.airavata.mft.api.service.FileMetadataResponse.updateTime', index=3, + number=4, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='md5sum', full_name='org.apache.airavata.mft.api.service.FileMetadataResponse.md5sum', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='resourcePath', full_name='org.apache.airavata.mft.api.service.FileMetadataResponse.resourcePath', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='parentResourceId', full_name='org.apache.airavata.mft.api.service.FileMetadataResponse.parentResourceId', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='parentResourceType', full_name='org.apache.airavata.mft.api.service.FileMetadataResponse.parentResourceType', index=7, + number=8, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1591, + serialized_end=1790, +) + + +_DIRECTORYMETADATARESPONSE = _descriptor.Descriptor( + name='DirectoryMetadataResponse', + full_name='org.apache.airavata.mft.api.service.DirectoryMetadataResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='friendlyName', full_name='org.apache.airavata.mft.api.service.DirectoryMetadataResponse.friendlyName', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='createdTime', full_name='org.apache.airavata.mft.api.service.DirectoryMetadataResponse.createdTime', index=1, + number=2, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='updateTime', full_name='org.apache.airavata.mft.api.service.DirectoryMetadataResponse.updateTime', index=2, + number=3, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='resourcePath', full_name='org.apache.airavata.mft.api.service.DirectoryMetadataResponse.resourcePath', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='parentResourceId', full_name='org.apache.airavata.mft.api.service.DirectoryMetadataResponse.parentResourceId', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='parentResourceType', full_name='org.apache.airavata.mft.api.service.DirectoryMetadataResponse.parentResourceType', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='directories', full_name='org.apache.airavata.mft.api.service.DirectoryMetadataResponse.directories', index=6, + number=7, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='files', full_name='org.apache.airavata.mft.api.service.DirectoryMetadataResponse.files', index=7, + number=8, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='lazyInitialized', full_name='org.apache.airavata.mft.api.service.DirectoryMetadataResponse.lazyInitialized', index=8, + number=9, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1793, + serialized_end=2143, +) + + +_FETCHRESOURCEMETADATAREQUEST = _descriptor.Descriptor( + name='FetchResourceMetadataRequest', + full_name='org.apache.airavata.mft.api.service.FetchResourceMetadataRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='resourceId', full_name='org.apache.airavata.mft.api.service.FetchResourceMetadataRequest.resourceId', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='resourceType', full_name='org.apache.airavata.mft.api.service.FetchResourceMetadataRequest.resourceType', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='resourceToken', full_name='org.apache.airavata.mft.api.service.FetchResourceMetadataRequest.resourceToken', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='resourceBackend', full_name='org.apache.airavata.mft.api.service.FetchResourceMetadataRequest.resourceBackend', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='resourceCredentialBackend', full_name='org.apache.airavata.mft.api.service.FetchResourceMetadataRequest.resourceCredentialBackend', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='targetAgentId', full_name='org.apache.airavata.mft.api.service.FetchResourceMetadataRequest.targetAgentId', index=5, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='childPath', full_name='org.apache.airavata.mft.api.service.FetchResourceMetadataRequest.childPath', index=6, + number=8, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.FetchResourceMetadataRequest.mftAuthorizationToken', index=7, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2146, + serialized_end=2374, +) + +_TRANSFERAPIREQUEST_TARGETAGENTSENTRY.containing_type = _TRANSFERAPIREQUEST +_TRANSFERAPIREQUEST.fields_by_name['targetAgents'].message_type = _TRANSFERAPIREQUEST_TARGETAGENTSENTRY +_DIRECTORYMETADATARESPONSE.fields_by_name['directories'].message_type = _DIRECTORYMETADATARESPONSE +_DIRECTORYMETADATARESPONSE.fields_by_name['files'].message_type = _FILEMETADATARESPONSE +DESCRIPTOR.message_types_by_name['TransferApiRequest'] = _TRANSFERAPIREQUEST +DESCRIPTOR.message_types_by_name['TransferApiResponse'] = _TRANSFERAPIRESPONSE +DESCRIPTOR.message_types_by_name['HttpUploadApiRequest'] = _HTTPUPLOADAPIREQUEST +DESCRIPTOR.message_types_by_name['HttpUploadApiResponse'] = _HTTPUPLOADAPIRESPONSE +DESCRIPTOR.message_types_by_name['HttpDownloadApiRequest'] = _HTTPDOWNLOADAPIREQUEST +DESCRIPTOR.message_types_by_name['HttpDownloadApiResponse'] = _HTTPDOWNLOADAPIRESPONSE +DESCRIPTOR.message_types_by_name['TransferStateApiRequest'] = _TRANSFERSTATEAPIREQUEST +DESCRIPTOR.message_types_by_name['TransferStateApiResponse'] = _TRANSFERSTATEAPIRESPONSE +DESCRIPTOR.message_types_by_name['ResourceAvailabilityRequest'] = _RESOURCEAVAILABILITYREQUEST +DESCRIPTOR.message_types_by_name['ResourceAvailabilityResponse'] = _RESOURCEAVAILABILITYRESPONSE +DESCRIPTOR.message_types_by_name['FileMetadataResponse'] = _FILEMETADATARESPONSE +DESCRIPTOR.message_types_by_name['DirectoryMetadataResponse'] = _DIRECTORYMETADATARESPONSE +DESCRIPTOR.message_types_by_name['FetchResourceMetadataRequest'] = _FETCHRESOURCEMETADATAREQUEST +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +TransferApiRequest = _reflection.GeneratedProtocolMessageType('TransferApiRequest', (_message.Message,), { + + 'TargetAgentsEntry' : _reflection.GeneratedProtocolMessageType('TargetAgentsEntry', (_message.Message,), { + 'DESCRIPTOR' : _TRANSFERAPIREQUEST_TARGETAGENTSENTRY, + '__module__' : 'MFTApi_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.api.service.TransferApiRequest.TargetAgentsEntry) + }) + , + 'DESCRIPTOR' : _TRANSFERAPIREQUEST, + '__module__' : 'MFTApi_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.api.service.TransferApiRequest) + }) +_sym_db.RegisterMessage(TransferApiRequest) +_sym_db.RegisterMessage(TransferApiRequest.TargetAgentsEntry) + +TransferApiResponse = _reflection.GeneratedProtocolMessageType('TransferApiResponse', (_message.Message,), { + 'DESCRIPTOR' : _TRANSFERAPIRESPONSE, + '__module__' : 'MFTApi_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.api.service.TransferApiResponse) + }) +_sym_db.RegisterMessage(TransferApiResponse) + +HttpUploadApiRequest = _reflection.GeneratedProtocolMessageType('HttpUploadApiRequest', (_message.Message,), { + 'DESCRIPTOR' : _HTTPUPLOADAPIREQUEST, + '__module__' : 'MFTApi_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.api.service.HttpUploadApiRequest) + }) +_sym_db.RegisterMessage(HttpUploadApiRequest) + +HttpUploadApiResponse = _reflection.GeneratedProtocolMessageType('HttpUploadApiResponse', (_message.Message,), { + 'DESCRIPTOR' : _HTTPUPLOADAPIRESPONSE, + '__module__' : 'MFTApi_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.api.service.HttpUploadApiResponse) + }) +_sym_db.RegisterMessage(HttpUploadApiResponse) + +HttpDownloadApiRequest = _reflection.GeneratedProtocolMessageType('HttpDownloadApiRequest', (_message.Message,), { + 'DESCRIPTOR' : _HTTPDOWNLOADAPIREQUEST, + '__module__' : 'MFTApi_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.api.service.HttpDownloadApiRequest) + }) +_sym_db.RegisterMessage(HttpDownloadApiRequest) + +HttpDownloadApiResponse = _reflection.GeneratedProtocolMessageType('HttpDownloadApiResponse', (_message.Message,), { + 'DESCRIPTOR' : _HTTPDOWNLOADAPIRESPONSE, + '__module__' : 'MFTApi_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.api.service.HttpDownloadApiResponse) + }) +_sym_db.RegisterMessage(HttpDownloadApiResponse) + +TransferStateApiRequest = _reflection.GeneratedProtocolMessageType('TransferStateApiRequest', (_message.Message,), { + 'DESCRIPTOR' : _TRANSFERSTATEAPIREQUEST, + '__module__' : 'MFTApi_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.api.service.TransferStateApiRequest) + }) +_sym_db.RegisterMessage(TransferStateApiRequest) + +TransferStateApiResponse = _reflection.GeneratedProtocolMessageType('TransferStateApiResponse', (_message.Message,), { + 'DESCRIPTOR' : _TRANSFERSTATEAPIRESPONSE, + '__module__' : 'MFTApi_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.api.service.TransferStateApiResponse) + }) +_sym_db.RegisterMessage(TransferStateApiResponse) + +ResourceAvailabilityRequest = _reflection.GeneratedProtocolMessageType('ResourceAvailabilityRequest', (_message.Message,), { + 'DESCRIPTOR' : _RESOURCEAVAILABILITYREQUEST, + '__module__' : 'MFTApi_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.api.service.ResourceAvailabilityRequest) + }) +_sym_db.RegisterMessage(ResourceAvailabilityRequest) + +ResourceAvailabilityResponse = _reflection.GeneratedProtocolMessageType('ResourceAvailabilityResponse', (_message.Message,), { + 'DESCRIPTOR' : _RESOURCEAVAILABILITYRESPONSE, + '__module__' : 'MFTApi_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.api.service.ResourceAvailabilityResponse) + }) +_sym_db.RegisterMessage(ResourceAvailabilityResponse) + +FileMetadataResponse = _reflection.GeneratedProtocolMessageType('FileMetadataResponse', (_message.Message,), { + 'DESCRIPTOR' : _FILEMETADATARESPONSE, + '__module__' : 'MFTApi_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.api.service.FileMetadataResponse) + }) +_sym_db.RegisterMessage(FileMetadataResponse) + +DirectoryMetadataResponse = _reflection.GeneratedProtocolMessageType('DirectoryMetadataResponse', (_message.Message,), { + 'DESCRIPTOR' : _DIRECTORYMETADATARESPONSE, + '__module__' : 'MFTApi_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.api.service.DirectoryMetadataResponse) + }) +_sym_db.RegisterMessage(DirectoryMetadataResponse) + +FetchResourceMetadataRequest = _reflection.GeneratedProtocolMessageType('FetchResourceMetadataRequest', (_message.Message,), { + 'DESCRIPTOR' : _FETCHRESOURCEMETADATAREQUEST, + '__module__' : 'MFTApi_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.api.service.FetchResourceMetadataRequest) + }) +_sym_db.RegisterMessage(FetchResourceMetadataRequest) + + +DESCRIPTOR._options = None +_TRANSFERAPIREQUEST_TARGETAGENTSENTRY._options = None + +_MFTAPISERVICE = _descriptor.ServiceDescriptor( + name='MFTApiService', + full_name='org.apache.airavata.mft.api.service.MFTApiService', + file=DESCRIPTOR, + index=0, + serialized_options=None, + create_key=_descriptor._internal_create_key, + serialized_start=2377, + serialized_end=3838, + methods=[ + _descriptor.MethodDescriptor( + name='submitTransfer', + full_name='org.apache.airavata.mft.api.service.MFTApiService.submitTransfer', + index=0, + containing_service=None, + input_type=_TRANSFERAPIREQUEST, + output_type=_TRANSFERAPIRESPONSE, + serialized_options=b'\202\323\344\223\002\024\"\022/v1.0/api/transfer', + create_key=_descriptor._internal_create_key, + ), + _descriptor.MethodDescriptor( + name='submitHttpUpload', + full_name='org.apache.airavata.mft.api.service.MFTApiService.submitHttpUpload', + index=1, + containing_service=None, + input_type=_HTTPUPLOADAPIREQUEST, + output_type=_HTTPUPLOADAPIRESPONSE, + serialized_options=b'\202\323\344\223\002\027\"\025/v1.0/api/http-upload', + create_key=_descriptor._internal_create_key, + ), + _descriptor.MethodDescriptor( + name='submitHttpDownload', + full_name='org.apache.airavata.mft.api.service.MFTApiService.submitHttpDownload', + index=2, + containing_service=None, + input_type=_HTTPDOWNLOADAPIREQUEST, + output_type=_HTTPDOWNLOADAPIRESPONSE, + serialized_options=b'\202\323\344\223\002\031\"\027/v1.0/api/http-download', + create_key=_descriptor._internal_create_key, + ), + _descriptor.MethodDescriptor( + name='getTransferStates', + full_name='org.apache.airavata.mft.api.service.MFTApiService.getTransferStates', + index=3, + containing_service=None, + input_type=_TRANSFERSTATEAPIREQUEST, + output_type=_TRANSFERSTATEAPIRESPONSE, + serialized_options=b'\202\323\344\223\002\033\022\031/v1.0/api/transfer/states', + create_key=_descriptor._internal_create_key, + ), + _descriptor.MethodDescriptor( + name='getTransferState', + full_name='org.apache.airavata.mft.api.service.MFTApiService.getTransferState', + index=4, + containing_service=None, + input_type=_TRANSFERSTATEAPIREQUEST, + output_type=_TRANSFERSTATEAPIRESPONSE, + serialized_options=b'\202\323\344\223\002\032\022\030/v1.0/api/transfer/state', + create_key=_descriptor._internal_create_key, + ), + _descriptor.MethodDescriptor( + name='getResourceAvailability', + full_name='org.apache.airavata.mft.api.service.MFTApiService.getResourceAvailability', + index=5, + containing_service=None, + input_type=_RESOURCEAVAILABILITYREQUEST, + output_type=_RESOURCEAVAILABILITYRESPONSE, + serialized_options=None, + create_key=_descriptor._internal_create_key, + ), + _descriptor.MethodDescriptor( + name='getFileResourceMetadata', + full_name='org.apache.airavata.mft.api.service.MFTApiService.getFileResourceMetadata', + index=6, + containing_service=None, + input_type=_FETCHRESOURCEMETADATAREQUEST, + output_type=_FILEMETADATARESPONSE, + serialized_options=b'\202\323\344\223\002\"\022 /v1.0/api/resource/metadata/file', + create_key=_descriptor._internal_create_key, + ), + _descriptor.MethodDescriptor( + name='getDirectoryResourceMetadata', + full_name='org.apache.airavata.mft.api.service.MFTApiService.getDirectoryResourceMetadata', + index=7, + containing_service=None, + input_type=_FETCHRESOURCEMETADATAREQUEST, + output_type=_DIRECTORYMETADATARESPONSE, + serialized_options=b'\202\323\344\223\002\'\022%/v1.0/api/resource/metadata/directory', + create_key=_descriptor._internal_create_key, + ), +]) +_sym_db.RegisterServiceDescriptor(_MFTAPISERVICE) + +DESCRIPTOR.services_by_name['MFTApiService'] = _MFTAPISERVICE + +# @@protoc_insertion_point(module_scope) diff --git a/airavata_django_portal_sdk/MFTApi_pb2_grpc.py b/airavata_django_portal_sdk/MFTApi_pb2_grpc.py new file mode 100644 index 0000000..6bb694c --- /dev/null +++ b/airavata_django_portal_sdk/MFTApi_pb2_grpc.py @@ -0,0 +1,297 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from . import MFTApi_pb2 as MFTApi__pb2 + + +class MFTApiServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.submitTransfer = channel.unary_unary( + '/org.apache.airavata.mft.api.service.MFTApiService/submitTransfer', + request_serializer=MFTApi__pb2.TransferApiRequest.SerializeToString, + response_deserializer=MFTApi__pb2.TransferApiResponse.FromString, + ) + self.submitHttpUpload = channel.unary_unary( + '/org.apache.airavata.mft.api.service.MFTApiService/submitHttpUpload', + request_serializer=MFTApi__pb2.HttpUploadApiRequest.SerializeToString, + response_deserializer=MFTApi__pb2.HttpUploadApiResponse.FromString, + ) + self.submitHttpDownload = channel.unary_unary( + '/org.apache.airavata.mft.api.service.MFTApiService/submitHttpDownload', + request_serializer=MFTApi__pb2.HttpDownloadApiRequest.SerializeToString, + response_deserializer=MFTApi__pb2.HttpDownloadApiResponse.FromString, + ) + self.getTransferStates = channel.unary_stream( + '/org.apache.airavata.mft.api.service.MFTApiService/getTransferStates', + request_serializer=MFTApi__pb2.TransferStateApiRequest.SerializeToString, + response_deserializer=MFTApi__pb2.TransferStateApiResponse.FromString, + ) + self.getTransferState = channel.unary_unary( + '/org.apache.airavata.mft.api.service.MFTApiService/getTransferState', + request_serializer=MFTApi__pb2.TransferStateApiRequest.SerializeToString, + response_deserializer=MFTApi__pb2.TransferStateApiResponse.FromString, + ) + self.getResourceAvailability = channel.unary_unary( + '/org.apache.airavata.mft.api.service.MFTApiService/getResourceAvailability', + request_serializer=MFTApi__pb2.ResourceAvailabilityRequest.SerializeToString, + response_deserializer=MFTApi__pb2.ResourceAvailabilityResponse.FromString, + ) + self.getFileResourceMetadata = channel.unary_unary( + '/org.apache.airavata.mft.api.service.MFTApiService/getFileResourceMetadata', + request_serializer=MFTApi__pb2.FetchResourceMetadataRequest.SerializeToString, + response_deserializer=MFTApi__pb2.FileMetadataResponse.FromString, + ) + self.getDirectoryResourceMetadata = channel.unary_unary( + '/org.apache.airavata.mft.api.service.MFTApiService/getDirectoryResourceMetadata', + request_serializer=MFTApi__pb2.FetchResourceMetadataRequest.SerializeToString, + response_deserializer=MFTApi__pb2.DirectoryMetadataResponse.FromString, + ) + + +class MFTApiServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def submitTransfer(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def submitHttpUpload(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def submitHttpDownload(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def getTransferStates(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def getTransferState(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def getResourceAvailability(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def getFileResourceMetadata(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def getDirectoryResourceMetadata(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_MFTApiServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'submitTransfer': grpc.unary_unary_rpc_method_handler( + servicer.submitTransfer, + request_deserializer=MFTApi__pb2.TransferApiRequest.FromString, + response_serializer=MFTApi__pb2.TransferApiResponse.SerializeToString, + ), + 'submitHttpUpload': grpc.unary_unary_rpc_method_handler( + servicer.submitHttpUpload, + request_deserializer=MFTApi__pb2.HttpUploadApiRequest.FromString, + response_serializer=MFTApi__pb2.HttpUploadApiResponse.SerializeToString, + ), + 'submitHttpDownload': grpc.unary_unary_rpc_method_handler( + servicer.submitHttpDownload, + request_deserializer=MFTApi__pb2.HttpDownloadApiRequest.FromString, + response_serializer=MFTApi__pb2.HttpDownloadApiResponse.SerializeToString, + ), + 'getTransferStates': grpc.unary_stream_rpc_method_handler( + servicer.getTransferStates, + request_deserializer=MFTApi__pb2.TransferStateApiRequest.FromString, + response_serializer=MFTApi__pb2.TransferStateApiResponse.SerializeToString, + ), + 'getTransferState': grpc.unary_unary_rpc_method_handler( + servicer.getTransferState, + request_deserializer=MFTApi__pb2.TransferStateApiRequest.FromString, + response_serializer=MFTApi__pb2.TransferStateApiResponse.SerializeToString, + ), + 'getResourceAvailability': grpc.unary_unary_rpc_method_handler( + servicer.getResourceAvailability, + request_deserializer=MFTApi__pb2.ResourceAvailabilityRequest.FromString, + response_serializer=MFTApi__pb2.ResourceAvailabilityResponse.SerializeToString, + ), + 'getFileResourceMetadata': grpc.unary_unary_rpc_method_handler( + servicer.getFileResourceMetadata, + request_deserializer=MFTApi__pb2.FetchResourceMetadataRequest.FromString, + response_serializer=MFTApi__pb2.FileMetadataResponse.SerializeToString, + ), + 'getDirectoryResourceMetadata': grpc.unary_unary_rpc_method_handler( + servicer.getDirectoryResourceMetadata, + request_deserializer=MFTApi__pb2.FetchResourceMetadataRequest.FromString, + response_serializer=MFTApi__pb2.DirectoryMetadataResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'org.apache.airavata.mft.api.service.MFTApiService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class MFTApiService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def submitTransfer(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/org.apache.airavata.mft.api.service.MFTApiService/submitTransfer', + MFTApi__pb2.TransferApiRequest.SerializeToString, + MFTApi__pb2.TransferApiResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def submitHttpUpload(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/org.apache.airavata.mft.api.service.MFTApiService/submitHttpUpload', + MFTApi__pb2.HttpUploadApiRequest.SerializeToString, + MFTApi__pb2.HttpUploadApiResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def submitHttpDownload(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/org.apache.airavata.mft.api.service.MFTApiService/submitHttpDownload', + MFTApi__pb2.HttpDownloadApiRequest.SerializeToString, + MFTApi__pb2.HttpDownloadApiResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def getTransferStates(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/org.apache.airavata.mft.api.service.MFTApiService/getTransferStates', + MFTApi__pb2.TransferStateApiRequest.SerializeToString, + MFTApi__pb2.TransferStateApiResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def getTransferState(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/org.apache.airavata.mft.api.service.MFTApiService/getTransferState', + MFTApi__pb2.TransferStateApiRequest.SerializeToString, + MFTApi__pb2.TransferStateApiResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def getResourceAvailability(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/org.apache.airavata.mft.api.service.MFTApiService/getResourceAvailability', + MFTApi__pb2.ResourceAvailabilityRequest.SerializeToString, + MFTApi__pb2.ResourceAvailabilityResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def getFileResourceMetadata(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/org.apache.airavata.mft.api.service.MFTApiService/getFileResourceMetadata', + MFTApi__pb2.FetchResourceMetadataRequest.SerializeToString, + MFTApi__pb2.FileMetadataResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def getDirectoryResourceMetadata(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/org.apache.airavata.mft.api.service.MFTApiService/getDirectoryResourceMetadata', + MFTApi__pb2.FetchResourceMetadataRequest.SerializeToString, + MFTApi__pb2.DirectoryMetadataResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/airavata_django_portal_sdk/user_storage.py b/airavata_django_portal_sdk/user_storage.py index 76d749a..5fb07c5 100644 --- a/airavata_django_portal_sdk/user_storage.py +++ b/airavata_django_portal_sdk/user_storage.py @@ -6,9 +6,11 @@ import mimetypes import os import shutil import warnings +from datetime import datetime from http import HTTPStatus from urllib.parse import quote, unquote, urlparse +import grpc import requests from airavata.model.data.replica.ttypes import ( DataProductModel, @@ -23,6 +25,7 @@ from django.core.files import File from django.core.files.move import file_move_safe from django.core.files.storage import FileSystemStorage +from . import MFTApi_pb2, MFTApi_pb2_grpc from .util import convert_iso8601_to_datetime logger = logging.getLogger(__name__) @@ -63,6 +66,7 @@ def move_from_filepath( name=None, content_type=None): "Move a file from filesystem into user's storage." + # TODO: deprecate this method username = request.user.username file_name = name if name is not None else os.path.basename(source_path) full_path = _Datastore().move_external( @@ -100,6 +104,8 @@ def save_input_file(request, file, name=None, content_type=None): def copy_input_file(request, data_product=None, data_product_uri=None): + # TODO: we could probably deprecate this as well, since we do an open/save + # to copy instead. Or at least, we don't need it in UserStorageProvider. if data_product is None: data_product = _get_data_product(request, data_product_uri) path = _get_replica_filepath(data_product) @@ -115,6 +121,7 @@ def copy_input_file(request, data_product=None, data_product_uri=None): def is_input_file(request, data_product=None, data_product_uri=None): + # TODO: don't need this in UserStorageProvider if data_product is None: data_product = _get_data_product(request, data_product_uri) if _is_remote_api(): @@ -134,6 +141,7 @@ def is_input_file(request, data_product=None, data_product_uri=None): def move_input_file(request, data_product=None, path=None, data_product_uri=None): + # TODO: don't need this in UserStorageProvider if data_product is None: data_product = _get_data_product(request, data_product_uri) source_path = _get_replica_filepath(data_product) @@ -152,6 +160,7 @@ def move_input_file(request, data_product=None, path=None, data_product_uri=None def move_input_file_from_filepath( request, source_path, name=None, content_type=None ): + # TODO: don't need this in UserStorageProvider "Move a file from filesystem into user's input file staging area." username = request.user.username file_name = name if name is not None else os.path.basename(source_path) @@ -218,7 +227,8 @@ def dir_exists(request, path): resp.raise_for_status() return resp.json()['isDir'] else: - return _Datastore().dir_exists(request.user.username, path) + user_storage_provider = MFTApiUserStorageProvider() + return user_storage_provider.dir_exists(request, path) def user_file_exists(request, path): @@ -311,32 +321,9 @@ def get_file(request, path): file['mime_type'] = file['mimeType'] file['data-product-uri'] = file['dataProductURI'] return file - datastore = _Datastore() - if datastore.exists(request.user.username, path): - created_time = datastore.get_created_time( - request.user.username, path) - size = datastore.size(request.user.username, path) - full_path = datastore.path(request.user.username, path) - data_product_uri = _get_data_product_uri(request, full_path) - dir_path, file_name = os.path.split(path) - data_product = request.airavata_client.getDataProduct( - request.authz_token, data_product_uri) - mime_type = None - if 'mime-type' in data_product.productMetadata: - mime_type = data_product.productMetadata['mime-type'] - - return { - 'name': full_path, - 'path': dir_path, - 'data-product-uri': data_product_uri, - 'created_time': created_time, - 'mime_type': mime_type, - 'size': size, - 'hidden': False - } - else: - raise ObjectDoesNotExist("User storage file path does not exist") + user_storage_provider = MFTApiUserStorageProvider() + return user_storage_provider.get_file(request, path) def delete(request, data_product=None, data_product_uri=None): @@ -388,58 +375,8 @@ def listdir(request, path): file['data-product-uri'] = file['dataProductURI'] return data['directories'], data['files'] - datastore = _Datastore() - if datastore.dir_exists(request.user.username, path): - directories, files = datastore.list_user_dir( - request.user.username, path) - directories_data = [] - for d in directories: - dpath = os.path.join(path, d) - created_time = datastore.get_created_time( - request.user.username, dpath) - size = datastore.size(request.user.username, dpath) - directories_data.append( - { - "name": d, - "path": dpath, - "created_time": created_time, - "size": size, - "hidden": dpath == TMP_INPUT_FILE_UPLOAD_DIR, - } - ) - files_data = [] - for f in files: - user_rel_path = os.path.join(path, f) - if not datastore.exists(request.user.username, user_rel_path): - logger.warning(f"listdir skipping {request.user.username}:{user_rel_path}, " - "does not exist (broken symlink?)") - continue - created_time = datastore.get_created_time( - request.user.username, user_rel_path - ) - size = datastore.size(request.user.username, user_rel_path) - full_path = datastore.path(request.user.username, user_rel_path) - data_product_uri = _get_data_product_uri(request, full_path) - - data_product = request.airavata_client.getDataProduct( - request.authz_token, data_product_uri) - mime_type = None - if 'mime-type' in data_product.productMetadata: - mime_type = data_product.productMetadata['mime-type'] - files_data.append( - { - "name": f, - "path": user_rel_path, - "data-product-uri": data_product_uri, - "created_time": created_time, - "mime_type": mime_type, - "size": size, - "hidden": False, - } - ) - return directories_data, files_data - else: - raise ObjectDoesNotExist("User storage path does not exist") + user_storage_provider = MFTApiUserStorageProvider() + return user_storage_provider.listdir(request, path) def list_experiment_dir(request, experiment_id, path=""): @@ -993,3 +930,267 @@ class _Datastore: if os.path.exists(fp): total_size += os.path.getsize(fp) return total_size + + +class UserStorageProvider: + def dir_exists(self, request, path): + raise NotImplementedError() + + def listdir(self, request, path): + raise NotImplementedError() + + def get_file(self, request, path): + raise NotImplementedError() + + +class FileSystemUserStorageProvider(UserStorageProvider): + def dir_exists(self, request, path): + return _Datastore().dir_exists(request.user.username, path) + + def listdir(self, request, path): + datastore = _Datastore() + if datastore.dir_exists(request.user.username, path): + directories, files = datastore.list_user_dir( + request.user.username, path) + directories_data = [] + for d in directories: + dpath = os.path.join(path, d) + created_time = datastore.get_created_time( + request.user.username, dpath) + size = datastore.size(request.user.username, dpath) + directories_data.append( + { + "name": d, + "path": dpath, + "created_time": created_time, + "size": size, + "hidden": dpath == TMP_INPUT_FILE_UPLOAD_DIR, + } + ) + files_data = [] + for f in files: + user_rel_path = os.path.join(path, f) + if not datastore.exists(request.user.username, user_rel_path): + logger.warning(f"listdir skipping {request.user.username}:{user_rel_path}, " + "does not exist (broken symlink?)") + continue + created_time = datastore.get_created_time( + request.user.username, user_rel_path + ) + size = datastore.size(request.user.username, user_rel_path) + full_path = datastore.path(request.user.username, user_rel_path) + data_product_uri = _get_data_product_uri(request, full_path) + + data_product = request.airavata_client.getDataProduct( + request.authz_token, data_product_uri) + mime_type = None + if 'mime-type' in data_product.productMetadata: + mime_type = data_product.productMetadata['mime-type'] + files_data.append( + { + "name": f, + "path": user_rel_path, + "data-product-uri": data_product_uri, + "created_time": created_time, + "mime_type": mime_type, + "size": size, + "hidden": False, + } + ) + return directories_data, files_data + else: + raise ObjectDoesNotExist("User storage path does not exist") + + def get_file(self, request, path): + + if _is_remote_api(): + resp = _call_remote_api(request, + "/user-storage/~/{path}", + path_params={"path": path}, + raise_for_status=False + ) + _raise_404(resp, "User storage file path does not exist") + data = resp.json() + if data["isDir"]: + raise Exception("User storage path is a directory, not a file") + file = data['files'][0] + file['created_time'] = convert_iso8601_to_datetime(file['createdTime']) + file['mime_type'] = file['mimeType'] + file['data-product-uri'] = file['dataProductURI'] + return file + datastore = _Datastore() + if datastore.exists(request.user.username, path): + created_time = datastore.get_created_time( + request.user.username, path) + size = datastore.size(request.user.username, path) + full_path = datastore.path(request.user.username, path) + data_product_uri = _get_data_product_uri(request, full_path) + dir_path, file_name = os.path.split(path) + + data_product = request.airavata_client.getDataProduct( + request.authz_token, data_product_uri) + mime_type = None + if 'mime-type' in data_product.productMetadata: + mime_type = data_product.productMetadata['mime-type'] + + return { + 'name': full_path, + 'path': dir_path, + 'data-product-uri': data_product_uri, + 'created_time': created_time, + 'mime_type': mime_type, + 'size': size, + 'hidden': False + } + else: + raise ObjectDoesNotExist("User storage file path does not exist") + + +class MFTApiUserStorageProvider(UserStorageProvider): + def __init__(self) -> None: + super().__init__() + + def dir_exists(self, request, path): + with grpc.insecure_channel('localhost:7004') as channel: + # remove trailing slash and figure out parent path + # FIXME remove the hard coded /tmp path + parent_path, child_path = os.path.split(f"/tmp/{path}".rstrip("/")) + logger.debug(f"parent_path={parent_path}, child_path={child_path}") + stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel) + # Get metadata for parent directory and see if child_path exists + request = MFTApi_pb2.FetchResourceMetadataRequest( + resourceId="remote-ssh-dir-resource", + resourceType="SCP", + resourceToken="local-ssh-cred", + resourceBackend="FILE", + resourceCredentialBackend="FILE", + targetAgentId="agent0", + childPath=parent_path, + mftAuthorizationToken="user token") + response = stub.getDirectoryResourceMetadata(request) + # if not child_path, then return True since the response was + # successful and we just need to confirm the existence of the root dir + if child_path == '': + return True + return child_path in map(lambda f: f.friendlyName, response.directories) + + def listdir(self, request, path): + # TODO setup resourceId, etc from __init__ arguments + channel = grpc.insecure_channel('localhost:7004') + stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel) + request = MFTApi_pb2.FetchResourceMetadataRequest( + resourceId="remote-ssh-dir-resource", + resourceType="SCP", + resourceToken="local-ssh-cred", + resourceBackend="FILE", + resourceCredentialBackend="FILE", + targetAgentId="agent0", + childPath=f"/tmp/{path}", + mftAuthorizationToken="user token") + response = stub.getDirectoryResourceMetadata(request) + directories_data = [] + for d in response.directories: + + dpath = os.path.join(path, d.friendlyName) + created_time = datetime.fromtimestamp(d.createdTime) + # TODO MFT API doesn't report size + size = 0 + directories_data.append( + { + "name": d.friendlyName, + "path": dpath, + "created_time": created_time, + "size": size, + # TODO how to handle hidden directories or directories for + # staging input file uploads + "hidden": False + } + ) + files_data = [] + for f in response.files: + user_rel_path = os.path.join(path, f.friendlyName) + # TODO do we need to check for broken symlinks? + created_time = datetime.fromtimestamp(f.createdTime) + # TODO get the size as well + size = 0 + # full_path = datastore.path(request.user.username, user_rel_path) + # TODO how do we register these as data products, do we need to? + # data_product_uri = _get_data_product_uri(request, full_path) + + # data_product = request.airavata_client.getDataProduct( + # request.authz_token, data_product_uri) + # mime_type = None + # if 'mime-type' in data_product.productMetadata: + # mime_type = data_product.productMetadata['mime-type'] + files_data.append( + { + "name": f.friendlyName, + "path": user_rel_path, + "data-product-uri": None, + "created_time": created_time, + "mime_type": None, + "size": size, + "hidden": False, + } + ) + return directories_data, files_data + + def get_file(self, request, path): + # FIXME remove hard coded /tmp path + path = f"/tmp/{path}".rstrip("/") + file_metadata = self._get_file(path) + if file_metadata is not None: + user_rel_path = os.path.join(path, file_metadata.friendlyName) + created_time = datetime.fromtimestamp(file_metadata.createdTime) + # TODO get the size as well + size = 0 + + return { + "name": file_metadata.friendlyName, + "path": user_rel_path, + "data-product-uri": None, + "created_time": created_time, + "mime_type": None, + "size": size, + "hidden": False, + } + else: + raise ObjectDoesNotExist("User storage file path does not exist") + + def _get_file(self, path): + with grpc.insecure_channel('localhost:7004') as channel: + stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel) + # Get metadata for parent directory and see if child_path exists + request = MFTApi_pb2.FetchResourceMetadataRequest( + resourceId="remote-ssh-dir-resource", + resourceType="SCP", + resourceToken="local-ssh-cred", + resourceBackend="FILE", + resourceCredentialBackend="FILE", + targetAgentId="agent0", + childPath=path, + mftAuthorizationToken="user token") + try: + # TODO is there a better way to check if file exists than catching exception? + return stub.getFileResourceMetadata(request) + except Exception: + logger.exception(f"_get_file({path})") + return None + + def _get_download_url(self, path): + + with grpc.insecure_channel('localhost:7004') as channel: + stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel) + download_request = MFTApi_pb2.HttpDownloadApiRequest(sourceStoreId="remote-ssh-storage", + sourcePath="/tmp/a.txt", + sourceToken="local-ssh-cred", + sourceType="SCP", + targetAgent="agent0", + mftAuthorizationToken="") + try: + # TODO is there a better way to check if file exists than catching exception? + # response stub.submitHttpDownload(request) + pass + except Exception: + logger.exception(f"_get_file({path})") + return None diff --git a/airavata_django_portal_sdk/user_storage_provider.py b/airavata_django_portal_sdk/user_storage_provider.py new file mode 100644 index 0000000..cc214db --- /dev/null +++ b/airavata_django_portal_sdk/user_storage_provider.py @@ -0,0 +1,160 @@ +import logging +import os +from datetime import datetime + +import grpc +from django.core.exceptions import ObjectDoesNotExist + +from . import MFTApi_pb2, MFTApi_pb2_grpc +# from .user_storage import (TMP_INPUT_FILE_UPLOAD_DIR, _Datastore, +# _get_data_product_uri) + +logger = logging.getLogger(__name__) + + +class UserStorageProvider: + def __init__(self, authz_token, *args, **kwargs): + self.authz_token = authz_token + + def save(self, authz_token, path, file, name=None, content_type=None): + raise NotImplementedError() + + def get_upload_url(self, authz_token, path): + raise NotImplementedError() + + def open(self, authz_token, resource_id=None): + raise NotImplementedError() + + def get_download_url(self, authz_token, resource_id=None): + raise NotImplementedError() + + def exists(self, authz_token, resource_id=None): + raise NotImplementedError() + + def is_file(self, authz_token, resource_id=None): + # TODO: is this needed if we have get_metadata? + raise NotImplementedError() + + def is_dir(self, authz_token, resource_id=None): + # TODO: is this needed if we have get_metadata? + raise NotImplementedError() + + def get_metadata(self, authz_token, resource_id=None): + raise NotImplementedError() + + def delete(self, authz_token, resource_id=None): + raise NotImplementedError() + + def update(self, authz_token, resource_id, file): + raise NotImplementedError() + + +class FileSystemUserStorageProvider(UserStorageProvider): + def listdir(self, request, path): + datastore = _Datastore() + if datastore.dir_exists(request.user.username, path): + directories, files = datastore.list_user_dir( + request.user.username, path) + directories_data = [] + for d in directories: + dpath = os.path.join(path, d) + created_time = datastore.get_created_time( + request.user.username, dpath) + size = datastore.size(request.user.username, dpath) + directories_data.append( + { + "name": d, + "path": dpath, + "created_time": created_time, + "size": size, + "hidden": dpath == TMP_INPUT_FILE_UPLOAD_DIR, + } + ) + files_data = [] + for f in files: + user_rel_path = os.path.join(path, f) + if not datastore.exists(request.user.username, user_rel_path): + logger.warning(f"listdir skipping {request.user.username}:{user_rel_path}, " + "does not exist (broken symlink?)") + continue + created_time = datastore.get_created_time( + request.user.username, user_rel_path + ) + size = datastore.size(request.user.username, user_rel_path) + full_path = datastore.path(request.user.username, user_rel_path) + data_product_uri = _get_data_product_uri(request, full_path) + + data_product = request.airavata_client.getDataProduct( + request.authz_token, data_product_uri) + mime_type = None + if 'mime-type' in data_product.productMetadata: + mime_type = data_product.productMetadata['mime-type'] + files_data.append( + { + "name": f, + "path": user_rel_path, + "data-product-uri": data_product_uri, + "created_time": created_time, + "mime_type": mime_type, + "size": size, + "hidden": False, + } + ) + return directories_data, files_data + else: + raise ObjectDoesNotExist("User storage path does not exist") + + +class MFTApiUserStorageProvider(UserStorageProvider): + def __init__(self) -> None: + super().__init__() + + def dir_exists(self, path): + channel = grpc.insecure_channel('localhost:7004') + stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel) + request = MFTApi_pb2.FetchResourceMetadataRequest( + resourceId="remote-ssh-dir-resource", + resourceType="SCP", + resourceToken="local-ssh-cred", + resourceBackend="FILE", + resourceCredentialBackend="FILE", + targetAgentId="agent0", + childPath=path, + mftAuthorizationToken="user token") + response = stub.getDirectoryResourceMetadata(request) + + def listdir(self, request, path): + # TODO setup resourceId, etc from __init__ arguments + channel = grpc.insecure_channel('localhost:7004') + stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel) + request = MFTApi_pb2.FetchResourceMetadataRequest( + resourceId="remote-ssh-dir-resource", + resourceType="SCP", + resourceToken="local-ssh-cred", + resourceBackend="FILE", + resourceCredentialBackend="FILE", + targetAgentId="agent0", + childPath=path, + mftAuthorizationToken="user token") + response = stub.getDirectoryResourceMetadata(request) + directories_data = [] + for d in response.directories: + + dpath = os.path.join(path, d.friendlyName) + created_time = datetime.fromtimestamp(d.createdTime) + # TODO MFT API doesn't report size + size = 0 + directories_data.append( + { + "name": d.friendlyName, + "path": dpath, + "created_time": created_time, + "size": size, + # TODO how to handle hidden directories or directories for + # staging input file uploads + "hidden": False + } + ) + # TODO implement + files_data = [] + return directories_data, files_data diff --git a/requirements.txt b/requirements.txt index 6866bc1..62b1685 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,21 @@ bcrypt==3.1.7 cffi==1.14.1 cryptography==3.0 -Django==1.11.29 +Django==2.2.17 +google-api-python-client==1.12.8 +grpcio-tools==1.34.1 +grpcio==1.34.1 oauthlib==3.1.0 paramiko==2.7.1 pycparser==2.20 PyNaCl==1.4.0 pysftp==0.2.9 pytz==2020.1 -requests==2.13.0 requests-oauthlib==0.7.0 +requests==2.18.4 scp==0.13.2 six==1.15.0 -thrift==0.10.0 thrift-connector==0.24 +thrift==0.10.0 -e git+https://github.com/apache/airavata.git@develop#egg=airavata_python_sdk&subdirectory=airavata-api/airavata-client-sdks/airavata-python-sdk
