This is an automated email from the ASF dual-hosted git repository. machristie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit 0bdc7219f37718055bde53f4cb0a68522272b87c Author: Marcus Christie <[email protected]> AuthorDate: Thu Sep 5 18:51:11 2019 -0400 AIRAVATA-3081 Moving tus upload file into data storage --- django_airavata/apps/api/data_products_helper.py | 20 ++++++++++++++++++++ django_airavata/apps/api/datastore.py | 15 +++++++++++++++ django_airavata/apps/api/views.py | 17 +++++++++-------- 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/django_airavata/apps/api/data_products_helper.py b/django_airavata/apps/api/data_products_helper.py index 1614acd..91e1411 100644 --- a/django_airavata/apps/api/data_products_helper.py +++ b/django_airavata/apps/api/data_products_helper.py @@ -28,6 +28,16 @@ def save(request, path, file, name=None): return data_product +def move_from_filepath(request, source_path, target_path, name=None): + "Move a file from filesystem into user's storage." + username = request.user.username + file_name = name if name is not None else os.path.basename(source_path) + full_path = datastore.move_external( + source_path, username, target_path, file_name) + data_product = _save_data_product(request, full_path, name=file_name) + return data_product + + def save_input_file_upload(request, file, name=None): """Save input file in staging area for input file uploads.""" username = request.user.username @@ -72,6 +82,16 @@ def move_input_file_upload(request, data_product, path): return data_product +def move_input_file_upload_from_filepath(request, source_path, name=None): + "Move a file from filesystem into user's input file staging area." + username = request.user.username + file_name = name if name is not None else os.path.basename(source_path) + full_path = datastore.move_external( + source_path, username, TMP_INPUT_FILE_UPLOAD_DIR, file_name) + data_product = _save_data_product(request, full_path, name=file_name) + return data_product + + def open(request, data_product): "Return file object for replica if it exists in user storage." path = _get_replica_filepath(data_product) diff --git a/django_airavata/apps/api/datastore.py b/django_airavata/apps/api/datastore.py index 76c93cb..8d62750 100644 --- a/django_airavata/apps/api/datastore.py +++ b/django_airavata/apps/api/datastore.py @@ -55,6 +55,21 @@ def move(source_username, source_path, target_username, target_dir, file_name): return target_full_path +def move_external(external_path, target_username, target_dir, file_name): + user_data_storage = _user_data_storage(target_username) + # Make file_name a valid filename + target_path = os.path.join(target_dir, + user_data_storage.get_valid_name(file_name)) + # Get available file path: if there is an existing file at target_path + # create a uniquely named path + target_path = user_data_storage.get_available_name(target_path) + if not exists(target_username, target_dir): + create_user_dir(target_username, target_dir) + target_full_path = path_(target_username, target_path) + file_move_safe(external_path, target_full_path) + return target_full_path + + def create_user_dir(username, path): user_data_storage = _user_data_storage(username) if not user_data_storage.exists(path): diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py index 9b0246b..2e35642 100644 --- a/django_airavata/apps/api/views.py +++ b/django_airavata/apps/api/views.py @@ -919,12 +919,13 @@ def tus_upload_finish(request): log.debug(f"upload_bin_path={upload_bin_path}") upload_info_path = os.path.join(settings.TUS_DATA_DIR, f"{upload_uuid}.info") - with open(upload_info_path) as upload_info_file, \ - open(upload_bin_path, "rb") as upload_file: + with open(upload_info_path) as upload_info_file: upload_info = json.load(upload_info_file) filename = upload_info['MetaData']['filename'] - data_product = data_products_helper.save_input_file_upload( - request, upload_file, name=filename) + data_product = data_products_helper\ + .move_input_file_upload_from_filepath( + request, upload_bin_path, name=filename) + os.remove(upload_info_path) serializer = serializers.DataProductSerializer( data_product, context={'request': request}) return JsonResponse({'uploaded': True, @@ -1471,12 +1472,12 @@ class UserStoragePathView(APIView): log.debug(f"upload_bin_path={upload_bin_path}") upload_info_path = os.path.join(settings.TUS_DATA_DIR, f"{upload_uuid}.info") - with open(upload_info_path) as upload_info_file, \ - open(upload_bin_path, "rb") as upload_file: + with open(upload_info_path) as upload_info_file: upload_info = json.load(upload_info_file) filename = upload_info['MetaData']['filename'] - data_product = data_products_helper.save( - request, path, upload_file, name=filename) + data_product = data_products_helper.move_from_filepath( + request, upload_bin_path, path, name=filename) + os.remove(upload_info_path) return self._create_response(request, path, uploaded=data_product) def delete(self, request, path="/", format=None):
