This is an automated email from the ASF dual-hosted git repository. machristie pushed a commit to branch airavata-3016 in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit eeb471b5cbcb6189eec758425a961344d450b565 Author: Marcus Christie <[email protected]> AuthorDate: Sun May 26 15:06:09 2019 -0400 AIRAVATA-3032 Move tmp input file uploads to data dir at launch --- django_airavata/apps/api/data_products_helper.py | 16 +++++++++ django_airavata/apps/api/datastore.py | 26 +++++++++++--- django_airavata/apps/api/views.py | 43 ++++++++++++++++++++++-- 3 files changed, 78 insertions(+), 7 deletions(-) diff --git a/django_airavata/apps/api/data_products_helper.py b/django_airavata/apps/api/data_products_helper.py index eb7f0ad..b085e87 100644 --- a/django_airavata/apps/api/data_products_helper.py +++ b/django_airavata/apps/api/data_products_helper.py @@ -36,6 +36,22 @@ def save_input_file_upload(request, file): return data_product +def is_input_file_upload(request, data_product): + path = _get_replica_filepath(data_product) + rel_path = datastore.rel_path(request.user.username, path) + return os.path.dirname(rel_path) == TMP_INPUT_FILE_UPLOAD_DIR + + +def move_input_file_upload(request, data_product, path): + source_path = _get_replica_filepath(data_product) + file_name = data_product.productName + target_path = os.path.join(path, file_name) + full_path = datastore.move(request.user.username, source_path, target_path) + _delete_data_product(request, source_path) + data_product = _save_data_product(request, full_path, name=file_name) + return data_product + + def open(request, data_product): "Return file object for replica if it exists in user storage." path = _get_replica_filepath(data_product) diff --git a/django_airavata/apps/api/datastore.py b/django_airavata/apps/api/datastore.py index 124949f..2fc0965 100644 --- a/django_airavata/apps/api/datastore.py +++ b/django_airavata/apps/api/datastore.py @@ -4,6 +4,7 @@ import shutil from django.conf import settings from django.core.exceptions import ObjectDoesNotExist, SuspiciousFileOperation +from django.core.files.move import file_move_safe from django.core.files.storage import FileSystemStorage experiment_data_storage = FileSystemStorage( @@ -40,6 +41,13 @@ def save(username, path, file): return input_file_fullpath +def move(username, source_path, target_path): + source_full_path = path_(username, source_path) + target_full_path = path_(username, target_path) + file_move_safe(source_full_path, target_full_path) + return target_full_path + + def create_user_dir(username, path): user_data_storage = _user_data_storage(username) if not user_data_storage.exists(path): @@ -73,7 +81,6 @@ def delete_dir(username, path): raise ObjectDoesNotExist("File path does not exist: {}".format(path)) -# TODO: update this to just return an available experiment directory name def get_experiment_dir( username, project_name=None, @@ -85,16 +92,22 @@ def get_experiment_dir( experiment_dir_name = os.path.join( user_experiment_data_storage.get_valid_name(project_name), user_experiment_data_storage.get_valid_name(experiment_name)) + # Since there may already be another experiment with the same name in + # this project, we need to check for available name + experiment_dir_name = user_experiment_data_storage.get_available_name( + experiment_dir_name) experiment_dir = user_experiment_data_storage.path(experiment_dir_name) else: # path can be relative to the user's storage space or absolute (as long # as it is still inside the user's storage space) + # if path is passed in, assumption is that it has already been created user_experiment_data_storage = _user_data_storage(username) experiment_dir = user_experiment_data_storage.path(path) if not user_experiment_data_storage.exists(experiment_dir): - os.makedirs(experiment_dir, - mode=user_experiment_data_storage.directory_permissions_mode) - # os.mkdir mode isn't always respected so need to chmod to be sure + os.makedirs( + experiment_dir, + mode=user_experiment_data_storage.directory_permissions_mode) + # os.makedirs mode isn't always respected so need to chmod to be sure os.chmod(experiment_dir, mode=user_experiment_data_storage.directory_permissions_mode) return experiment_dir @@ -137,6 +150,11 @@ def path(username, file_path): return path_(username, file_path) +def rel_path(username, file_path): + full_path = path_(username, file_path) + return os.path.relpath(full_path, path_(username, "")) + + def path_(username, file_path): user_data_storage = _user_data_storage(username) return user_data_storage.path(file_path) diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py index 819b16b..4ec398e 100644 --- a/django_airavata/apps/api/views.py +++ b/django_airavata/apps/api/views.py @@ -179,7 +179,6 @@ class ExperimentViewSet(APIBackedViewSet): experiment = serializer.save( gatewayId=self.gateway_id, userName=self.username) - self._set_storage_id_and_data_dir(experiment) experiment_id = self.request.airavata_client.createExperiment( self.authz_token, self.gateway_id, experiment) self._update_most_recent_project(experiment.projectId) @@ -189,8 +188,6 @@ class ExperimentViewSet(APIBackedViewSet): experiment = serializer.save( gatewayId=self.gateway_id, userName=self.username) - # The project or exp name may have changed, so update the exp data dir - # self._set_storage_id_and_data_dir(experiment) self.request.airavata_client.updateExperiment( self.authz_token, experiment.experimentId, experiment) self._update_most_recent_project(experiment.projectId) @@ -215,9 +212,49 @@ class ExperimentViewSet(APIBackedViewSet): path=experiment.userConfigurationData.experimentDataDir) experiment.userConfigurationData.experimentDataDir = exp_dir + def _move_tmp_input_file_uploads_to_data_dir(self, experiment): + exp_data_dir = experiment.userConfigurationData.experimentDataDir + for experiment_input in experiment.experimentInputs: + if experiment_input.type == DataType.URI: + experiment_input.value = self._move_if_tmp_input_file_upload( + experiment_input.value, exp_data_dir) + elif experiment_input.type == DataType.URI_COLLECTION: + data_product_uris = experiment_input.value.split( + ",") if experiment_input.value else [] + moved_data_product_uris = [] + for data_product_uri in data_product_uris: + moved_data_product_uris.append( + self._move_if_tmp_input_file_upload(data_product_uri, + exp_data_dir)) + experiment_input.value = ",".join(moved_data_product_uris) + + def _move_if_tmp_input_file_upload( + self, data_product_uri, experiment_data_dir): + """ + Conditionally moves tmp input file to data dir and returns new dp URI. + """ + data_product = self.request.airavata_client.getDataProduct( + self.authz_token, data_product_uri) + if data_products_helper.is_input_file_upload( + self.request, data_product): + moved_data_product = \ + data_products_helper.move_input_file_upload( + self.request, + data_product, + experiment_data_dir) + return moved_data_product.productUri + else: + return data_product_uri + @detail_route(methods=['post']) def launch(self, request, experiment_id=None): try: + experiment = request.airavata_client.getExperiment( + self.authz_token, experiment_id) + self._set_storage_id_and_data_dir(experiment) + self._move_tmp_input_file_uploads_to_data_dir(experiment) + request.airavata_client.updateExperiment( + self.authz_token, experiment_id, experiment) request.airavata_client.launchExperiment( request.authz_token, experiment_id, self.gateway_id) return Response({'success': True})
