This is an automated email from the ASF dual-hosted git repository. machristie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit cfc33747f98593fb3b237935045bff618ee11273 Author: Marcus Christie <[email protected]> AuthorDate: Thu Dec 13 14:49:08 2018 -0500 AIRAVATA-2616 Allow removing and changing input files --- django_airavata/apps/api/datastore.py | 14 ++++ django_airavata/apps/api/serializers.py | 24 ++++++- .../api/static/django_airavata_api/js/index.js | 3 + .../django_airavata_api/js/service_config.js | 10 +++ django_airavata/apps/api/urls.py | 3 + django_airavata/apps/api/views.py | 41 +++++++++--- .../js/components/experiment/DataProductViewer.vue | 40 ++++++++++++ .../js/components/experiment/ExperimentSummary.vue | 31 ++++----- .../experiment/input-editors/FileInputEditor.vue | 76 +++++++++++++++++++--- .../js/containers/EditExperimentContainer.vue | 2 +- 10 files changed, 205 insertions(+), 39 deletions(-) diff --git a/django_airavata/apps/api/datastore.py b/django_airavata/apps/api/datastore.py index aa4c17b..16576c0 100644 --- a/django_airavata/apps/api/datastore.py +++ b/django_airavata/apps/api/datastore.py @@ -72,6 +72,20 @@ def save(username, project_name, experiment_name, file): return data_product +def delete(data_product): + """Delete replica for data product in this data store.""" + if exists(data_product): + filepath = _get_replica_filepath(data_product) + try: + experiment_data_storage.delete(filepath) + except Exception as e: + logger.error("Unable to delete file {} for data product uri {}" + .format(filepath, data_product.productUri)) + raise + else: + raise ObjectDoesNotExist("Replica file does not exist") + + def get_experiment_dir(username, project_name, experiment_name): """Return an experiment directory (full path) for the given experiment.""" experiment_dir_name = os.path.join( diff --git a/django_airavata/apps/api/serializers.py b/django_airavata/apps/api/serializers.py index 13bbeb5..74cdf04 100644 --- a/django_airavata/apps/api/serializers.py +++ b/django_airavata/apps/api/serializers.py @@ -33,7 +33,7 @@ from airavata.model.appcatalog.parser.ttypes import Parser from airavata.model.appcatalog.storageresource.ttypes import ( StorageResourceDescription ) -from airavata.model.application.io.ttypes import InputDataObjectType +from airavata.model.application.io.ttypes import DataType, InputDataObjectType from airavata.model.credential.store.ttypes import ( CredentialSummary, SummaryType @@ -419,6 +419,24 @@ class ExperimentSerializer( request.authz_token, experiment.experimentId, ResourcePermissionType.WRITE) + def update(self, instance, validated_data): + result = super().update(instance, validated_data) + removed_input_files = self._find_removed_input_files( + instance.experimentInputs, result.experimentInputs) + result._removed_input_files = removed_input_files + return result + + def _find_removed_input_files(self, + old_experiment_inputs, + new_experiment_inputs): + old_input_data_product_uris = set( + inp.value for inp in old_experiment_inputs + if inp.type == DataType.URI) + new_input_data_product_uris = set( + inp.value for inp in new_experiment_inputs + if inp.type == DataType.URI) + return old_input_data_product_uris - new_input_data_product_uris + class DataReplicaLocationSerializer( thrift_utils.create_serializer_class(DataReplicaLocationModel)): @@ -432,6 +450,10 @@ class DataProductSerializer( lastModifiedTime = UTCPosixTimestampDateTimeField() replicaLocations = DataReplicaLocationSerializer(many=True) downloadURL = serializers.SerializerMethodField() + url = FullyEncodedHyperlinkedIdentityField( + view_name='django_airavata_api:data-product-detail', + lookup_field='productUri', + lookup_url_kwarg='product_uri') def get_downloadURL(self, data_product): """Getter for downloadURL field.""" diff --git a/django_airavata/apps/api/static/django_airavata_api/js/index.js b/django_airavata/apps/api/static/django_airavata_api/js/index.js index 34842a9..406494f 100644 --- a/django_airavata/apps/api/static/django_airavata_api/js/index.js +++ b/django_airavata/apps/api/static/django_airavata_api/js/index.js @@ -11,6 +11,7 @@ import BatchQueue from "./models/BatchQueue"; import BatchQueueResourcePolicy from "./models/BatchQueueResourcePolicy"; import CommandObject from "./models/CommandObject"; import ComputeResourcePolicy from "./models/ComputeResourcePolicy"; +import DataProduct from "./models/DataProduct"; import DataType from "./models/DataType"; import Experiment from "./models/Experiment"; import ExperimentState from "./models/ExperimentState"; @@ -63,6 +64,7 @@ exports.models = { BatchQueueResourcePolicy, CommandObject, ComputeResourcePolicy, + DataProduct, DataType, Experiment, ExperimentState, @@ -92,6 +94,7 @@ exports.services = { CloudJobSubmissionService, ComputeResourceService: ServiceFactory.service("ComputeResources"), CredentialSummaryService: ServiceFactory.service("CredentialSummaries"), + DataProductService: ServiceFactory.service("DataProducts"), ExperimentSearchService: ServiceFactory.service("ExperimentSearch"), ExperimentService: ServiceFactory.service("Experiments"), FullExperimentService, diff --git a/django_airavata/apps/api/static/django_airavata_api/js/service_config.js b/django_airavata/apps/api/static/django_airavata_api/js/service_config.js index e3adc09..9e7a7c8 100644 --- a/django_airavata/apps/api/static/django_airavata_api/js/service_config.js +++ b/django_airavata/apps/api/static/django_airavata_api/js/service_config.js @@ -4,6 +4,7 @@ import ApplicationModule from "./models/ApplicationModule"; import BatchQueue from "./models/BatchQueue"; import ComputeResourceDescription from "./models/ComputeResourceDescription"; import CredentialSummary from "./models/CredentialSummary"; +import DataProduct from "./models/DataProduct"; import Experiment from "./models/Experiment"; import ExperimentSummary from "./models/ExperimentSummary"; import GatewayResourceProfile from "./models/GatewayResourceProfile"; @@ -184,6 +185,15 @@ export default { ], modelClass: CredentialSummary }, + DataProducts: { + url: "/api/data-products/", + viewSet: [ + { + name: "retrieve" + } + ], + modelClass: DataProduct + }, Experiments: { url: "/api/experiments/", viewSet: [ diff --git a/django_airavata/apps/api/urls.py b/django_airavata/apps/api/urls.py index eb81baf..064bd1c 100644 --- a/django_airavata/apps/api/urls.py +++ b/django_airavata/apps/api/urls.py @@ -41,6 +41,9 @@ router.register(r'storage-preferences', views.StoragePreferenceViewSet, base_name='storage-preference') router.register(r'parsers', views.ParserViewSet, base_name='parser') +router.register(r'data-products', + views.DataProductViewSet, + base_name='data-product') app_name = 'django_airavata_api' urlpatterns = [ diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py index 52a959d..0ff7e61 100644 --- a/django_airavata/apps/api/views.py +++ b/django_airavata/apps/api/views.py @@ -159,15 +159,7 @@ class ExperimentViewSet(APIBackedViewSet): experiment = serializer.save( gatewayId=self.gateway_id, userName=self.username) - experiment.userConfigurationData.storageId = \ - settings.GATEWAY_DATA_STORE_RESOURCE_ID - # Set the experimentDataDir - project = self.request.airavata_client.getProject( - self.authz_token, experiment.projectId) - exp_dir = datastore.get_experiment_dir(self.username, - project.name, - experiment.experimentName) - experiment.userConfigurationData.experimentDataDir = exp_dir + self._set_storage_id_and_data_dir(experiment) experiment_id = self.request.airavata_client.createExperiment( self.authz_token, self.gateway_id, experiment) experiment.experimentId = experiment_id @@ -176,8 +168,28 @@ class ExperimentViewSet(APIBackedViewSet): experiment = serializer.save( gatewayId=self.gateway_id, userName=self.username) + # The project or exp name may have changed, so update the exp data dir + self._set_storage_id_and_data_dir(experiment) self.request.airavata_client.updateExperiment( self.authz_token, experiment.experimentId, experiment) + # Process experiment._removed_input_files, removing them from storage + for removed_input_file in experiment._removed_input_files: + data_product = self.request.airavata_client.getDataProduct( + self.authz_token, removed_input_file) + datastore.delete(data_product) + + def _set_storage_id_and_data_dir(self, experiment): + # Storage ID + experiment.userConfigurationData.storageId = \ + settings.GATEWAY_DATA_STORE_RESOURCE_ID + # Create experiment dir and set it on model + project = self.request.airavata_client.getProject( + self.authz_token, experiment.projectId) + exp_dir = datastore.get_experiment_dir(self.username, + project.name, + experiment.experimentName) + experiment.userConfigurationData.experimentDataDir = exp_dir + @detail_route(methods=['post']) def launch(self, request, experiment_id=None): @@ -611,6 +623,17 @@ class LocalDataMovementView(APIView): instance=data_movement).data) +class DataProductViewSet(mixins.RetrieveModelMixin, + GenericAPIBackedViewSet): + serializer_class = serializers.DataProductSerializer + lookup_field = 'product_uri' + lookup_value_regex = '.*' + + def get_instance(self, lookup_value): + return self.request.airavata_client.getDataProduct( + self.request.authz_token, lookup_value) + + @login_required def upload_input_file(request): try: diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/DataProductViewer.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/DataProductViewer.vue new file mode 100644 index 0000000..4f12b4f --- /dev/null +++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/DataProductViewer.vue @@ -0,0 +1,40 @@ +<template> + + <span v-if="dataProduct.downloadURL"> + <a :href="dataProduct.downloadURL"> + <i class="fa fa-download"></i> + {{ filename }} + </a> + </span> + <span v-else>{{ filename }}</span> +</template> + +<script> +import { models } from "django-airavata-api"; +export default { + name: "data-product-viewer", + props: { + dataProduct: { + type: models.DataProduct, + required: true + }, + inputFile: { + type: Boolean, + default: false + } + }, + computed: { + filename() { + if (this.inputFile) { + // productName captures the user provided name of the file, which may + // not match the name of the file on the storage system (for example, + // because of file name collision) + return this.dataProduct.productName; + } else { + return this.dataProduct.filename; + } + } + } +}; +</script> + diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentSummary.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentSummary.vue index af529f4..9b2892d 100644 --- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentSummary.vue +++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentSummary.vue @@ -33,15 +33,7 @@ <tr> <th scope="row">Outputs</th> <td> - <template v-for="output in localFullExperiment.outputDataProducts"> - <span v-if="output.downloadURL" :key="output.productUri"> - <a :href="output.downloadURL"> - <i class="fa fa-download"></i> - {{ output.filename }} - </a> - </span> - <span v-else :key="output.productUri">{{ output.filename }}</span> - </template> + <data-product-viewer v-for="output in localFullExperiment.outputDataProducts" :data-product="output" class="data-product" :key="output.productUri"/> </td> </tr> <!-- Going to leave this out for now --> @@ -130,15 +122,8 @@ <tr> <th scope="row">Inputs</th> <td> - <template v-for="input in localFullExperiment.inputDataProducts"> - <span v-if="input.downloadURL" :key="input.productUri"> - <a :href="input.downloadURL"> - <i class="fa fa-download"></i> - {{ input.filename }} - </a> - </span> - <span v-else :key="input.productUri">{{ input.filename }}</span> - </template> + <data-product-viewer v-for="input in localFullExperiment.inputDataProducts" + :data-product="input" :input-file="true" class="data-product" :key="input.productUri"/> </td> </tr> <tr> @@ -157,6 +142,7 @@ <script> import { models, services } from "django-airavata-api"; +import DataProductViewer from "./DataProductViewer.vue"; import moment from "moment"; @@ -177,7 +163,9 @@ export default { localFullExperiment: this.fullExperiment.clone() }; }, - components: {}, + components: { + DataProductViewer, + }, computed: { creationTime: function() { return moment(this.localFullExperiment.experiment.creationTime).fromNow(); @@ -224,5 +212,8 @@ export default { }; </script> -<style> +<style scoped> +.data-product + .data-product { + margin-left: 0.5em; +} </style> diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue index 964aa9f..9f5b496 100644 --- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue +++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue @@ -1,14 +1,74 @@ <template> - <b-form-file :id="id" v-model="data" - :placeholder="experimentInput.userFriendlyDescription" - :state="componentValidState" - @input="valueChanged"/> + <div> + <div + class="row" + v-if="isDataProductURI && dataProduct" + > + <div class="col mr-auto"> + <data-product-viewer :data-product="dataProduct" :input-file="true"/> + </div> + <div class="col-auto"> + <delete-link @delete="deleteDataProduct"> + Are you sure you want to delete input file {{ dataProduct.filename }}? + </delete-link> + </div> + </div> + <div class="row"> + <div class="col"> + + <b-form-file + :id="id" + v-model="data" + v-if="!isDataProductURI" + :placeholder="experimentInput.userFriendlyDescription" + :state="componentValidState" + @input="valueChanged" + /> + </div> + </div> + </div> </template> <script> -import {InputEditorMixin} from 'django-airavata-workspace-plugin-api' +import { services } from "django-airavata-api"; +import { InputEditorMixin } from "django-airavata-workspace-plugin-api"; +import DataProductViewer from "../DataProductViewer.vue"; +import { components } from "django-airavata-common-ui"; + export default { - name: 'file-input-editor', - mixins: [InputEditorMixin], -} + name: "file-input-editor", + mixins: [InputEditorMixin], + components: { + DataProductViewer, + "delete-link": components.DeleteLink + }, + computed: { + isDataProductURI() { + // Just assume that if the value is a string then it's a data product URL + return this.value && typeof this.value === "string"; + } + }, + data() { + return { + dataProduct: null + }; + }, + created() { + if (this.isDataProductURI) { + this.loadDataProduct(this.value); + } + }, + methods: { + loadDataProduct(dataProductURI) { + services.DataProductService.retrieve({ lookup: dataProductURI }).then( + dataProduct => (this.dataProduct = dataProduct) + ); + }, + deleteDataProduct() { + // Just null out the 'data' field. Backend will delete the file from storage + this.data = null; + this.valueChanged(); + } + } +}; </script> diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/EditExperimentContainer.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/EditExperimentContainer.vue index a90d214..a2b9f2a 100644 --- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/EditExperimentContainer.vue +++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/EditExperimentContainer.vue @@ -17,7 +17,7 @@ import ExperimentEditor from "../components/experiment/ExperimentEditor.vue"; import moment from "moment"; export default { - name: "create-experiment-container", + name: "edit-experiment-container", props: { experimentId: { type: String,
