This is an automated email from the ASF dual-hosted git repository. machristie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit af63d170dda382e014e6f9b446185789c2464ca9 Author: Marcus Christie <[email protected]> AuthorDate: Thu Jul 18 14:53:09 2019 -0400 AIRAVATA-3144 Max file upload limit --- django_airavata/apps/api/serializers.py | 4 ++ .../api/static/django_airavata_api/js/index.js | 1 + .../django_airavata_api/js/models/Settings.js | 9 +++ .../django_airavata_api/js/service_config.js | 17 ++++- django_airavata/apps/api/urls.py | 3 +- django_airavata/apps/api/views.py | 13 ++++ .../experiment/input-editors/FileInputEditor.vue | 78 +++++++++++++++++++--- .../js/containers/UserStorageContainer.vue | 77 ++++++++++++++++----- django_airavata/settings.py | 5 ++ django_airavata/uploadhandler.py | 22 ++++++ 10 files changed, 199 insertions(+), 30 deletions(-) diff --git a/django_airavata/apps/api/serializers.py b/django_airavata/apps/api/serializers.py index f7143b7..e9e4ade 100644 --- a/django_airavata/apps/api/serializers.py +++ b/django_airavata/apps/api/serializers.py @@ -928,3 +928,7 @@ class LogRecordSerializer(serializers.Serializer): message = serializers.CharField() details = StoredJSONField() stacktrace = serializers.ListField(child=serializers.CharField()) + + +class SettingsSerializer(serializers.Serializer): + fileUploadMaxFileSize = serializers.IntegerField() diff --git a/django_airavata/apps/api/static/django_airavata_api/js/index.js b/django_airavata/apps/api/static/django_airavata_api/js/index.js index 3f019eb..d4cc2c4 100644 --- a/django_airavata/apps/api/static/django_airavata_api/js/index.js +++ b/django_airavata/apps/api/static/django_airavata_api/js/index.js @@ -126,6 +126,7 @@ const services = { ProjectService: ServiceFactory.service("Projects"), SCPDataMovementService, ServiceFactory, + SettingsService: ServiceFactory.service("Settings"), SharedEntityService: ServiceFactory.service("SharedEntities"), SshJobSubmissionService, StoragePreferenceService: ServiceFactory.service("StoragePreferences"), diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/Settings.js b/django_airavata/apps/api/static/django_airavata_api/js/models/Settings.js new file mode 100644 index 0000000..1a5c3f3 --- /dev/null +++ b/django_airavata/apps/api/static/django_airavata_api/js/models/Settings.js @@ -0,0 +1,9 @@ +import BaseModel from "./BaseModel"; + +const FIELDS = ["fileUploadMaxFileSize"]; + +export default class Settings extends BaseModel { + constructor(data = {}) { + super(FIELDS, data); + } +} diff --git a/django_airavata/apps/api/static/django_airavata_api/js/service_config.js b/django_airavata/apps/api/static/django_airavata_api/js/service_config.js index 1855a75..82261a6 100644 --- a/django_airavata/apps/api/static/django_airavata_api/js/service_config.js +++ b/django_airavata/apps/api/static/django_airavata_api/js/service_config.js @@ -15,8 +15,10 @@ import Group from "./models/Group"; import GroupResourceProfile from "./models/GroupResourceProfile"; import IAMUserProfile from "./models/IAMUserProfile"; import LogRecord from "./models/LogRecord"; +import Notification from "./models/Notification"; import Parser from "./models/Parser"; import Project from "./models/Project"; +import Settings from "./models/Settings"; import SharedEntity from "./models/SharedEntity"; import StoragePreference from "./models/StoragePreference"; import StorageResourceDescription from "./models/StorageResourceDescription"; @@ -24,7 +26,6 @@ import UnverifiedEmailUserProfile from "./models/UnverifiedEmailUserProfile"; import UserProfile from "./models/UserProfile"; import UserStoragePath from "./models/UserStoragePath"; import WorkspacePreferences from "./models/WorkspacePreferences"; -import Notification from "./models/Notification"; /* examples: @@ -259,7 +260,7 @@ export default { url: "/api/log", methods: { send: { - url: '/api/log', + url: "/api/log", requestType: "post", bodyParams: { name: "data" @@ -289,6 +290,16 @@ export default { queryParams: ["limit", "offset"], modelClass: Project }, + Settings: { + url: "/api/settings/", + methods: { + get: { + url: "/api/settings/", + requestType: "get", + modelClass: Settings + } + } + }, SharedEntities: { url: "/api/shared-entities", viewSet: ["retrieve", "update"], @@ -358,5 +369,5 @@ export default { viewSet: true, pagination: false, modelClass: Notification - }, + } }; diff --git a/django_airavata/apps/api/urls.py b/django_airavata/apps/api/urls.py index d04b87f..61d8b6d 100644 --- a/django_airavata/apps/api/urls.py +++ b/django_airavata/apps/api/urls.py @@ -88,7 +88,8 @@ urlpatterns = [ name="experiment-statistics"), url(r'ack-notifications/<slug:id>/', views.AckNotificationViewSet.as_view(), name="ack-notifications"), url(r'ack-notifications/', views.AckNotificationViewSet.as_view(), name="ack-notifications"), - url(r'^log', views.LogRecordConsumer.as_view(), name='log') + url(r'^log', views.LogRecordConsumer.as_view(), name='log'), + url(r'^settings', views.SettingsAPIView.as_view(), name='settings'), ] if logger.isEnabledFor(logging.DEBUG): diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py index 1ae5f87..d24a562 100644 --- a/django_airavata/apps/api/views.py +++ b/django_airavata/apps/api/views.py @@ -1646,3 +1646,16 @@ class LogRecordConsumer(APIView): json.dumps(log_record['details'], indent=4), stacktrace)) return Response(serializer.data) + + +class SettingsAPIView(APIView): + serializer_class = serializers.SettingsSerializer + + def get(self, request, format=None): + data = { + 'fileUploadMaxFileSize': settings.FILE_UPLOAD_MAX_FILE_SIZE + } + serializer = self.serializer_class( + data, context={'request': request}) + return Response(serializer.data) + diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue index 5446dcd..6aec649 100644 --- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue +++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue @@ -39,16 +39,26 @@ class="d-flex align-items-baseline" v-if="!isSelectingFile && !isDataProductURI" > - <b-button @click="isSelectingFile=true" class="input-file-option">Select file from storage</b-button> + <b-button + @click="isSelectingFile=true" + class="input-file-option" + >Select file from storage</b-button> <span class="text-muted mx-3">OR</span> - <b-form-file - :id="id" - v-model="file" - v-if="!isDataProductURI" - placeholder="Upload file" - @input="fileChanged" + <b-form-group + :description="maxFileUploadSizeMessage" + :state="fileUploadState" + :invalid-feedback="fileUploadInvalidFeedback" class="input-file-option" - /> + > + <b-form-file + :id="id" + v-model="file" + v-if="!isDataProductURI" + placeholder="Upload file" + @input="fileChanged" + :state="fileUploadState" + /> + </b-form-group> </div> </div> </template> @@ -70,7 +80,11 @@ export default { computed: { isDataProductURI() { // Just assume that if the value is a string then it's a data product URL - return this.value && typeof this.value === "string" && this.value.startsWith("airavata-dp://"); + return ( + this.value && + typeof this.value === "string" && + this.value.startsWith("airavata-dp://") + ); }, // When used in the MultiFileInputEditor, don't allow selecting the same // file more than once. This computed property creates an array of already @@ -84,19 +98,61 @@ export default { } else { return []; } + }, + maxFileUploadSizeMB() { + return this.settings + ? this.settings.fileUploadMaxFileSize / 1024 / 1024 + : 0; + }, + maxFileUploadSizeMessage() { + if (this.maxFileUploadSizeMB) { + return ( + "Max file upload size is " + + Math.round(this.maxFileUploadSizeMB) + + " MB" + ); + } else { + return null; + } + }, + fileTooLarge() { + return ( + this.settings && + this.settings.fileUploadMaxFileSize && + this.file && + this.file.size > this.settings.fileUploadMaxFileSize + ); + }, + fileUploadState() { + if (this.fileTooLarge) { + return false; + } else { + return null; + } + }, + fileUploadInvalidFeedback() { + if (this.fileTooLarge) { + return ( + "File selected is larger than " + this.maxFileUploadSizeMB + " MB" + ); + } else { + return null; + } } }, data() { return { dataProduct: null, file: null, - isSelectingFile: false + isSelectingFile: false, + settings: null }; }, created() { if (this.isDataProductURI) { this.loadDataProduct(this.value); } + services.SettingsService.get().then(s => (this.settings = s)); }, methods: { loadDataProduct(dataProductURI) { @@ -126,7 +182,7 @@ export default { .catch(utils.FetchUtils.reportError); }, fileChanged() { - if (this.file) { + if (this.file && !this.fileTooLarge) { let data = new FormData(); data.append("file", this.file); this.$emit("uploadstart"); diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/UserStorageContainer.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/UserStorageContainer.vue index 7e39fd2..567de2e 100644 --- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/UserStorageContainer.vue +++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/UserStorageContainer.vue @@ -12,13 +12,20 @@ </div> <div class="row"> <div class="col"> - <b-form-file - v-model="file" - ref="file-input" - placeholder="Add file" - @input="fileChanged" - class="mb-2" - ></b-form-file> + <b-form-group + :description="maxFileUploadSizeMessage" + :state="fileUploadState" + :invalid-feedback="fileUploadInvalidFeedback" + > + <b-form-file + v-model="file" + ref="file-input" + placeholder="Add file" + @input="fileChanged" + class="mb-2" + :state="fileUploadState" + ></b-form-file> + </b-form-group> </div> <div class="col"> <b-input-group> @@ -67,13 +74,54 @@ export default { }, username() { return session.Session.username; + }, + maxFileUploadSizeMB() { + return this.settings + ? this.settings.fileUploadMaxFileSize / 1024 / 1024 + : 0; + }, + maxFileUploadSizeMessage() { + if (this.maxFileUploadSizeMB) { + return ( + "Max file upload size is " + + Math.round(this.maxFileUploadSizeMB) + + " MB" + ); + } else { + return null; + } + }, + fileTooLarge() { + return ( + this.settings && + this.settings.fileUploadMaxFileSize && + this.file && + this.file.size > this.settings.fileUploadMaxFileSize + ); + }, + fileUploadState() { + if (this.fileTooLarge) { + return false; + } else { + return null; + } + }, + fileUploadInvalidFeedback() { + if (this.fileTooLarge) { + return ( + "File selected is larger than " + this.maxFileUploadSizeMB + " MB" + ); + } else { + return null; + } } }, data() { return { userStoragePath: null, file: null, - dirName: null + dirName: null, + settings: null }; }, methods: { @@ -105,7 +153,7 @@ export default { ); }, fileChanged() { - if (this.file) { + if (this.file && !this.fileTooLarge) { let data = new FormData(); data.append("file", this.file); utils.FetchUtils.post( @@ -125,12 +173,10 @@ export default { newDirPath = newDirPath + "/"; } newDirPath = newDirPath + this.dirName; - utils.FetchUtils.post("/api/user-storage/" + newDirPath).then( - () => { - this.dirName = null; - this.loadUserStoragePath(this.storagePath); - } - ); + utils.FetchUtils.post("/api/user-storage/" + newDirPath).then(() => { + this.dirName = null; + this.loadUserStoragePath(this.storagePath); + }); } }, deleteDir(path) { @@ -156,6 +202,7 @@ export default { } else { this.loadUserStoragePath(this.storagePath); } + services.SettingsService.get().then(s => (this.settings = s)); }, watch: { $route() { diff --git a/django_airavata/settings.py b/django_airavata/settings.py index d8bb59e..a72317a 100644 --- a/django_airavata/settings.py +++ b/django_airavata/settings.py @@ -214,6 +214,11 @@ MEDIA_URL = '/media/' # Data storage FILE_UPLOAD_DIRECTORY_PERMISSIONS = 0o777 +FILE_UPLOAD_MAX_FILE_SIZE = 64 * 1024 * 1024 # 64 MB +FILE_UPLOAD_HANDLERS = [ + 'django.core.files.uploadhandler.MemoryFileUploadHandler', + 'django_airavata.uploadhandler.MaxFileSizeTemporaryFileUploadHandler', +] # Django REST Framework configuration REST_FRAMEWORK = { diff --git a/django_airavata/uploadhandler.py b/django_airavata/uploadhandler.py new file mode 100644 index 0000000..dfa28a5 --- /dev/null +++ b/django_airavata/uploadhandler.py @@ -0,0 +1,22 @@ +from django.conf import settings +from django.core.files.uploadhandler import ( + StopUpload, + TemporaryFileUploadHandler +) + + +class MaxFileSizeTemporaryFileUploadHandler(TemporaryFileUploadHandler): + + def handle_raw_input(self, + input_data, + META, + content_length, + boundary, + encoding=None): + """ + Use the content_length to enforce max size limit. + """ + # Check the content-length header to see if we should + # If the post is too large, we cannot use the Memory handler. + if content_length > settings.FILE_UPLOAD_MAX_FILE_SIZE: + raise StopUpload
