This is an automated email from the ASF dual-hosted git repository. machristie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit 6f9d189033e06df0e1541d56d2736c48df80e8ff Author: Marcus Christie <[email protected]> AuthorDate: Thu Jul 2 18:01:35 2020 -0400 AIRAVATA-3342 Changed SDK to Django app so it could define model Also migrated unittests --- .gitignore | 1 + README.md | 6 + airavata_django_portal_sdk/apps.py | 5 + airavata_django_portal_sdk/base.py | 10 -- .../migrations/0001_initial.py | 26 +++ airavata_django_portal_sdk/migrations/__init__.py | 0 airavata_django_portal_sdk/models.py | 15 ++ airavata_django_portal_sdk/user_storage.py | 81 ++++----- runtests.py | 15 ++ tests/__init__.py | 0 tests/test_settings.py | 18 ++ tests/test_user_storage.py | 182 +++++++++++++++++++++ 12 files changed, 300 insertions(+), 59 deletions(-) diff --git a/.gitignore b/.gitignore index c05fa1e..d692369 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ airavata_django_portal_sdk.egg-info *.pyc __pycache__ .vscode +db.sqlite3 diff --git a/README.md b/README.md index 647570e..eb3c4f6 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,9 @@ ``` pip install . ``` + +## Migrations + +``` +django-admin makemigrations --settings=tests.test_settings airavata_django_portal_sdk +``` diff --git a/airavata_django_portal_sdk/apps.py b/airavata_django_portal_sdk/apps.py new file mode 100644 index 0000000..88d3ec7 --- /dev/null +++ b/airavata_django_portal_sdk/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class AiravataDjangoPortalSDKConfig(AppConfig): + name = "airavata_django_portal_sdk" diff --git a/airavata_django_portal_sdk/base.py b/airavata_django_portal_sdk/base.py deleted file mode 100644 index 9090bff..0000000 --- a/airavata_django_portal_sdk/base.py +++ /dev/null @@ -1,10 +0,0 @@ -from django.db import models - -class UserFiles(models.Model): - """Base model that should be implemented in Airavata Django Portal.""" - username = models.CharField(max_length=64) - file_path = models.TextField() - file_dpu = models.CharField(max_length=255, primary_key=True) - - class Meta: - abstract = True diff --git a/airavata_django_portal_sdk/migrations/0001_initial.py b/airavata_django_portal_sdk/migrations/0001_initial.py new file mode 100644 index 0000000..6e1da6b --- /dev/null +++ b/airavata_django_portal_sdk/migrations/0001_initial.py @@ -0,0 +1,26 @@ +# Generated by Django 3.0.7 on 2020-07-01 11:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='UserFiles', + fields=[ + ('username', models.CharField(max_length=64)), + ('file_path', models.TextField()), + ('file_dpu', models.CharField(max_length=255, primary_key=True, serialize=False)), + ], + ), + migrations.AddIndex( + model_name='userfiles', + index=models.Index(fields=['username'], name='userfiles_username_idx'), + ), + ] diff --git a/airavata_django_portal_sdk/migrations/__init__.py b/airavata_django_portal_sdk/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/airavata_django_portal_sdk/models.py b/airavata_django_portal_sdk/models.py new file mode 100644 index 0000000..5490b57 --- /dev/null +++ b/airavata_django_portal_sdk/models.py @@ -0,0 +1,15 @@ +from django.db import models + +class UserFiles(models.Model): + """Base model that should be implemented in Airavata Django Portal.""" + username = models.CharField(max_length=64) + file_path = models.TextField() + file_dpu = models.CharField(max_length=255, primary_key=True) + + class Meta: + indexes = [ + # FIXME: ideally we would include file_path in the index to make + # lookups faster, but Django/MariaDB don't support key length on a + # TEXT column which is required to create an index + models.Index(fields=['username'], name='userfiles_username_idx') + ] diff --git a/airavata_django_portal_sdk/user_storage.py b/airavata_django_portal_sdk/user_storage.py index e108196..00fb079 100644 --- a/airavata_django_portal_sdk/user_storage.py +++ b/airavata_django_portal_sdk/user_storage.py @@ -16,7 +16,7 @@ from airavata.model.data.replica.ttypes import (DataProductModel, ReplicaLocationCategory, ReplicaPersistentType) -from . import base +from . import models import copy logger = logging.getLogger(__name__) @@ -27,7 +27,7 @@ TMP_INPUT_FILE_UPLOAD_DIR = "tmp" def save(request, path, file, name=None, content_type=None): "Save file in path in the user's storage." username = request.user.username - full_path = _datastore.save(username, path, file, name=name) + full_path = _Datastore().save(username, path, file, name=name) data_product = _save_data_product( request, full_path, name=name, content_type=content_type ) @@ -38,7 +38,7 @@ def move_from_filepath(request, source_path, target_path, name=None, content_typ "Move a file from filesystem into user's storage." username = request.user.username file_name = name if name is not None else os.path.basename(source_path) - full_path = _datastore.move_external(source_path, username, target_path, file_name) + full_path = _Datastore().move_external(source_path, username, target_path, file_name) data_product = _save_data_product( request, full_path, name=file_name, content_type=content_type ) @@ -49,7 +49,7 @@ def save_input_file_upload(request, file, name=None, content_type=None): """Save input file in staging area for input file uploads.""" username = request.user.username file_name = name if name is not None else os.path.basename(file.name) - full_path = _datastore.save(username, TMP_INPUT_FILE_UPLOAD_DIR, file) + full_path = _Datastore().save(username, TMP_INPUT_FILE_UPLOAD_DIR, file) data_product = _save_data_product( request, full_path, name=file_name, content_type=content_type ) @@ -59,7 +59,7 @@ def save_input_file_upload(request, file, name=None, content_type=None): def copy_input_file_upload(request, data_product): path = _get_replica_filepath(data_product) name = data_product.productName - full_path = _datastore.copy( + full_path = _Datastore().copy( data_product.ownerName, path, request.user.username, @@ -72,8 +72,8 @@ def copy_input_file_upload(request, data_product): def is_input_file_upload(request, data_product): # Check if file is one of user's files and in TMP_INPUT_FILE_UPLOAD_DIR path = _get_replica_filepath(data_product) - if _datastore.exists(request.user.username, path): - rel_path = _datastore.rel_path(request.user.username, path) + if _Datastore().exists(request.user.username, path): + rel_path = _Datastore().rel_path(request.user.username, path) return os.path.dirname(rel_path) == TMP_INPUT_FILE_UPLOAD_DIR else: return False @@ -82,7 +82,7 @@ def is_input_file_upload(request, data_product): def move_input_file_upload(request, data_product, path): source_path = _get_replica_filepath(data_product) file_name = data_product.productName - full_path = _datastore.move( + full_path = _Datastore().move( data_product.ownerName, source_path, request.user.username, path, file_name ) _delete_data_product(data_product.ownerName, source_path) @@ -96,7 +96,7 @@ def move_input_file_upload_from_filepath( "Move a file from filesystem into user's input file staging area." username = request.user.username file_name = name if name is not None else os.path.basename(source_path) - full_path = _datastore.move_external( + full_path = _Datastore().move_external( source_path, username, TMP_INPUT_FILE_UPLOAD_DIR, file_name ) data_product = _save_data_product( @@ -108,23 +108,23 @@ def move_input_file_upload_from_filepath( def open_file(request, data_product): "Return file object for replica if it exists in user storage." path = _get_replica_filepath(data_product) - return _datastore.open(data_product.ownerName, path) + return _Datastore().open(data_product.ownerName, path) def exists(request, data_product): "Return True if replica for data_product exists in user storage." path = _get_replica_filepath(data_product) - return _datastore.exists(data_product.ownerName, path) + return _Datastore().exists(data_product.ownerName, path) def dir_exists(request, path): - return _datastore.dir_exists(request.user.username, path) + return _Datastore().dir_exists(request.user.username, path) def user_file_exists(request, path): """If file exists, return data product URI, else None.""" - if _datastore.exists(request.user.username, path): - full_path = _datastore.path(request.user.username, path) + if _Datastore().exists(request.user.username, path): + full_path = _Datastore().path(request.user.username, path) data_product_uri = _get_data_product_uri(request, full_path) return data_product_uri else: @@ -132,14 +132,14 @@ def user_file_exists(request, path): def delete_dir(request, path): - return _datastore.delete_dir(request.user.username, path) + return _Datastore().delete_dir(request.user.username, path) def delete(request, data_product): "Delete replica for data product in this data store." path = _get_replica_filepath(data_product) try: - _datastore.delete(data_product.ownerName, path) + _Datastore().delete(data_product.ownerName, path) _delete_data_product(data_product.ownerName, path) except Exception as e: logger.exception( @@ -151,13 +151,14 @@ def delete(request, data_product): def listdir(request, path): - if _datastore.dir_exists(request.user.username, path): - directories, files = _datastore.list_user_dir(request.user.username, path) + datastore = _Datastore() + if datastore.dir_exists(request.user.username, path): + directories, files = datastore.list_user_dir(request.user.username, path) directories_data = [] for d in directories: dpath = os.path.join(path, d) - created_time = _datastore.get_created_time(request.user.username, dpath) - size = _datastore.size(request.user.username, dpath) + created_time = datastore.get_created_time(request.user.username, dpath) + size = datastore.size(request.user.username, dpath) directories_data.append( { "name": d, @@ -170,11 +171,11 @@ def listdir(request, path): files_data = [] for f in files: user_rel_path = os.path.join(path, f) - created_time = _datastore.get_created_time( + created_time = datastore.get_created_time( request.user.username, user_rel_path ) - size = _datastore.size(request.user.username, user_rel_path) - full_path = _datastore.path(request.user.username, user_rel_path) + size = datastore.size(request.user.username, user_rel_path) + full_path = datastore.path(request.user.username, user_rel_path) data_product_uri = _get_data_product_uri(request, full_path) files_data.append( { @@ -192,22 +193,21 @@ def listdir(request, path): def get_experiment_dir(request, project_name=None, experiment_name=None, path=None): - return _datastore.get_experiment_dir( + return _Datastore().get_experiment_dir( request.user.username, project_name, experiment_name, path ) def create_user_dir(request, path): - return _datastore.create_user_dir(request.user.username, path) + return _Datastore().create_user_dir(request.user.username, path) def get_rel_path(request, path): - return _datastore.rel_path(request.user.username, path) + return _Datastore().rel_path(request.user.username, path) def _get_data_product_uri(request, full_path): - user_files_model = _get_user_files_model() - user_file = user_files_model.objects.filter( + user_file = models.UserFiles.objects.filter( username=request.user.username, file_path=full_path) if user_file.exists(): product_uri = user_file[0].file_dpu @@ -231,27 +231,13 @@ def _register_data_product(request, full_path, data_product): product_uri = request.airavata_client.registerDataProduct( request.authz_token, data_product ) - UserFiles = _get_user_files_model() - user_file_instance = UserFiles( + user_file_instance = models.UserFiles( username=request.user.username, file_path=full_path, file_dpu=product_uri ) user_file_instance.save() return product_uri -def _get_user_files_model(): - user_files_model = getattr( - settings, "USER_FILES_MODEL", "django_airavata_api.User_Files" - ) - UserFiles = apps.get_model(user_files_model) - if not issubclass(UserFiles, base.UserFiles): - raise Exception( - f"Class {UserFiles} ({user_files_model}) is not an " - "instance of airavata_django_portal_sdk.base.UserFiles" - ) - return UserFiles - - def _save_copy_of_data_product(request, full_path, data_product): """Save copy of a data product with a different path.""" data_product_copy = _copy_data_product(request, data_product, full_path) @@ -275,8 +261,7 @@ def _copy_data_product(request, data_product, full_path): def _delete_data_product(username, full_path): # TODO: call API to delete data product from replica catalog when it is # available (not currently implemented) - user_files_model = _get_user_files_model() - user_file = user_files_model.objects.filter(username=username, file_path=full_path) + user_file = models.UserFiles.objects.filter(username=username, file_path=full_path) if user_file.exists(): user_file.delete() @@ -344,11 +329,12 @@ def _get_replica_filepath(data_product): class _Datastore: """Internal datastore abstraction.""" - directory = settings.GATEWAY_DATA_STORE_DIR def __init__(self, directory=None): if directory: self.directory = directory + else: + self.directory = settings.GATEWAY_DATA_STORE_DIR def exists(self, username, path): """Check if file path exists in this data store.""" @@ -528,6 +514,3 @@ class _Datastore: if os.path.exists(fp): total_size += os.path.getsize(fp) return total_size - - -_datastore = _Datastore() diff --git a/runtests.py b/runtests.py new file mode 100755 index 0000000..ca8149f --- /dev/null +++ b/runtests.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +import os +import sys + +import django +from django.conf import settings +from django.test.utils import get_runner + +if __name__ == "__main__": + os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.test_settings' + django.setup() + TestRunner = get_runner(settings) + test_runner = TestRunner() + failures = test_runner.run_tests(["tests"]) + sys.exit(bool(failures)) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 0000000..debc022 --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,18 @@ +import os + +BASEDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +SECRET_KEY = "abc123" +INSTALLED_APPS = [ + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'airavata_django_portal_sdk', +] +GATEWAY_DATA_STORE_DIR = "/tmp" +GATEWAY_DATA_STORE_RESOURCE_ID = "resourceId" +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASEDIR, 'db.sqlite3'), + } +} diff --git a/tests/test_user_storage.py b/tests/test_user_storage.py new file mode 100644 index 0000000..795bce9 --- /dev/null +++ b/tests/test_user_storage.py @@ -0,0 +1,182 @@ +import io +import os +import tempfile +import uuid +from unittest.mock import MagicMock +from urllib.parse import urlparse + +from django.contrib.auth.models import User +from django.test import RequestFactory, TestCase, override_settings + +from airavata.model.data.replica.ttypes import ( + DataProductModel, + DataProductType, + DataReplicaLocationModel, + ReplicaLocationCategory +) +from airavata_django_portal_sdk import user_storage + +GATEWAY_ID = 'test-gateway' + + +@override_settings(GATEWAY_ID=GATEWAY_ID) +class BaseTestCase(TestCase): + + def setUp(self): + self.user = User.objects.create_user('testuser') + self.factory = RequestFactory() + # Dummy POST request + self.request = self.factory.post('/upload', {}) + self.request.user = self.user + self.request.airavata_client = MagicMock(name="airavata_client") + self.product_uri = f"airavata-dp://{uuid.uuid4()}" + self.request.airavata_client.registerDataProduct.return_value = \ + self.product_uri + self.request.authz_token = "dummy" + + +class SaveTests(BaseTestCase): + + def test_save_with_defaults(self): + "Test save with default name and content type" + with tempfile.TemporaryDirectory() as tmpdirname, \ + self.settings(GATEWAY_DATA_STORE_DIR=tmpdirname, + GATEWAY_DATA_STORE_HOSTNAME="gateway.com"): + # path is just the user directory in gateway storage + path = os.path.join(tmpdirname, self.user.username) + file = io.StringIO("Foo file") + file.name = "foo.txt" + data_product = user_storage.save(self.request, path, file) + + self.assertEqual(data_product.productUri, self.product_uri) + self.request.airavata_client.registerDataProduct.\ + assert_called_once() + args, kws = self.request.airavata_client.registerDataProduct.\ + call_args + dp = args[1] + self.assertEqual(self.user.username, dp.ownerName) + self.assertEqual("foo.txt", dp.productName) + self.assertEqual(DataProductType.FILE, dp.dataProductType) + self.assertDictEqual({'mime-type': 'text/plain'}, + dp.productMetadata) + self.assertEqual(1, len(dp.replicaLocations)) + self.assertEqual(f"file://gateway.com:{path}/{file.name}", + dp.replicaLocations[0].filePath) + + def test_save_with_name_and_content_type(self): + "Test save with specified name and content type" + # TODO: either change _Datastore to be created anew each time + # or create the _Datastore manually with the tmpdirname + with tempfile.TemporaryDirectory() as tmpdirname, \ + self.settings(GATEWAY_DATA_STORE_DIR=tmpdirname, + GATEWAY_DATA_STORE_HOSTNAME="gateway.com"): + # path is just the user directory in gateway storage + path = os.path.join(tmpdirname, self.user.username) + file = io.StringIO("Foo file") + file.name = "foo.txt" + data_product = user_storage.save( + self.request, path, file, name="bar.txt", + content_type="application/some-app") + + self.assertEqual(data_product.productUri, self.product_uri) + self.request.airavata_client.registerDataProduct.\ + assert_called_once() + args, kws = self.request.airavata_client.registerDataProduct.\ + call_args + dp = args[1] + self.assertEqual(self.user.username, dp.ownerName) + self.assertEqual("bar.txt", dp.productName) + self.assertEqual(DataProductType.FILE, dp.dataProductType) + self.assertDictEqual({'mime-type': 'application/some-app'}, + dp.productMetadata) + self.assertEqual(1, len(dp.replicaLocations)) + self.assertEqual(f"file://gateway.com:{path}/bar.txt", + dp.replicaLocations[0].filePath) + + def test_save_with_unknown_text_file_type(self): + "Test save with unknown file ext for text file" + with tempfile.TemporaryDirectory() as tmpdirname, \ + self.settings(GATEWAY_DATA_STORE_DIR=tmpdirname, + GATEWAY_DATA_STORE_HOSTNAME="gateway.com"): + path = os.path.join( + tmpdirname, "foo.someext") + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w') as f: + f.write("Some Unicode text") + with open(path, 'r') as f: + dp = user_storage.save( + self.request, "some/path", f, + content_type="application/octet-stream") + # Make sure that the file contents are tested to see if text + self.assertDictEqual({'mime-type': 'text/plain'}, + dp.productMetadata) + + def test_save_with_unknown_binary_file_type(self): + "Test save with unknown file ext for binary file" + with tempfile.TemporaryDirectory() as tmpdirname, \ + self.settings(GATEWAY_DATA_STORE_DIR=tmpdirname, + GATEWAY_DATA_STORE_HOSTNAME="gateway.com"): + path = os.path.join( + tmpdirname, "foo.someext") + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'wb') as f: + f.write(bytes(range(256))) + with open(path, 'rb') as f: + dp = user_storage.save( + self.request, "some/path", f, + content_type="application/octet-stream") + # Make sure that DID NOT determine file contents are text + self.assertDictEqual({'mime-type': 'application/octet-stream'}, + dp.productMetadata) + + +class CopyInputFileUploadTests(BaseTestCase): + def test_copy_input_file_upload(self): + "Test copy input file upload copies data product" + with tempfile.TemporaryDirectory() as tmpdirname, \ + self.settings(GATEWAY_DATA_STORE_DIR=tmpdirname, + GATEWAY_DATA_STORE_HOSTNAME="gateway.com"): + # path is just the user directory in gateway storage + source_path = os.path.join( + tmpdirname, self.user.username, "foo.ext") + os.makedirs(os.path.dirname(source_path)) + with open(source_path, 'wb') as f: + f.write(b"123") + + data_product = DataProductModel() + data_product.productUri = f"airavata-dp://{uuid.uuid4()}" + data_product.gatewayId = GATEWAY_ID + data_product.ownerName = self.user.username + data_product.productName = "foo.ext" + data_product.dataProductType = DataProductType.FILE + data_product.productMetadata = { + 'mime-type': 'application/some-app' + } + replica_category = ReplicaLocationCategory.GATEWAY_DATA_STORE + replica_path = f"file://gateway.com:{source_path}" + data_product.replicaLocations = [ + DataReplicaLocationModel( + filePath=replica_path, + replicaLocationCategory=replica_category)] + + data_product_copy = user_storage.copy_input_file_upload( + self.request, data_product) + + self.request.airavata_client.registerDataProduct.\ + assert_called_once() + self.assertIsNot(data_product_copy, data_product) + self.assertNotEqual(data_product_copy.productUri, + data_product.productUri) + self.assertDictEqual(data_product_copy.productMetadata, + data_product.productMetadata) + self.assertEqual(data_product_copy.productName, + data_product.productName) + self.assertEqual(data_product_copy.dataProductType, + data_product.dataProductType) + replica_copy_path = data_product_copy.replicaLocations[0].filePath + self.assertNotEqual(replica_copy_path, replica_path) + replica_copy_filepath = urlparse(replica_copy_path).path + self.assertEqual( + os.path.dirname(replica_copy_filepath), + os.path.join(tmpdirname, self.user.username, "tmp"), + msg="Verify input file copied to user's tmp dir")
