Repository: airavata Updated Branches: refs/heads/develop b68d84c2b -> 92cc0dfea
http://git-wip-us.apache.org/repos/asf/airavata/blob/e1a0772f/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManager.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManager.java b/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManager.java new file mode 100644 index 0000000..fd06f8c --- /dev/null +++ b/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManager.java @@ -0,0 +1,121 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.data.manager.cpi; + +import org.apache.airavata.model.data.resource.DataReplicaLocationModel; +import org.apache.airavata.model.data.resource.DataResourceModel; + +import java.util.List; + +public interface DataManager { + + /** + * To create a new dataResourceModel. This is how the system comes to know about already + * existing resources + * @param dataResourceModel + * @return + */ + String registerResource(DataResourceModel dataResourceModel) throws DataManagerException; + + /** + * To remove a resource entry from the replica catalog + * @param resourceId + * @return + */ + boolean removeResource(String resourceId) throws DataManagerException; + + + /** + * To update an existing data resource model + * @param dataResourceModel + * @return + * @throws DataManagerException + */ + boolean updateResource(DataResourceModel dataResourceModel) throws DataManagerException; + + /** + * To retrieve a resource object providing the resourceId + * @param resourceId + * @return + */ + DataResourceModel getResource(String resourceId) throws DataManagerException; + + /** + * To create a new data replica location. This is how the system comes to know about already + * existing resources + * @param dataReplicaLocationModel + * @return + */ + String registerReplicaLocation(DataReplicaLocationModel dataReplicaLocationModel) throws DataManagerException; + + /** + * To remove a replica entry from the replica catalog + * @param replicaId + * @return + */ + boolean removeReplicaLocation(String replicaId) throws DataManagerException; + + /** + * To update an existing data replica model + * @param dataReplicaLocationModel + * @return + * @throws DataManagerException + */ + boolean updateReplicaLocation(DataReplicaLocationModel dataReplicaLocationModel) throws DataManagerException; + + /** + * To retrieve a replica object providing the replicaId + * @param replicaId + * @return + */ + DataReplicaLocationModel getReplicaLocation(String replicaId) throws DataManagerException; + + /** + * To retrieve all the replica entries for a given resource id + * @param resourceId + * @return + * @throws DataCatalogException + */ + List<DataReplicaLocationModel> getAllReplicaLocations(String resourceId) throws DataManagerException; + + + /** + * API method to copy a resource to the provided destination storage resource. Only resources of type FILE can be + * copied using this API method. Method returns the new replicaId. + * @param dataResourceId + * @param destStorageResourceId + * @param destinationParentPath + * @return + */ + String copyResource(String dataResourceId, String destStorageResourceId, String destinationParentPath) throws DataManagerException; + + /** + * API method to copy the specified replica to the provided destination storage resource. Only resources of type FILE + * can be copied using this API method. Method returns the new replicaId + * @param dataResourceId + * @param replicaId + * @param destStorageResourceId + * @param destinationParentPath + * @return + * @throws DataManagerException + */ + String copyReplica(String dataResourceId, String replicaId, String destStorageResourceId, String destinationParentPath) throws DataManagerException; +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e1a0772f/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerConstants.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerConstants.java b/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerConstants.java new file mode 100644 index 0000000..d46604a --- /dev/null +++ b/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerConstants.java @@ -0,0 +1,28 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.data.manager.cpi; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DataManagerConstants { + private final static Logger logger = LoggerFactory.getLogger(DataManagerConstants.class); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/e1a0772f/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerException.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerException.java b/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerException.java new file mode 100644 index 0000000..b1c1cb8 --- /dev/null +++ b/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerException.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.airavata.data.manager.cpi; + +public class DataManagerException extends Exception{ + + public DataManagerException(Throwable e) { + super(e); + } + + public DataManagerException(String message) { + super(message, null); + } + + public DataManagerException(String message, Throwable e) { + super(message, e); + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e1a0772f/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/FileTransferService.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/FileTransferService.java b/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/FileTransferService.java new file mode 100644 index 0000000..f701e49 --- /dev/null +++ b/modules/file-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/FileTransferService.java @@ -0,0 +1,40 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.data.manager.cpi; + +import org.apache.airavata.model.file.FileNode; + +public interface FileTransferService { + + void uploadFile(String fileName, byte[] fileData, String destinationResourceId, String destinationPath) throws DataManagerException; + + void importFile(String sourceUrl, String destinationResourceId, String destinationPath) throws DataManagerException; + + void transferFile(String sourceResourceId, String sourcePath, String destinationResourceId, String destinationPath) throws DataManagerException; + + FileNode getDirectoryListing(String storageResourceId, String directoryPath) throws DataManagerException; + + void moveFile(String storageResourceId, String sourcePath, String destinationPath) throws DataManagerException; + + void renameFile(String storageResourceId, String sourcePath, String newName) throws DataManagerException; + + void mkdir(String storageResourceId, String directoryPath) throws DataManagerException; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/e1a0772f/modules/file-manager/pom.xml ---------------------------------------------------------------------- diff --git a/modules/file-manager/pom.xml b/modules/file-manager/pom.xml new file mode 100644 index 0000000..eb81d40 --- /dev/null +++ b/modules/file-manager/pom.xml @@ -0,0 +1,54 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>airavata</artifactId> + <groupId>org.apache.airavata</groupId> + <version>0.16-SNAPSHOT</version> + <relativePath>../../pom.xml</relativePath> + </parent> + + <modelVersion>4.0.0</modelVersion> + <artifactId>data-manager</artifactId> + <packaging>pom</packaging> + <name>Airavata Data Manager</name> + <url>http://airavata.apache.org/</url> + + <modules> + <module>data-manager-cpi</module> + <module>data-manager-core</module> + </modules> + <dependencies> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>airavata-data-models</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>airavata-registry-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>airavata-registry-cpi</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.7</version> + <scope>test</scope> + </dependency> + </dependencies> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/e1a0772f/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index c82cc8d..70df549 100644 --- a/pom.xml +++ b/pom.xml @@ -551,7 +551,7 @@ <module>modules/commons</module> <module>modules/messaging</module> <module>modules/gfac</module> - <module>modules/data-manager</module> + <module>modules/file-manager</module> <module>modules/registry</module> <module>modules/security</module> <module>modules/credential-store</module> http://git-wip-us.apache.org/repos/asf/airavata/blob/e1a0772f/thrift-interface-descriptions/airavata-apis/airavata_api.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/airavata-apis/airavata_api.thrift b/thrift-interface-descriptions/airavata-apis/airavata_api.thrift index 5d842d7..2eca176 100644 --- a/thrift-interface-descriptions/airavata-apis/airavata_api.thrift +++ b/thrift-interface-descriptions/airavata-apis/airavata_api.thrift @@ -40,7 +40,7 @@ include "../data-models/resource-catalog-models/storage_resource_model.thrift" include "../data-models/resource-catalog-models/gateway_resource_profile_model.thrift" include "../data-models/resource-catalog-models/data_movement_models.thrift" include "../data-models/workflow-models/workflow_data_model.thrift" -include "../data-models/data-catalog-models/replica_catalog_models.thrift" +include "../data-models/data-catalog-models/replica_models.thrift" namespace java org.apache.airavata.api namespace php Airavata.API http://git-wip-us.apache.org/repos/asf/airavata/blob/e1a0772f/thrift-interface-descriptions/data-models/airavata_data_models.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/data-models/airavata_data_models.thrift b/thrift-interface-descriptions/data-models/airavata_data_models.thrift index 7cb9fd4..edea087 100644 --- a/thrift-interface-descriptions/data-models/airavata_data_models.thrift +++ b/thrift-interface-descriptions/data-models/airavata_data_models.thrift @@ -30,7 +30,8 @@ include "experiment-catalog-models/process_model.thrift" include "experiment-catalog-models/scheduling_model.thrift" include "experiment-catalog-models/status_models.thrift" include "resource-catalog-models/data_movement_models.thrift" -include "data-catalog-models/replica_catalog_models.thrift" +include "data-manager-models/replica_models.thrift" +include "data-manager-models/file_models.thrift" namespace java org.apache.airavata.model namespace php Airavata.Model http://git-wip-us.apache.org/repos/asf/airavata/blob/e1a0772f/thrift-interface-descriptions/data-models/data-catalog-models/replica_catalog_models.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/data-models/data-catalog-models/replica_catalog_models.thrift b/thrift-interface-descriptions/data-models/data-catalog-models/replica_catalog_models.thrift deleted file mode 100644 index 6b344fb..0000000 --- a/thrift-interface-descriptions/data-models/data-catalog-models/replica_catalog_models.thrift +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - - include "../resource-catalog-models/data_movement_models.thrift" - - namespace java org.apache.airavata.model.data.resource - namespace php Airavata.Model.Data.Resource - namespace cpp apache.airavata.model.data.resource - namespace py apache.airavata.model.data.resource - -enum ReplicaLocationCategory { - GATEWAY_DATA_STORE, - COMPUTE_RESOURCE, - LONG_TERM_STORAGE_RESOURCE, - OTHER -} - -enum ReplicaPersistentType { - TRANSIENT, - PERSISTENT -} - -enum DataResourceType { - COLLECTION, - FILE -} - -struct DataResourceModel { - 1: optional string resourceId, - 2: optional string gatewayId, - 3: optional string parentResourceId, - 4: optional string resourceName, - 5: optional string resourceDescription, - 6: optional string ownerName, - 7: optional DataResourceType dataResourceType, - 8: optional i32 resourceSize, - 9: optional i64 creationTime, - 10: optional i64 lastModifiedTime, - 11: optional map<string, string> resourceMetadata, - 12: optional list<DataReplicaLocationModel> replicaLocations, - 13: optional list<DataResourceModel> childResources -} - -struct DataReplicaLocationModel { - 1: optional string replicaId, - 2: optional string resourceId, - 3: optional string replicaName, - 4: optional string replicaDescription, - 5: optional i64 creationTime, - 6: optional i64 lastModifiedTime, - 7: optional i64 validUntilTime, - 8: optional ReplicaLocationCategory replicaLocationCategory, - 9: optional ReplicaPersistentType replicaPersistentType, - 10: optional string storageResourceId, - 11: optional string fileAbsolutePath, - 12: optional map<string, string> replicaMetadata -} http://git-wip-us.apache.org/repos/asf/airavata/blob/e1a0772f/thrift-interface-descriptions/data-models/data-manager-models/file_models.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/data-models/data-manager-models/file_models.thrift b/thrift-interface-descriptions/data-models/data-manager-models/file_models.thrift new file mode 100644 index 0000000..bdcf784 --- /dev/null +++ b/thrift-interface-descriptions/data-models/data-manager-models/file_models.thrift @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + + namespace java org.apache.airavata.model.file + namespace php Airavata.Model.File + namespace cpp apache.airavata.model.file + namespace py apache.airavata.model.file + + enum FileNodeTypes{ + DIRECTORY, + FILE + } + + struct FileNode { + 1: optional FileNodeTypes type, + 2: optional string size, + 3: optional string nativeType, + 4: optional string name, + 5: optional string path, + 6: optional string storageResourceId, + 7: optional i64 lastModifiedType, + 8: optional i64 createdTime, + 9: optional list<FileNode> childNodes + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/e1a0772f/thrift-interface-descriptions/data-models/data-manager-models/metadata_models.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/data-models/data-manager-models/metadata_models.thrift b/thrift-interface-descriptions/data-models/data-manager-models/metadata_models.thrift new file mode 100644 index 0000000..927c92f --- /dev/null +++ b/thrift-interface-descriptions/data-models/data-manager-models/metadata_models.thrift @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ http://git-wip-us.apache.org/repos/asf/airavata/blob/e1a0772f/thrift-interface-descriptions/data-models/data-manager-models/replica_models.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/data-models/data-manager-models/replica_models.thrift b/thrift-interface-descriptions/data-models/data-manager-models/replica_models.thrift new file mode 100644 index 0000000..f010b6c --- /dev/null +++ b/thrift-interface-descriptions/data-models/data-manager-models/replica_models.thrift @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + + include "../resource-catalog-models/data_movement_models.thrift" + + namespace java org.apache.airavata.model.replica + namespace php Airavata.Model.Replica + namespace cpp apache.airavata.model.replica + namespace py apache.airavata.model.replica + +enum ReplicaLocationCategory { + GATEWAY_DATA_STORE, + COMPUTE_RESOURCE, + LONG_TERM_STORAGE_RESOURCE, + OTHER +} + +enum ReplicaPersistentType { + TRANSIENT, + PERSISTENT +} + +enum DataResourceType { + COLLECTION, + FILE +} + +struct DataResourceModel { + 1: optional string resourceId, + 2: optional string gatewayId, + 3: optional string parentResourceId, + 4: optional string resourceName, + 5: optional string resourceDescription, + 6: optional string ownerName, + 7: optional string sha256Checksum, + 8: optional DataResourceType dataResourceType, + 9: optional i32 resourceSize, + 10: optional string nativeFormat, + 11: optional i64 creationTime, + 12: optional i64 lastModifiedTime, + 13: optional map<string, string> resourceMetadata, + 14: optional list<DataReplicaLocationModel> replicaLocations, + 15: optional list<DataResourceModel> childResources +} + +struct DataReplicaLocationModel { + 1: optional string replicaId, + 2: optional string resourceId, + 3: optional string replicaName, + 4: optional string replicaDescription, + 5: optional string sourceReplicaId, + 6: optional i64 creationTime, + 7: optional i64 lastModifiedTime, + 8: optional i64 validUntilTime, + 9: optional ReplicaLocationCategory replicaLocationCategory, + 10: optional ReplicaPersistentType replicaPersistentType, + 11: optional string storageResourceId, + 12: optional string fileAbsolutePath, + 13: optional map<string, string> replicaMetadata +}
