restructureing the file manager module
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/5881af94 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/5881af94 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/5881af94 Branch: refs/heads/develop Commit: 5881af943bf99c5ed68b95996cffb53e2e19edad Parents: e1a0772 Author: scnakandala <[email protected]> Authored: Wed Dec 23 01:23:54 2015 -0500 Committer: scnakandala <[email protected]> Committed: Wed Dec 23 01:23:56 2015 -0500 ---------------------------------------------------------------------- airavata-api/airavata-api-server/pom.xml | 5 - .../server/handler/AiravataServerHandler.java | 98 - .../api/server/handler/utils/DataCatInit.java | 315 - .../java/org/apache/airavata/api/Airavata.java | 45811 +++++++---------- .../main/resources/lib/airavata/Airavata.cpp | 21585 ++++---- .../src/main/resources/lib/airavata/Airavata.h | 1002 +- .../lib/airavata/Airavata_server.skeleton.cpp | 37 - .../main/resources/lib/airavata/Workflow.cpp | 2514 - .../src/main/resources/lib/airavata/Workflow.h | 1171 - .../lib/airavata/Workflow_server.skeleton.cpp | 91 - .../lib/airavata/airavataAPI_constants.cpp | 36 - .../lib/airavata/airavataAPI_constants.h | 42 - .../lib/airavata/airavataAPI_types.cpp | 30 - .../resources/lib/airavata/airavataAPI_types.h | 49 - .../airavata/airavataDataModel_constants.cpp | 34 - .../lib/airavata/airavataDataModel_constants.h | 41 - .../lib/airavata/airavataDataModel_types.cpp | 30 - .../lib/airavata/airavataDataModel_types.h | 43 - .../lib/airavata/airavataErrors_constants.cpp | 34 - .../lib/airavata/airavataErrors_constants.h | 41 - .../lib/airavata/airavataErrors_types.cpp | 837 - .../lib/airavata/airavataErrors_types.h | 526 - .../resources/lib/airavata/airavata_api_types.h | 1 - .../lib/airavata/airavata_data_models_types.h | 3 +- .../applicationDeploymentModel_constants.cpp | 36 - .../applicationDeploymentModel_constants.h | 42 - .../applicationDeploymentModel_types.cpp | 582 - .../airavata/applicationDeploymentModel_types.h | 314 - .../applicationInterfaceModel_constants.cpp | 36 - .../applicationInterfaceModel_constants.h | 42 - .../applicationInterfaceModel_types.cpp | 631 - .../airavata/applicationInterfaceModel_types.h | 427 - .../airavata/computeResourceModel_constants.cpp | 36 - .../airavata/computeResourceModel_constants.h | 42 - .../lib/airavata/computeResourceModel_types.cpp | 1946 - .../lib/airavata/computeResourceModel_types.h | 1088 - .../airavata/data_resource_models_constants.cpp | 34 - .../airavata/data_resource_models_constants.h | 41 - .../lib/airavata/data_resource_models_types.cpp | 707 - .../lib/airavata/data_resource_models_types.h | 291 - .../lib/airavata/experimentModel_constants.cpp | 40 - .../lib/airavata/experimentModel_constants.h | 44 - .../lib/airavata/experimentModel_types.cpp | 3411 -- .../lib/airavata/experimentModel_types.h | 2132 - .../file__transfer_models_constants.cpp | 34 + .../airavata/file__transfer_models_constants.h | 41 + .../airavata/file__transfer_models_types.cpp | 823 + .../lib/airavata/file__transfer_models_types.h | 384 + .../gatewayResourceProfileModel_constants.cpp | 34 - .../gatewayResourceProfileModel_constants.h | 41 - .../gatewayResourceProfileModel_types.cpp | 298 - .../gatewayResourceProfileModel_types.h | 208 - .../lib/airavata/messagingEvents_constants.cpp | 36 - .../lib/airavata/messagingEvents_constants.h | 42 - .../lib/airavata/messagingEvents_types.cpp | 1441 - .../lib/airavata/messagingEvents_types.h | 793 - .../replica_catalog_models_constants.cpp | 34 - .../airavata/replica_catalog_models_constants.h | 41 - .../airavata/replica_catalog_models_types.cpp | 909 - .../lib/airavata/replica_catalog_models_types.h | 379 - .../lib/airavata/replica_models_constants.cpp | 34 + .../lib/airavata/replica_models_constants.h | 41 + .../lib/airavata/replica_models_types.cpp | 909 + .../lib/airavata/replica_models_types.h | 379 + .../lib/airavata/workflowAPI_constants.cpp | 36 - .../lib/airavata/workflowAPI_constants.h | 42 - .../lib/airavata/workflowAPI_types.cpp | 30 - .../resources/lib/airavata/workflowAPI_types.h | 47 - .../airavata/workflowDataModel_constants.cpp | 36 - .../lib/airavata/workflowDataModel_constants.h | 42 - .../lib/airavata/workflowDataModel_types.cpp | 207 - .../lib/airavata/workflowDataModel_types.h | 133 - .../lib/airavata/workspaceModel_constants.cpp | 34 - .../lib/airavata/workspaceModel_constants.h | 41 - .../lib/airavata/workspaceModel_types.cpp | 508 - .../lib/airavata/workspaceModel_types.h | 321 - .../resources/lib/thrift/concurrency/Thread.h | 0 .../resources/lib/thrift/server/TServer.cpp | 0 .../main/resources/lib/thrift/thrift-config.h | 0 .../lib/thrift/transport/TPipeServer.h | 0 .../resources/lib/Airavata/API/Airavata.php | 9737 ++-- .../lib/Airavata/Model/Data/Resource/Types.php | 923 - .../resources/lib/Airavata/Model/File/Types.php | 795 + .../lib/Airavata/Model/Replica/Types.php | 923 + .../lib/apache/airavata/api/Airavata-remote | 42 - .../lib/apache/airavata/api/Airavata.py | 6187 +-- .../resources/lib/apache/airavata/api/ttypes.py | 1 - .../airavata/model/data/resource/__init__.py | 1 - .../airavata/model/data/resource/constants.py | 11 - .../airavata/model/data/resource/ttypes.py | 574 - .../lib/apache/airavata/model/file/__init__.py | 1 + .../lib/apache/airavata/model/file/constants.py | 11 + .../lib/apache/airavata/model/file/ttypes.py | 550 + .../apache/airavata/model/replica/__init__.py | 1 + .../apache/airavata/model/replica/constants.py | 11 + .../lib/apache/airavata/model/replica/ttypes.py | 574 + .../lib/apache/airavata/model/ttypes.py | 3 +- .../model/workspace/experiment/__init__.py | 1 - .../model/workspace/experiment/constants.py | 14 - .../model/workspace/experiment/ttypes.py | 3474 -- .../client/samples/DataManagerSample.java | 79 - .../org/apache/airavata/model/Workflow.java | 2 +- .../ApplicationDeploymentDescription.java | 2 +- .../appdeployment/ApplicationModule.java | 2 +- .../appcatalog/appdeployment/CommandObject.java | 2 +- .../appcatalog/appdeployment/SetEnvPaths.java | 2 +- .../application_deployment_modelConstants.java | 58 - .../ApplicationInterfaceDescription.java | 2 +- .../application_interface_modelConstants.java | 58 - .../appcatalog/computeresource/BatchQueue.java | 2 +- .../computeresource/CloudJobSubmission.java | 2 +- .../ComputeResourceDescription.java | 2 +- .../computeresource/GlobusJobSubmission.java | 2 +- .../computeresource/JobSubmissionInterface.java | 2 +- .../computeresource/LOCALSubmission.java | 2 +- .../computeresource/ResourceJobManager.java | 2 +- .../computeresource/SSHJobSubmission.java | 2 +- .../computeresource/UnicoreJobSubmission.java | 2 +- .../compute_resource_modelConstants.java | 58 - .../ComputeResourcePreference.java | 2 +- .../gatewayprofile/DataStoragePreference.java | 728 - .../gatewayprofile/GatewayResourceProfile.java | 2 +- .../gatewayprofile/StoragePreference.java | 2 +- .../StorageResourceDescription.java | 2 +- .../application/io/InputDataObjectType.java | 2 +- .../application/io/OutputDataObjectType.java | 2 +- .../airavata/model/commons/ErrorModel.java | 2 +- .../model/commons/ValidationResults.java | 2 +- .../airavata/model/commons/ValidatorResult.java | 2 +- .../data/movement/DataMovementInterface.java | 2 +- .../data/movement/GridFTPDataMovement.java | 2 +- .../model/data/movement/LOCALDataMovement.java | 2 +- .../model/data/movement/SCPDataMovement.java | 2 +- .../data/movement/UnicoreDataMovement.java | 2 +- .../data/resource/DataReplicaLocationModel.java | 1741 - .../model/data/resource/DataResourceModel.java | 2049 - .../model/data/resource/DataResourceType.java | 62 - .../data/resource/ReplicaLocationCategory.java | 68 - .../data/resource/ReplicaPersistentType.java | 62 - .../model/error/AiravataClientException.java | 2 +- .../model/error/AiravataSystemException.java | 2 +- .../model/error/AuthenticationException.java | 2 +- .../model/error/AuthorizationException.java | 2 +- .../error/ExperimentNotFoundException.java | 2 +- .../model/error/InvalidRequestException.java | 2 +- .../model/error/LaunchValidationException.java | 2 +- .../model/error/ProjectNotFoundException.java | 2 +- .../airavata/model/error/TimedOutException.java | 2 +- .../airavata/model/error/ValidationResults.java | 2 +- .../airavata/model/error/ValidatorResult.java | 2 +- .../model/experiment/ExperimentModel.java | 2 +- .../model/experiment/ExperimentStatistics.java | 2 +- .../experiment/ExperimentSummaryModel.java | 2 +- .../experiment/UserConfigurationDataModel.java | 2 +- .../apache/airavata/model/file/FileNode.java | 1147 + .../airavata/model/file/FileNodeTypes.java | 62 + .../airavata/model/file/FileTransferMode.java | 62 + .../model/file/FileTransferRequest.java | 2328 + .../airavata/model/file/FileTransferStatus.java | 71 + .../model/file/StorageResourceProtocol.java | 74 + .../org/apache/airavata/model/job/JobModel.java | 2 +- .../event/ExperimentStatusChangeEvent.java | 2 +- .../model/messaging/event/JobIdentifier.java | 2 +- .../messaging/event/JobStatusChangeEvent.java | 2 +- .../event/JobStatusChangeRequestEvent.java | 2 +- .../airavata/model/messaging/event/Message.java | 2 +- .../messaging/event/ProcessIdentifier.java | 2 +- .../event/ProcessStatusChangeEvent.java | 2 +- .../event/ProcessStatusChangeRequestEvent.java | 2 +- .../messaging/event/ProcessSubmitEvent.java | 2 +- .../messaging/event/ProcessTerminateEvent.java | 2 +- .../model/messaging/event/TaskIdentifier.java | 2 +- .../messaging/event/TaskOutputChangeEvent.java | 2 +- .../messaging/event/TaskStatusChangeEvent.java | 2 +- .../event/TaskStatusChangeRequestEvent.java | 2 +- .../model/messaging/event/TaskSubmitEvent.java | 709 - .../messaging/event/TaskTerminateEvent.java | 709 - .../event/messaging_eventsConstants.java | 58 - .../airavata/model/process/ProcessModel.java | 2 +- .../model/replica/DataReplicaLocationModel.java | 1741 + .../model/replica/DataResourceModel.java | 2049 + .../model/replica/DataResourceType.java | 62 + .../model/replica/ReplicaLocationCategory.java | 68 + .../model/replica/ReplicaPersistentType.java | 62 + .../ComputationalResourceSchedulingModel.java | 2 +- .../airavata/model/security/AuthzToken.java | 2 +- .../airavata/model/status/ExperimentStatus.java | 2 +- .../apache/airavata/model/status/JobStatus.java | 2 +- .../airavata/model/status/ProcessStatus.java | 2 +- .../airavata/model/status/TaskStatus.java | 2 +- .../model/task/DataStagingTaskModel.java | 2 +- .../model/task/EnvironmentSetupTaskModel.java | 2 +- .../model/task/JobSubmissionTaskModel.java | 2 +- .../airavata/model/task/MonitorTaskModel.java | 2 +- .../apache/airavata/model/task/TaskModel.java | 2 +- .../model/workflow_data_modelConstants.java | 58 - .../airavata/model/workspace/Gateway.java | 2 +- .../apache/airavata/model/workspace/Group.java | 2 +- .../airavata/model/workspace/Project.java | 2 +- .../apache/airavata/model/workspace/User.java | 2 +- modules/file-manager/data-manager-core/pom.xml | 61 - .../data/manager/core/DataManagerFactory.java | 34 - .../data/manager/core/DataManagerImpl.java | 444 - .../data/manager/core/ssh/CommandOutput.java | 34 - .../data/manager/core/ssh/SSHApiException.java | 33 - .../data/manager/core/ssh/SSHUtils.java | 506 - .../manager/core/ssh/StandardOutReader.java | 86 - .../manager/core/DataManagerFactoryTest.java | 45 - .../data/manager/core/DataManagerImplTest.java | 208 - .../data/manager/core/utils/AppCatInit.java | 320 - .../data/manager/core/utils/DataCatInit.java | 315 - modules/file-manager/data-manager-cpi/pom.xml | 30 - .../airavata/data/manager/cpi/DataManager.java | 121 - .../data/manager/cpi/DataManagerConstants.java | 28 - .../data/manager/cpi/DataManagerException.java | 35 - .../data/manager/cpi/FileTransferService.java | 40 - modules/file-manager/file-manager-core/pom.xml | 72 + .../file/manager/core/FileManagerFactory.java | 34 + .../file/manager/core/FileManagerImpl.java | 29 + .../manager/core/FileTransferServiceImpl.java | 474 + .../conversion/AbstractThriftDeserializer.java | 154 + .../db/conversion/AbstractThriftSerializer.java | 122 + .../FileTransferRequestDeserializer.java | 45 + .../FileTransferRequestSerializer.java | 40 + .../db/conversion/ModelConversionHelper.java | 82 + .../core/db/dao/FileTransferRequestDao.java | 117 + .../file/manager/core/db/utils/MongoUtils.java | 69 + .../core/remote/client/RemoteStorageClient.java | 108 + .../remote/client/http/HTTPStorageClient.java | 217 + .../core/remote/client/scp/CommandOutput.java | 34 + .../core/remote/client/scp/SCPApiException.java | 33 + .../remote/client/scp/SCPStorageClient.java | 425 + .../remote/client/scp/StandardOutReader.java | 86 + .../remote/client/sftp/SFTPStorageClient.java | 209 + .../core/utils/FileManagerConstants.java | 33 + .../core/utils/FileManagerProperties.java | 58 + .../src/main/resources/file-manager.properties | 24 + .../client/http/HTTPStorageClientTest.java | 45 + modules/file-manager/file-manager-cpi/pom.xml | 30 + .../airavata/file/manager/cpi/FileManager.java | 24 + .../file/manager/cpi/FileManagerConstants.java | 28 + .../file/manager/cpi/FileManagerException.java | 35 + .../file/manager/cpi/FileTransferService.java | 206 + .../manager/cpi/ReplicaManagementService.java | 25 + modules/file-manager/pom.xml | 8 +- .../core/data/catalog/impl/DataCatalogImpl.java | 286 - .../core/data/catalog/model/Configuration.java | 55 - .../data/catalog/model/DataReplicaLocation.java | 169 - .../data/catalog/model/DataReplicaMetaData.java | 77 - .../catalog/model/DataReplicaMetaData_PK.java | 59 - .../core/data/catalog/model/DataResource.java | 177 - .../catalog/model/DataResourceMetaData.java | 77 - .../catalog/model/DataResourceMetaData_PK.java | 59 - .../catalog/utils/DataCatalogConstants.java | 49 - .../data/catalog/utils/DataCatalogJPAUtils.java | 82 - .../utils/DataCatalogQueryGenerator.java | 111 - .../utils/ThriftDataModelConversion.java | 213 - .../catalog/impl/RegistryFactory.java | 15 - .../registry/core/impl/RegistryImpl.java | 6 - .../src/main/resources/META-INF/persistence.xml | 9 - .../src/main/resources/datacatalog-derby.sql | 77 - .../src/main/resources/datacatalog-mysql.sql | 76 - .../airavata/data/catalog/DataCatalogTest.java | 219 - .../airavata/data/catalog/util/Initialize.java | 315 - .../airavata/registry/cpi/DataCatalog.java | 45 - .../registry/cpi/DataCatalogException.java | 36 - .../apache/airavata/registry/cpi/Registry.java | 1 - .../airavata-apis/airavata_api.thrift | 46 - .../data-models/airavata_data_models.thrift | 4 +- .../data-manager-models/file_models.thrift | 41 - .../data-manager-models/metadata_models.thrift | 19 - .../data-manager-models/replica_models.thrift | 77 - .../file__transfer_models.thrift | 74 + .../file-manager-models/metadata_models.thrift | 19 + .../file-manager-models/replica_models.thrift | 77 + 275 files changed, 49989 insertions(+), 90670 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/airavata-api/airavata-api-server/pom.xml ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-api-server/pom.xml b/airavata-api/airavata-api-server/pom.xml index 9e3c27b..e03bba1 100644 --- a/airavata-api/airavata-api-server/pom.xml +++ b/airavata-api/airavata-api-server/pom.xml @@ -74,11 +74,6 @@ </dependency> <dependency> <groupId>org.apache.airavata</groupId> - <artifactId>data-manager-core</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.airavata</groupId> <artifactId>data-manager-cpi</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java b/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java index 1ceb797..67b616e 100644 --- a/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java +++ b/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java @@ -32,9 +32,6 @@ import org.apache.airavata.credential.store.client.CredentialStoreClientFactory; import org.apache.airavata.credential.store.cpi.CredentialStoreService; import org.apache.airavata.credential.store.datamodel.SSHCredential; import org.apache.airavata.credential.store.exception.CredentialStoreException; -import org.apache.airavata.data.manager.core.DataManagerFactory; -import org.apache.airavata.data.manager.cpi.DataManager; -import org.apache.airavata.data.manager.cpi.DataManagerException; import org.apache.airavata.messaging.core.MessageContext; import org.apache.airavata.messaging.core.Publisher; import org.apache.airavata.messaging.core.PublisherFactory; @@ -51,7 +48,6 @@ import org.apache.airavata.model.application.io.InputDataObjectType; import org.apache.airavata.model.application.io.OutputDataObjectType; import org.apache.airavata.model.data.movement.*; import org.apache.airavata.model.data.movement.DMType; -import org.apache.airavata.model.data.resource.DataResourceModel; import org.apache.airavata.model.error.*; import org.apache.airavata.model.experiment.*; import org.apache.airavata.model.job.JobModel; @@ -4070,100 +4066,6 @@ public class AiravataServerHandler implements Airavata.Iface { } } - /** - * * Data Manager Related API Methods - * * - */ - - @Override - @SecurityCheck - public String registerDataResource(AuthzToken authzToken, DataResourceModel dataResourceModel) throws InvalidRequestException, AiravataClientException, AiravataSystemException, AuthorizationException, TException { - try { - DataManager dataManager = DataManagerFactory.getDataManager(); - return dataManager.registerResource(dataResourceModel); - } catch (DataManagerException e) { - String msg = "Error in publishing the data resource"+dataResourceModel.getResourceName()+"."; - logger.error(msg, e); - AiravataSystemException exception = new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR); - exception.setMessage(msg+" More info : " + e.getMessage()); - throw exception; - } - } - - @Override - @SecurityCheck - public void updateDataResource(AuthzToken authzToken, DataResourceModel dataResourceModel) throws InvalidRequestException, AiravataClientException, AiravataSystemException, AuthorizationException, TException { - try { - DataManager dataManager = DataManagerFactory.getDataManager(); - dataManager.updateResource(dataResourceModel); - } catch (DataManagerException e) { - String msg = "Error in updating the data resource"+dataResourceModel.getResourceName()+"."; - logger.error(msg, e); - AiravataSystemException exception = new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR); - exception.setMessage(msg+" More info : " + e.getMessage()); - throw exception; - } - } - - @Override - @SecurityCheck - public void removeDataResource(AuthzToken authzToken, String resourceId) throws InvalidRequestException, AiravataClientException, AiravataSystemException, AuthorizationException, TException { - try { - DataManager dataManager = DataManagerFactory.getDataManager(); - dataManager.removeResource(resourceId); - } catch (DataManagerException e) { - String msg = "Error in removing the data resource "+resourceId+"."; - logger.error(msg, e); - AiravataSystemException exception = new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR); - exception.setMessage(msg+" More info : " + e.getMessage()); - throw exception; - } - } - - @Override - @SecurityCheck - public DataResourceModel getDataResource(AuthzToken authzToken, String resourceId) throws InvalidRequestException, - AiravataClientException, AiravataSystemException, AuthorizationException, TException { - try { - DataManager dataManager = DataManagerFactory.getDataManager(); - return dataManager.getResource(resourceId); - } catch (DataManagerException e) { - String msg = "Error in retreiving the data resource "+resourceId+"."; - logger.error(msg, e); - AiravataSystemException exception = new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR); - exception.setMessage(msg+" More info : " + e.getMessage()); - throw exception; - } - } - - @Override - public String copyDataResource(AuthzToken authzToken, String resourceId, String destStorageResourceId, String destinationParentPath) throws InvalidRequestException, AiravataClientException, AiravataSystemException, AuthorizationException, TException { - try { - DataManager dataManager = DataManagerFactory.getDataManager(); - return dataManager.copyResource(resourceId, destStorageResourceId, destinationParentPath); - } catch (DataManagerException e) { - String msg = "Error in copying the data resource "+resourceId+"."; - logger.error(msg, e); - AiravataSystemException exception = new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR); - exception.setMessage(msg+" More info : " + e.getMessage()); - throw exception; - } - } - - @Override - public String copyDataReplica(AuthzToken authzToken, String resourceId, String replicaId, String destStorageResourceId, String destinationParentPath) throws InvalidRequestException, AiravataClientException, AiravataSystemException, AuthorizationException, TException { - try { - DataManager dataManager = DataManagerFactory.getDataManager(); - return dataManager.copyReplica(resourceId, replicaId, destStorageResourceId, destinationParentPath); - } catch (DataManagerException e) { - String msg = "Error in copying the data replica "+replicaId+"."; - logger.error(msg, e); - AiravataSystemException exception = new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR); - exception.setMessage(msg+" More info : " + e.getMessage()); - throw exception; - } - } - private WorkflowCatalog getWorkflowCatalog() { if (workflowCatalog == null) { try { http://git-wip-us.apache.org/repos/asf/airavata/blob/5881af94/airavata-api/airavata-api-server/src/test/java/org/apache/airavata/api/server/handler/utils/DataCatInit.java ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-api-server/src/test/java/org/apache/airavata/api/server/handler/utils/DataCatInit.java b/airavata-api/airavata-api-server/src/test/java/org/apache/airavata/api/server/handler/utils/DataCatInit.java deleted file mode 100644 index 2b55692..0000000 --- a/airavata-api/airavata-api-server/src/test/java/org/apache/airavata/api/server/handler/utils/DataCatInit.java +++ /dev/null @@ -1,315 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -package org.apache.airavata.api.server.handler.utils; - -import org.apache.airavata.common.exception.ApplicationSettingsException; -import org.apache.airavata.common.utils.ServerSettings; -import org.apache.airavata.registry.core.data.catalog.utils.DataCatalogConstants; -import org.apache.derby.drda.NetworkServerControl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.InetAddress; -import java.net.URI; -import java.sql.*; -import java.util.StringTokenizer; - -public class DataCatInit { - private static final Logger logger = LoggerFactory.getLogger(DataCatInit.class); - public static final String DERBY_SERVER_MODE_SYS_PROPERTY = "derby.drda.startNetworkServer"; - public String scriptName = "datacatalog-derby.sql"; - private NetworkServerControl server; - private static final String delimiter = ";"; - private String jdbcUrl = null; - private String jdbcDriver = null; - private String jdbcUser = null; - private String jdbcPassword = null; - - public DataCatInit(String scriptName) { - this.scriptName = scriptName; - } - - public static boolean checkStringBufferEndsWith(StringBuffer buffer, String suffix) { - if (suffix.length() > buffer.length()) { - return false; - } - // this loop is done on purpose to avoid memory allocation performance - // problems on various JDKs - // StringBuffer.lastIndexOf() was introduced in jdk 1.4 and - // implementation is ok though does allocation/copying - // StringBuffer.toString().endsWith() does massive memory - // allocation/copying on JDK 1.5 - // See http://issues.apache.org/bugzilla/show_bug.cgi?id=37169 - int endIndex = suffix.length() - 1; - int bufferIndex = buffer.length() - 1; - while (endIndex >= 0) { - if (buffer.charAt(bufferIndex) != suffix.charAt(endIndex)) { - return false; - } - bufferIndex--; - endIndex--; - } - return true; - } - - private static boolean isServerStarted(NetworkServerControl server, int ntries) - { - for (int i = 1; i <= ntries; i ++) - { - try { - Thread.sleep(500); - server.ping(); - return true; - } - catch (Exception e) { - if (i == ntries) - return false; - } - } - return false; - } - - public void initializeDB() { - try{ - jdbcDriver = ServerSettings.getSetting("datacatalog.jdbc.driver"); - jdbcUrl = ServerSettings.getSetting("datacatalog.jdbc.url"); - jdbcUser = ServerSettings.getSetting("datacatalog.jdbc.user"); - jdbcPassword = ServerSettings.getSetting("datacatalog.jdbc.password"); - jdbcUrl = jdbcUrl + "?" + "user=" + jdbcUser + "&" + "password=" + jdbcPassword; - } catch (ApplicationSettingsException e) { - logger.error("Unable to read properties", e); - } - - startDerbyInServerMode(); - if(!isServerStarted(server, 20)){ - throw new RuntimeException("Derby server could not started within five seconds..."); - } - Connection conn = null; - try { - Class.forName(jdbcDriver).newInstance(); - conn = DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPassword); - if (!isDatabaseStructureCreated(DataCatalogConstants.CONFIGURATION, conn)) { - executeSQLScript(conn); - logger.info("New Database created for Data Catalog !!!"); - } else { - logger.debug("Database already created for Data Catalog!"); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - throw new RuntimeException("Database failure", e); - } finally { - try { - if (conn != null){ - if (!conn.getAutoCommit()) { - conn.commit(); - } - conn.close(); - } - } catch (SQLException e) { - logger.error(e.getMessage(), e); - } - } - } - - public static boolean isDatabaseStructureCreated(String tableName, Connection conn) { - try { - System.out.println("Running a query to test the database tables existence."); - // check whether the tables are already created with a query - Statement statement = null; - try { - statement = conn.createStatement(); - ResultSet rs = statement.executeQuery("select * from " + tableName); - if (rs != null) { - rs.close(); - } - } finally { - try { - if (statement != null) { - statement.close(); - } - } catch (SQLException e) { - return false; - } - } - } catch (SQLException e) { - return false; - } - - return true; - } - - private void executeSQLScript(Connection conn) throws Exception { - StringBuffer sql = new StringBuffer(); - BufferedReader reader = null; - try{ - - InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(scriptName); - reader = new BufferedReader(new InputStreamReader(inputStream)); - String line; - while ((line = reader.readLine()) != null) { - line = line.trim(); - if (line.startsWith("//")) { - continue; - } - if (line.startsWith("--")) { - continue; - } - StringTokenizer st = new StringTokenizer(line); - if (st.hasMoreTokens()) { - String token = st.nextToken(); - if ("REM".equalsIgnoreCase(token)) { - continue; - } - } - sql.append(" ").append(line); - - // SQL defines "--" as a comment to EOL - // and in Oracle it may contain a hint - // so we cannot just remove it, instead we must end it - if (line.indexOf("--") >= 0) { - sql.append("\n"); - } - if ((checkStringBufferEndsWith(sql, delimiter))) { - executeSQL(sql.substring(0, sql.length() - delimiter.length()), conn); - sql.replace(0, sql.length(), ""); - } - } - // Catch any statements not followed by ; - if (sql.length() > 0) { - executeSQL(sql.toString(), conn); - } - }catch (IOException e){ - logger.error("Error occurred while executing SQL script for creating Airavata Data Catalog database", e); - throw new Exception("Error occurred while executing SQL script for creating Airavata Data Catalog database", e); - }finally { - if (reader != null) { - reader.close(); - } - } - } - - private static void executeSQL(String sql, Connection conn) throws Exception { - // Check and ignore empty statements - if ("".equals(sql.trim())) { - return; - } - - Statement statement = null; - try { - logger.debug("SQL : " + sql); - - boolean ret; - int updateCount = 0, updateCountTotal = 0; - statement = conn.createStatement(); - ret = statement.execute(sql); - updateCount = statement.getUpdateCount(); - do { - if (!ret) { - if (updateCount != -1) { - updateCountTotal += updateCount; - } - } - ret = statement.getMoreResults(); - if (ret) { - updateCount = statement.getUpdateCount(); - } - } while (ret); - - logger.debug(sql + " : " + updateCountTotal + " rows affected"); - - SQLWarning warning = conn.getWarnings(); - while (warning != null) { - logger.warn(warning + " sql warning"); - warning = warning.getNextWarning(); - } - conn.clearWarnings(); - } catch (SQLException e) { - if (e.getSQLState().equals("X0Y32")) { - // eliminating the table already exception for the derby - // database - logger.info("Table Already Exists", e); - } else { - throw new Exception("Error occurred while executing : " + sql, e); - } - } finally { - if (statement != null) { - try { - statement.close(); - } catch (SQLException e) { - logger.error("Error occurred while closing result set.", e); - } - } - } - } - - private void startDerbyInServerMode() { - try { - System.setProperty(DERBY_SERVER_MODE_SYS_PROPERTY, "true"); - server = new NetworkServerControl(InetAddress.getByName("0.0.0.0"), - 20000, - jdbcUser, jdbcPassword); - java.io.PrintWriter consoleWriter = new java.io.PrintWriter(System.out, true); - server.start(consoleWriter); - } catch (IOException e) { - logger.error("Unable to start Apache derby in the server mode! Check whether " + - "specified port is available"); - } catch (Exception e) { - logger.error("Unable to start Apache derby in the server mode! Check whether " + - "specified port is available"); - } - - } - - public static int getPort(String jdbcURL){ - try{ - String cleanURI = jdbcURL.substring(5); - URI uri = URI.create(cleanURI); - return uri.getPort(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - return -1; - } - } - - private void startDerbyInEmbeddedMode(){ - try { - Class.forName("org.apache.derby.jdbc.EmbeddedDriver"); - DriverManager.getConnection("jdbc:derby:memory:unit-testing-jpa;create=true").close(); - } catch (ClassNotFoundException e) { - logger.error(e.getMessage(), e); - } catch (SQLException e) { - logger.error(e.getMessage(), e); - } - } - - public void stopDerbyServer() { - try { - server.shutdown(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } -}
