DImuthuUpe commented on a change in pull request #260:
URL: https://github.com/apache/airavata/pull/260#discussion_r478810191
##########
File path:
airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
##########
@@ -692,6 +695,80 @@ public String generateAndRegisterSSHKeys(AuthzToken
authzToken, String descripti
}
}
+ @Override
+ public void validateStorageLimit(AuthzToken authzToken, ExperimentModel
experiment, String storageResourceId)
+ throws InvalidRequestException, AiravataClientException,
AiravataSystemException, TException {
+ String gatewayId = authzToken.getClaimsMap().get(Constants.GATEWAY_ID);
+ RegistryService.Client regClient = registryClientPool.getResource();
+ try {
+ StoragePreference storagePreference =
regClient.getGatewayStoragePreference(gatewayId, storageResourceId);
+ if(storagePreference.getUserStorageQuota() == 0)
Review comment:
if (
##########
File path:
airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
##########
@@ -692,6 +695,80 @@ public String generateAndRegisterSSHKeys(AuthzToken
authzToken, String descripti
}
}
+ @Override
+ public void validateStorageLimit(AuthzToken authzToken, ExperimentModel
experiment, String storageResourceId)
+ throws InvalidRequestException, AiravataClientException,
AiravataSystemException, TException {
+ String gatewayId = authzToken.getClaimsMap().get(Constants.GATEWAY_ID);
+ RegistryService.Client regClient = registryClientPool.getResource();
+ try {
+ StoragePreference storagePreference =
regClient.getGatewayStoragePreference(gatewayId, storageResourceId);
+ if(storagePreference.getUserStorageQuota() == 0)
+ return; //quota isn't configured
+
+ String groupResourceProfileId =
experiment.getUserConfigurationData().getGroupResourceProfileId();
+ if (groupResourceProfileId == null) {
+ logger.error("Experiment not configured with a Group Resource
Profile: {}", experiment.getExperimentId());
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("Experiment not configured with a Group
Resource Profile: " + experiment.getExperimentId());
+ throw exception;
+ }
+
+ GatewayResourceProfile gatewayResourceProfile =
regClient.getGatewayResourceProfile(gatewayId);
+ String token;
+
if(isValid(storagePreference.getResourceSpecificCredentialStoreToken()))
Review comment:
if (
##########
File path:
airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
##########
@@ -692,6 +695,80 @@ public String generateAndRegisterSSHKeys(AuthzToken
authzToken, String descripti
}
}
+ @Override
+ public void validateStorageLimit(AuthzToken authzToken, ExperimentModel
experiment, String storageResourceId)
+ throws InvalidRequestException, AiravataClientException,
AiravataSystemException, TException {
+ String gatewayId = authzToken.getClaimsMap().get(Constants.GATEWAY_ID);
+ RegistryService.Client regClient = registryClientPool.getResource();
+ try {
+ StoragePreference storagePreference =
regClient.getGatewayStoragePreference(gatewayId, storageResourceId);
+ if(storagePreference.getUserStorageQuota() == 0)
+ return; //quota isn't configured
+
+ String groupResourceProfileId =
experiment.getUserConfigurationData().getGroupResourceProfileId();
+ if (groupResourceProfileId == null) {
+ logger.error("Experiment not configured with a Group Resource
Profile: {}", experiment.getExperimentId());
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("Experiment not configured with a Group
Resource Profile: " + experiment.getExperimentId());
+ throw exception;
+ }
+
+ GatewayResourceProfile gatewayResourceProfile =
regClient.getGatewayResourceProfile(gatewayId);
+ String token;
+
if(isValid(storagePreference.getResourceSpecificCredentialStoreToken()))
+ token =
storagePreference.getResourceSpecificCredentialStoreToken();
+ else
+ token = gatewayResourceProfile.getCredentialStoreToken();
+
+ StorageResourceAdaptor adaptor =
AdaptorSupportImpl.getInstance().fetchStorageAdaptor(
+ gatewayId,
+ storageResourceId,
+ DataMovementProtocol.SCP,
+ token,
+ storagePreference.getLoginUserName());
+ String experimentDataDir =
experiment.getUserConfigurationData().getExperimentDataDir();
+ String userDirectory = getUserDirectory(experimentDataDir);
+
+ //shouldn't retrieve size in giga bytes since the du -s command
rounds off the value to a higher bound.
+ CommandOutput output = adaptor.executeCommand("du -sm",
userDirectory);
+ if(!output.getStdError().isEmpty()) {
+ logger.error("The experiment data directory + " +
experimentDataDir + " configured on the experiment is wrong.");
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("The experiment data directory + " +
experimentDataDir + " configured on the experiment is wrong.");
+ throw exception;
+ }
+
+ double userDirectorySize =
Double.parseDouble(output.getStdOut().substring(0, output.getStdOut().length()
- 2).trim())/1024.0;
+ if(userDirectorySize >= storagePreference.getUserStorageQuota()) {
+ logger.error(String.format("The user directory %s has exceeded
the storage quota: %d GB", userDirectory,
storagePreference.getUserStorageQuota()));
+ AiravataSystemException exception = new
AiravataSystemException();
+ exception.setMessage(String.format("The user directory %s has
exceeded the storage quota: %d GB", userDirectory,
storagePreference.getUserStorageQuota()));
+ throw exception;
+ }
+ } catch (Exception ex) {
+ logger.error(ex.getMessage(), ex);
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage(ex.getMessage());
+ throw exception;
+ }
+ }
+
+ private String getUserDirectory(String experimentDataDirectory) {
+ //experiment directory is inside a project directory which is inside a
user directory
+ String[] directories = experimentDataDirectory.split("/");
+ StringBuilder userDirectory = new StringBuilder();
+
+ for(int i = 0; i < directories.length - 2; i++)
+ if(!directories[i].isEmpty())
Review comment:
if (
##########
File path:
airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
##########
@@ -692,6 +695,80 @@ public String generateAndRegisterSSHKeys(AuthzToken
authzToken, String descripti
}
}
+ @Override
+ public void validateStorageLimit(AuthzToken authzToken, ExperimentModel
experiment, String storageResourceId)
+ throws InvalidRequestException, AiravataClientException,
AiravataSystemException, TException {
+ String gatewayId = authzToken.getClaimsMap().get(Constants.GATEWAY_ID);
+ RegistryService.Client regClient = registryClientPool.getResource();
+ try {
+ StoragePreference storagePreference =
regClient.getGatewayStoragePreference(gatewayId, storageResourceId);
+ if(storagePreference.getUserStorageQuota() == 0)
+ return; //quota isn't configured
+
+ String groupResourceProfileId =
experiment.getUserConfigurationData().getGroupResourceProfileId();
+ if (groupResourceProfileId == null) {
+ logger.error("Experiment not configured with a Group Resource
Profile: {}", experiment.getExperimentId());
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("Experiment not configured with a Group
Resource Profile: " + experiment.getExperimentId());
+ throw exception;
+ }
+
+ GatewayResourceProfile gatewayResourceProfile =
regClient.getGatewayResourceProfile(gatewayId);
+ String token;
+
if(isValid(storagePreference.getResourceSpecificCredentialStoreToken()))
+ token =
storagePreference.getResourceSpecificCredentialStoreToken();
+ else
+ token = gatewayResourceProfile.getCredentialStoreToken();
+
+ StorageResourceAdaptor adaptor =
AdaptorSupportImpl.getInstance().fetchStorageAdaptor(
+ gatewayId,
+ storageResourceId,
+ DataMovementProtocol.SCP,
+ token,
+ storagePreference.getLoginUserName());
+ String experimentDataDir =
experiment.getUserConfigurationData().getExperimentDataDir();
+ String userDirectory = getUserDirectory(experimentDataDir);
+
+ //shouldn't retrieve size in giga bytes since the du -s command
rounds off the value to a higher bound.
+ CommandOutput output = adaptor.executeCommand("du -sm",
userDirectory);
+ if(!output.getStdError().isEmpty()) {
+ logger.error("The experiment data directory + " +
experimentDataDir + " configured on the experiment is wrong.");
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("The experiment data directory + " +
experimentDataDir + " configured on the experiment is wrong.");
+ throw exception;
+ }
+
+ double userDirectorySize =
Double.parseDouble(output.getStdOut().substring(0, output.getStdOut().length()
- 2).trim())/1024.0;
Review comment:
And do you think that the output format is same for all operating
systems (Linux)? If not, your algorithm might be wrong
##########
File path:
airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
##########
@@ -692,6 +695,80 @@ public String generateAndRegisterSSHKeys(AuthzToken
authzToken, String descripti
}
}
+ @Override
+ public void validateStorageLimit(AuthzToken authzToken, ExperimentModel
experiment, String storageResourceId)
+ throws InvalidRequestException, AiravataClientException,
AiravataSystemException, TException {
+ String gatewayId = authzToken.getClaimsMap().get(Constants.GATEWAY_ID);
+ RegistryService.Client regClient = registryClientPool.getResource();
+ try {
+ StoragePreference storagePreference =
regClient.getGatewayStoragePreference(gatewayId, storageResourceId);
+ if(storagePreference.getUserStorageQuota() == 0)
+ return; //quota isn't configured
+
+ String groupResourceProfileId =
experiment.getUserConfigurationData().getGroupResourceProfileId();
+ if (groupResourceProfileId == null) {
+ logger.error("Experiment not configured with a Group Resource
Profile: {}", experiment.getExperimentId());
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("Experiment not configured with a Group
Resource Profile: " + experiment.getExperimentId());
+ throw exception;
+ }
+
+ GatewayResourceProfile gatewayResourceProfile =
regClient.getGatewayResourceProfile(gatewayId);
+ String token;
+
if(isValid(storagePreference.getResourceSpecificCredentialStoreToken()))
+ token =
storagePreference.getResourceSpecificCredentialStoreToken();
+ else
+ token = gatewayResourceProfile.getCredentialStoreToken();
+
+ StorageResourceAdaptor adaptor =
AdaptorSupportImpl.getInstance().fetchStorageAdaptor(
+ gatewayId,
+ storageResourceId,
+ DataMovementProtocol.SCP,
+ token,
+ storagePreference.getLoginUserName());
+ String experimentDataDir =
experiment.getUserConfigurationData().getExperimentDataDir();
+ String userDirectory = getUserDirectory(experimentDataDir);
+
+ //shouldn't retrieve size in giga bytes since the du -s command
rounds off the value to a higher bound.
+ CommandOutput output = adaptor.executeCommand("du -sm",
userDirectory);
+ if(!output.getStdError().isEmpty()) {
Review comment:
Are you sure that output.getStdError() is not null?
##########
File path:
airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
##########
@@ -692,6 +695,80 @@ public String generateAndRegisterSSHKeys(AuthzToken
authzToken, String descripti
}
}
+ @Override
+ public void validateStorageLimit(AuthzToken authzToken, ExperimentModel
experiment, String storageResourceId)
+ throws InvalidRequestException, AiravataClientException,
AiravataSystemException, TException {
+ String gatewayId = authzToken.getClaimsMap().get(Constants.GATEWAY_ID);
+ RegistryService.Client regClient = registryClientPool.getResource();
+ try {
+ StoragePreference storagePreference =
regClient.getGatewayStoragePreference(gatewayId, storageResourceId);
+ if(storagePreference.getUserStorageQuota() == 0)
+ return; //quota isn't configured
+
+ String groupResourceProfileId =
experiment.getUserConfigurationData().getGroupResourceProfileId();
+ if (groupResourceProfileId == null) {
+ logger.error("Experiment not configured with a Group Resource
Profile: {}", experiment.getExperimentId());
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("Experiment not configured with a Group
Resource Profile: " + experiment.getExperimentId());
+ throw exception;
+ }
+
+ GatewayResourceProfile gatewayResourceProfile =
regClient.getGatewayResourceProfile(gatewayId);
+ String token;
+
if(isValid(storagePreference.getResourceSpecificCredentialStoreToken()))
+ token =
storagePreference.getResourceSpecificCredentialStoreToken();
+ else
+ token = gatewayResourceProfile.getCredentialStoreToken();
+
+ StorageResourceAdaptor adaptor =
AdaptorSupportImpl.getInstance().fetchStorageAdaptor(
+ gatewayId,
+ storageResourceId,
+ DataMovementProtocol.SCP,
+ token,
+ storagePreference.getLoginUserName());
+ String experimentDataDir =
experiment.getUserConfigurationData().getExperimentDataDir();
+ String userDirectory = getUserDirectory(experimentDataDir);
+
+ //shouldn't retrieve size in giga bytes since the du -s command
rounds off the value to a higher bound.
+ CommandOutput output = adaptor.executeCommand("du -sm",
userDirectory);
+ if(!output.getStdError().isEmpty()) {
+ logger.error("The experiment data directory + " +
experimentDataDir + " configured on the experiment is wrong.");
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("The experiment data directory + " +
experimentDataDir + " configured on the experiment is wrong.");
+ throw exception;
+ }
+
+ double userDirectorySize =
Double.parseDouble(output.getStdOut().substring(0, output.getStdOut().length()
- 2).trim())/1024.0;
+ if(userDirectorySize >= storagePreference.getUserStorageQuota()) {
+ logger.error(String.format("The user directory %s has exceeded
the storage quota: %d GB", userDirectory,
storagePreference.getUserStorageQuota()));
+ AiravataSystemException exception = new
AiravataSystemException();
+ exception.setMessage(String.format("The user directory %s has
exceeded the storage quota: %d GB", userDirectory,
storagePreference.getUserStorageQuota()));
+ throw exception;
+ }
+ } catch (Exception ex) {
+ logger.error(ex.getMessage(), ex);
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage(ex.getMessage());
+ throw exception;
+ }
+ }
+
+ private String getUserDirectory(String experimentDataDirectory) {
Review comment:
This is a blind algorithm. At least provide your assumptions for
experimentDataDirectory value as comments
##########
File path:
airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
##########
@@ -692,6 +695,80 @@ public String generateAndRegisterSSHKeys(AuthzToken
authzToken, String descripti
}
}
+ @Override
+ public void validateStorageLimit(AuthzToken authzToken, ExperimentModel
experiment, String storageResourceId)
+ throws InvalidRequestException, AiravataClientException,
AiravataSystemException, TException {
+ String gatewayId = authzToken.getClaimsMap().get(Constants.GATEWAY_ID);
+ RegistryService.Client regClient = registryClientPool.getResource();
+ try {
+ StoragePreference storagePreference =
regClient.getGatewayStoragePreference(gatewayId, storageResourceId);
+ if(storagePreference.getUserStorageQuota() == 0)
+ return; //quota isn't configured
+
+ String groupResourceProfileId =
experiment.getUserConfigurationData().getGroupResourceProfileId();
+ if (groupResourceProfileId == null) {
+ logger.error("Experiment not configured with a Group Resource
Profile: {}", experiment.getExperimentId());
Review comment:
Experiment {} not configured with a Group Resource Profile
##########
File path:
airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
##########
@@ -692,6 +695,80 @@ public String generateAndRegisterSSHKeys(AuthzToken
authzToken, String descripti
}
}
+ @Override
+ public void validateStorageLimit(AuthzToken authzToken, ExperimentModel
experiment, String storageResourceId)
+ throws InvalidRequestException, AiravataClientException,
AiravataSystemException, TException {
+ String gatewayId = authzToken.getClaimsMap().get(Constants.GATEWAY_ID);
+ RegistryService.Client regClient = registryClientPool.getResource();
+ try {
+ StoragePreference storagePreference =
regClient.getGatewayStoragePreference(gatewayId, storageResourceId);
+ if(storagePreference.getUserStorageQuota() == 0)
+ return; //quota isn't configured
+
+ String groupResourceProfileId =
experiment.getUserConfigurationData().getGroupResourceProfileId();
+ if (groupResourceProfileId == null) {
+ logger.error("Experiment not configured with a Group Resource
Profile: {}", experiment.getExperimentId());
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("Experiment not configured with a Group
Resource Profile: " + experiment.getExperimentId());
+ throw exception;
+ }
+
+ GatewayResourceProfile gatewayResourceProfile =
regClient.getGatewayResourceProfile(gatewayId);
+ String token;
+
if(isValid(storagePreference.getResourceSpecificCredentialStoreToken()))
+ token =
storagePreference.getResourceSpecificCredentialStoreToken();
+ else
+ token = gatewayResourceProfile.getCredentialStoreToken();
+
Review comment:
Null check for the token
##########
File path:
airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
##########
@@ -692,6 +695,80 @@ public String generateAndRegisterSSHKeys(AuthzToken
authzToken, String descripti
}
}
+ @Override
+ public void validateStorageLimit(AuthzToken authzToken, ExperimentModel
experiment, String storageResourceId)
+ throws InvalidRequestException, AiravataClientException,
AiravataSystemException, TException {
+ String gatewayId = authzToken.getClaimsMap().get(Constants.GATEWAY_ID);
+ RegistryService.Client regClient = registryClientPool.getResource();
Review comment:
Return the regClient back to the pool after using
##########
File path:
airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
##########
@@ -692,6 +695,80 @@ public String generateAndRegisterSSHKeys(AuthzToken
authzToken, String descripti
}
}
+ @Override
+ public void validateStorageLimit(AuthzToken authzToken, ExperimentModel
experiment, String storageResourceId)
+ throws InvalidRequestException, AiravataClientException,
AiravataSystemException, TException {
+ String gatewayId = authzToken.getClaimsMap().get(Constants.GATEWAY_ID);
+ RegistryService.Client regClient = registryClientPool.getResource();
+ try {
+ StoragePreference storagePreference =
regClient.getGatewayStoragePreference(gatewayId, storageResourceId);
+ if(storagePreference.getUserStorageQuota() == 0)
+ return; //quota isn't configured
+
+ String groupResourceProfileId =
experiment.getUserConfigurationData().getGroupResourceProfileId();
+ if (groupResourceProfileId == null) {
+ logger.error("Experiment not configured with a Group Resource
Profile: {}", experiment.getExperimentId());
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("Experiment not configured with a Group
Resource Profile: " + experiment.getExperimentId());
+ throw exception;
+ }
+
+ GatewayResourceProfile gatewayResourceProfile =
regClient.getGatewayResourceProfile(gatewayId);
+ String token;
+
if(isValid(storagePreference.getResourceSpecificCredentialStoreToken()))
+ token =
storagePreference.getResourceSpecificCredentialStoreToken();
+ else
+ token = gatewayResourceProfile.getCredentialStoreToken();
+
+ StorageResourceAdaptor adaptor =
AdaptorSupportImpl.getInstance().fetchStorageAdaptor(
+ gatewayId,
+ storageResourceId,
+ DataMovementProtocol.SCP,
+ token,
+ storagePreference.getLoginUserName());
+ String experimentDataDir =
experiment.getUserConfigurationData().getExperimentDataDir();
+ String userDirectory = getUserDirectory(experimentDataDir);
+
+ //shouldn't retrieve size in giga bytes since the du -s command
rounds off the value to a higher bound.
+ CommandOutput output = adaptor.executeCommand("du -sm",
userDirectory);
+ if(!output.getStdError().isEmpty()) {
+ logger.error("The experiment data directory + " +
experimentDataDir + " configured on the experiment is wrong.");
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("The experiment data directory + " +
experimentDataDir + " configured on the experiment is wrong.");
+ throw exception;
+ }
+
+ double userDirectorySize =
Double.parseDouble(output.getStdOut().substring(0, output.getStdOut().length()
- 2).trim())/1024.0;
+ if(userDirectorySize >= storagePreference.getUserStorageQuota()) {
+ logger.error(String.format("The user directory %s has exceeded
the storage quota: %d GB", userDirectory,
storagePreference.getUserStorageQuota()));
+ AiravataSystemException exception = new
AiravataSystemException();
+ exception.setMessage(String.format("The user directory %s has
exceeded the storage quota: %d GB", userDirectory,
storagePreference.getUserStorageQuota()));
+ throw exception;
+ }
+ } catch (Exception ex) {
+ logger.error(ex.getMessage(), ex);
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage(ex.getMessage());
+ throw exception;
+ }
+ }
+
+ private String getUserDirectory(String experimentDataDirectory) {
+ //experiment directory is inside a project directory which is inside a
user directory
+ String[] directories = experimentDataDirectory.split("/");
+ StringBuilder userDirectory = new StringBuilder();
+
+ for(int i = 0; i < directories.length - 2; i++)
Review comment:
Possible index out of bound error here
##########
File path:
airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
##########
@@ -692,6 +695,80 @@ public String generateAndRegisterSSHKeys(AuthzToken
authzToken, String descripti
}
}
+ @Override
+ public void validateStorageLimit(AuthzToken authzToken, ExperimentModel
experiment, String storageResourceId)
+ throws InvalidRequestException, AiravataClientException,
AiravataSystemException, TException {
+ String gatewayId = authzToken.getClaimsMap().get(Constants.GATEWAY_ID);
+ RegistryService.Client regClient = registryClientPool.getResource();
+ try {
+ StoragePreference storagePreference =
regClient.getGatewayStoragePreference(gatewayId, storageResourceId);
+ if(storagePreference.getUserStorageQuota() == 0)
+ return; //quota isn't configured
+
+ String groupResourceProfileId =
experiment.getUserConfigurationData().getGroupResourceProfileId();
+ if (groupResourceProfileId == null) {
+ logger.error("Experiment not configured with a Group Resource
Profile: {}", experiment.getExperimentId());
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("Experiment not configured with a Group
Resource Profile: " + experiment.getExperimentId());
+ throw exception;
+ }
+
+ GatewayResourceProfile gatewayResourceProfile =
regClient.getGatewayResourceProfile(gatewayId);
+ String token;
+
if(isValid(storagePreference.getResourceSpecificCredentialStoreToken()))
+ token =
storagePreference.getResourceSpecificCredentialStoreToken();
+ else
+ token = gatewayResourceProfile.getCredentialStoreToken();
+
+ StorageResourceAdaptor adaptor =
AdaptorSupportImpl.getInstance().fetchStorageAdaptor(
+ gatewayId,
+ storageResourceId,
+ DataMovementProtocol.SCP,
+ token,
+ storagePreference.getLoginUserName());
+ String experimentDataDir =
experiment.getUserConfigurationData().getExperimentDataDir();
+ String userDirectory = getUserDirectory(experimentDataDir);
+
+ //shouldn't retrieve size in giga bytes since the du -s command
rounds off the value to a higher bound.
+ CommandOutput output = adaptor.executeCommand("du -sm",
userDirectory);
Review comment:
Wrap this with try catch block and print as much as possible information
in the catch in case of an exception. Letting this to catch at main try catch
looses valuable information when debugging
##########
File path:
airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
##########
@@ -692,6 +695,80 @@ public String generateAndRegisterSSHKeys(AuthzToken
authzToken, String descripti
}
}
+ @Override
+ public void validateStorageLimit(AuthzToken authzToken, ExperimentModel
experiment, String storageResourceId)
+ throws InvalidRequestException, AiravataClientException,
AiravataSystemException, TException {
+ String gatewayId = authzToken.getClaimsMap().get(Constants.GATEWAY_ID);
+ RegistryService.Client regClient = registryClientPool.getResource();
+ try {
+ StoragePreference storagePreference =
regClient.getGatewayStoragePreference(gatewayId, storageResourceId);
+ if(storagePreference.getUserStorageQuota() == 0)
+ return; //quota isn't configured
+
+ String groupResourceProfileId =
experiment.getUserConfigurationData().getGroupResourceProfileId();
+ if (groupResourceProfileId == null) {
+ logger.error("Experiment not configured with a Group Resource
Profile: {}", experiment.getExperimentId());
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("Experiment not configured with a Group
Resource Profile: " + experiment.getExperimentId());
+ throw exception;
+ }
+
+ GatewayResourceProfile gatewayResourceProfile =
regClient.getGatewayResourceProfile(gatewayId);
+ String token;
+
if(isValid(storagePreference.getResourceSpecificCredentialStoreToken()))
+ token =
storagePreference.getResourceSpecificCredentialStoreToken();
+ else
+ token = gatewayResourceProfile.getCredentialStoreToken();
+
+ StorageResourceAdaptor adaptor =
AdaptorSupportImpl.getInstance().fetchStorageAdaptor(
+ gatewayId,
+ storageResourceId,
+ DataMovementProtocol.SCP,
+ token,
+ storagePreference.getLoginUserName());
+ String experimentDataDir =
experiment.getUserConfigurationData().getExperimentDataDir();
+ String userDirectory = getUserDirectory(experimentDataDir);
+
+ //shouldn't retrieve size in giga bytes since the du -s command
rounds off the value to a higher bound.
+ CommandOutput output = adaptor.executeCommand("du -sm",
userDirectory);
+ if(!output.getStdError().isEmpty()) {
+ logger.error("The experiment data directory + " +
experimentDataDir + " configured on the experiment is wrong.");
Review comment:
Be consistent with logging. logger.error("The experiment data directory
{} configured on the experiment is wrong.", experimentDataDir);
##########
File path:
airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
##########
@@ -692,6 +695,80 @@ public String generateAndRegisterSSHKeys(AuthzToken
authzToken, String descripti
}
}
+ @Override
+ public void validateStorageLimit(AuthzToken authzToken, ExperimentModel
experiment, String storageResourceId)
Review comment:
This method is a very risky one. You can see exceptions at most of the
lines. You can use fine grained exception handling by wrapping each sub section
with try catches to provide more debug information
##########
File path:
airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
##########
@@ -692,6 +695,80 @@ public String generateAndRegisterSSHKeys(AuthzToken
authzToken, String descripti
}
}
+ @Override
+ public void validateStorageLimit(AuthzToken authzToken, ExperimentModel
experiment, String storageResourceId)
+ throws InvalidRequestException, AiravataClientException,
AiravataSystemException, TException {
+ String gatewayId = authzToken.getClaimsMap().get(Constants.GATEWAY_ID);
+ RegistryService.Client regClient = registryClientPool.getResource();
+ try {
+ StoragePreference storagePreference =
regClient.getGatewayStoragePreference(gatewayId, storageResourceId);
+ if(storagePreference.getUserStorageQuota() == 0)
+ return; //quota isn't configured
+
+ String groupResourceProfileId =
experiment.getUserConfigurationData().getGroupResourceProfileId();
+ if (groupResourceProfileId == null) {
+ logger.error("Experiment not configured with a Group Resource
Profile: {}", experiment.getExperimentId());
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("Experiment not configured with a Group
Resource Profile: " + experiment.getExperimentId());
+ throw exception;
+ }
+
+ GatewayResourceProfile gatewayResourceProfile =
regClient.getGatewayResourceProfile(gatewayId);
+ String token;
+
if(isValid(storagePreference.getResourceSpecificCredentialStoreToken()))
+ token =
storagePreference.getResourceSpecificCredentialStoreToken();
+ else
+ token = gatewayResourceProfile.getCredentialStoreToken();
+
+ StorageResourceAdaptor adaptor =
AdaptorSupportImpl.getInstance().fetchStorageAdaptor(
+ gatewayId,
+ storageResourceId,
+ DataMovementProtocol.SCP,
+ token,
+ storagePreference.getLoginUserName());
+ String experimentDataDir =
experiment.getUserConfigurationData().getExperimentDataDir();
+ String userDirectory = getUserDirectory(experimentDataDir);
+
+ //shouldn't retrieve size in giga bytes since the du -s command
rounds off the value to a higher bound.
+ CommandOutput output = adaptor.executeCommand("du -sm",
userDirectory);
+ if(!output.getStdError().isEmpty()) {
+ logger.error("The experiment data directory + " +
experimentDataDir + " configured on the experiment is wrong.");
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("The experiment data directory + " +
experimentDataDir + " configured on the experiment is wrong.");
+ throw exception;
+ }
+
+ double userDirectorySize =
Double.parseDouble(output.getStdOut().substring(0, output.getStdOut().length()
- 2).trim())/1024.0;
+ if(userDirectorySize >= storagePreference.getUserStorageQuota()) {
Review comment:
if (
##########
File path:
airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
##########
@@ -692,6 +695,80 @@ public String generateAndRegisterSSHKeys(AuthzToken
authzToken, String descripti
}
}
+ @Override
+ public void validateStorageLimit(AuthzToken authzToken, ExperimentModel
experiment, String storageResourceId)
+ throws InvalidRequestException, AiravataClientException,
AiravataSystemException, TException {
+ String gatewayId = authzToken.getClaimsMap().get(Constants.GATEWAY_ID);
+ RegistryService.Client regClient = registryClientPool.getResource();
+ try {
+ StoragePreference storagePreference =
regClient.getGatewayStoragePreference(gatewayId, storageResourceId);
+ if(storagePreference.getUserStorageQuota() == 0)
+ return; //quota isn't configured
+
+ String groupResourceProfileId =
experiment.getUserConfigurationData().getGroupResourceProfileId();
+ if (groupResourceProfileId == null) {
+ logger.error("Experiment not configured with a Group Resource
Profile: {}", experiment.getExperimentId());
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("Experiment not configured with a Group
Resource Profile: " + experiment.getExperimentId());
+ throw exception;
+ }
+
+ GatewayResourceProfile gatewayResourceProfile =
regClient.getGatewayResourceProfile(gatewayId);
+ String token;
+
if(isValid(storagePreference.getResourceSpecificCredentialStoreToken()))
+ token =
storagePreference.getResourceSpecificCredentialStoreToken();
+ else
+ token = gatewayResourceProfile.getCredentialStoreToken();
+
+ StorageResourceAdaptor adaptor =
AdaptorSupportImpl.getInstance().fetchStorageAdaptor(
+ gatewayId,
+ storageResourceId,
+ DataMovementProtocol.SCP,
+ token,
+ storagePreference.getLoginUserName());
+ String experimentDataDir =
experiment.getUserConfigurationData().getExperimentDataDir();
+ String userDirectory = getUserDirectory(experimentDataDir);
+
+ //shouldn't retrieve size in giga bytes since the du -s command
rounds off the value to a higher bound.
+ CommandOutput output = adaptor.executeCommand("du -sm",
userDirectory);
+ if(!output.getStdError().isEmpty()) {
+ logger.error("The experiment data directory + " +
experimentDataDir + " configured on the experiment is wrong.");
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("The experiment data directory + " +
experimentDataDir + " configured on the experiment is wrong.");
+ throw exception;
+ }
+
+ double userDirectorySize =
Double.parseDouble(output.getStdOut().substring(0, output.getStdOut().length()
- 2).trim())/1024.0;
Review comment:
Print output.getStdOut() as a log before doing this. You don't know
what's happening here in case of and error
##########
File path:
airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
##########
@@ -692,6 +695,80 @@ public String generateAndRegisterSSHKeys(AuthzToken
authzToken, String descripti
}
}
+ @Override
+ public void validateStorageLimit(AuthzToken authzToken, ExperimentModel
experiment, String storageResourceId)
+ throws InvalidRequestException, AiravataClientException,
AiravataSystemException, TException {
+ String gatewayId = authzToken.getClaimsMap().get(Constants.GATEWAY_ID);
+ RegistryService.Client regClient = registryClientPool.getResource();
+ try {
+ StoragePreference storagePreference =
regClient.getGatewayStoragePreference(gatewayId, storageResourceId);
+ if(storagePreference.getUserStorageQuota() == 0)
+ return; //quota isn't configured
+
+ String groupResourceProfileId =
experiment.getUserConfigurationData().getGroupResourceProfileId();
+ if (groupResourceProfileId == null) {
+ logger.error("Experiment not configured with a Group Resource
Profile: {}", experiment.getExperimentId());
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("Experiment not configured with a Group
Resource Profile: " + experiment.getExperimentId());
+ throw exception;
+ }
+
+ GatewayResourceProfile gatewayResourceProfile =
regClient.getGatewayResourceProfile(gatewayId);
+ String token;
+
if(isValid(storagePreference.getResourceSpecificCredentialStoreToken()))
+ token =
storagePreference.getResourceSpecificCredentialStoreToken();
+ else
+ token = gatewayResourceProfile.getCredentialStoreToken();
+
+ StorageResourceAdaptor adaptor =
AdaptorSupportImpl.getInstance().fetchStorageAdaptor(
+ gatewayId,
+ storageResourceId,
+ DataMovementProtocol.SCP,
+ token,
+ storagePreference.getLoginUserName());
+ String experimentDataDir =
experiment.getUserConfigurationData().getExperimentDataDir();
+ String userDirectory = getUserDirectory(experimentDataDir);
+
+ //shouldn't retrieve size in giga bytes since the du -s command
rounds off the value to a higher bound.
+ CommandOutput output = adaptor.executeCommand("du -sm",
userDirectory);
+ if(!output.getStdError().isEmpty()) {
+ logger.error("The experiment data directory + " +
experimentDataDir + " configured on the experiment is wrong.");
+ AiravataSystemException exception = new
AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("The experiment data directory + " +
experimentDataDir + " configured on the experiment is wrong.");
+ throw exception;
+ }
+
+ double userDirectorySize =
Double.parseDouble(output.getStdOut().substring(0, output.getStdOut().length()
- 2).trim())/1024.0;
+ if(userDirectorySize >= storagePreference.getUserStorageQuota()) {
+ logger.error(String.format("The user directory %s has exceeded
the storage quota: %d GB", userDirectory,
storagePreference.getUserStorageQuota()));
+ AiravataSystemException exception = new
AiravataSystemException();
+ exception.setMessage(String.format("The user directory %s has
exceeded the storage quota: %d GB", userDirectory,
storagePreference.getUserStorageQuota()));
+ throw exception;
+ }
+ } catch (Exception ex) {
+ logger.error(ex.getMessage(), ex);
Review comment:
More detailed error message
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
