Repository: airavata Updated Branches: refs/heads/master 79750b7b5 -> 86f46412e
fixing https://issues.apache.org/jira/browse/AIRAVATA-1200 Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/7ec932b5 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/7ec932b5 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/7ec932b5 Branch: refs/heads/master Commit: 7ec932b5e48b9b1c42355ffc42abada59bed1eb3 Parents: a9e022b Author: lahiru <[email protected]> Authored: Tue May 6 13:42:40 2014 -0400 Committer: lahiru <[email protected]> Committed: Tue May 6 13:42:40 2014 -0400 ---------------------------------------------------------------------- .../server/OrchestratorServerHandler.java | 37 +++++++++++++++ .../core/OrchestratorConfiguration.java | 11 +++-- .../core/context/OrchestratorContext.java | 2 +- .../core/utils/OrchestratorUtils.java | 3 +- .../core/validator/JobMetadataValidator.java | 9 ++-- .../impl/AbstractJobMetadataValidator.java | 30 ------------ .../validator/impl/SimpleAppDataValidator.java | 40 ++++------------ .../airavata/orchestrator/cpi/Orchestrator.java | 11 +++++ .../cpi/impl/SimpleOrchestratorImpl.java | 50 ++++++++++++-------- .../orchestrator.cpi.service.thrift | 4 +- 10 files changed, 102 insertions(+), 95 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java index 073f65d..f456c65 100644 --- a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java +++ b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java @@ -107,6 +107,43 @@ public class OrchestratorServerHandler implements OrchestratorService.Iface { } return true; } + + + /** + * This method will validate the experiment before launching, if is failed we do not run the launch in airavata + * thrift service (only if validation is enabled + * @param experimentId + * @return + * @throws TException + */ + public boolean validateExperiment(String experimentId) throws TException { + //TODO: Write the Orchestrator implementaion + try { + List<TaskDetails> tasks = orchestrator.createTasks(experimentId); + if (tasks.size() > 1) { + log.info("There are multiple tasks for this experiment, So Orchestrator will launch multiple Jobs"); + } + List<String> ids = registry.getIds(RegistryModelType.WORKFLOW_NODE_DETAIL,WorkflowNodeConstants.EXPERIMENT_ID,experimentId); + for (String workflowNodeId : ids) { + WorkflowNodeDetails workflowNodeDetail = (WorkflowNodeDetails)registry.get(RegistryModelType.WORKFLOW_NODE_DETAIL, workflowNodeId); + List<Object> taskDetailList = registry.get(RegistryModelType.TASK_DETAIL, TaskDetailConstants.NODE_ID, workflowNodeId); + for (Object o : taskDetailList) { + TaskDetails taskID = (TaskDetails) o; + //iterate through all the generated tasks and performs the job submisssion+monitoring + Experiment experiment = (Experiment) registry.get(RegistryModelType.EXPERIMENT, experimentId); + if (experiment == null) { + log.error("Error retrieving the Experiment by the given experimentID: " + experimentId); + return false; + } + return orchestrator.validateExperiment(experiment, workflowNodeDetail, taskID); + } + } + + } catch (Exception e) { + throw new TException(e); + } + return false; + } public boolean terminateExperiment(String experimentId) throws TException { try { orchestrator.cancelExperiment(experimentId); http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java index fcb3552..b0cbe07 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java @@ -23,6 +23,7 @@ package org.apache.airavata.orchestrator.core; import org.apache.airavata.client.api.AiravataAPI; import java.net.URL; +import java.util.List; /** * This keeps configuration of orchestrator, mostly this keep static @@ -46,17 +47,17 @@ public class OrchestratorConfiguration { private boolean embeddedMode; - private String validatorClass; + private List<String> validatorClasses; private boolean enableValidation; - public String getValidatorClass() { - return validatorClass; + public List<String> getValidatorClasses() { + return validatorClasses; } - public void setValidatorClass(String validatorClass) { - this.validatorClass = validatorClass; + public void setValidatorClasses(List<String> validatorClasses) { + this.validatorClasses = validatorClasses; } public boolean isEmbeddedMode() { http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java index eb3f463..8de0482 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java @@ -29,7 +29,7 @@ import org.apache.airavata.registry.api.AiravataRegistry2; import org.apache.airavata.registry.cpi.Registry; /** - * This is the context object used in orchestrator whic h + * This is the context object used in orchestrator which */ public class OrchestratorContext { private List<GFACInstance> gfacInstanceList; http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java index 5abbe03..0af1ed8 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java @@ -21,6 +21,7 @@ package org.apache.airavata.orchestrator.core.utils; import java.io.IOException; +import java.util.Arrays; import org.apache.airavata.common.exception.ApplicationSettingsException; import org.apache.airavata.common.utils.ServerSettings; @@ -53,7 +54,7 @@ public class OrchestratorUtils { orchestratorConfiguration.setEmbeddedMode(Boolean.valueOf(ServerSettings.getSetting(OrchestratorConstants.EMBEDDED_MODE))); orchestratorConfiguration.setEnableValidation(Boolean.valueOf(ServerSettings.getSetting(OrchestratorConstants.ENABLE_VALIDATION))); if (orchestratorConfiguration.isEnableValidation()) { - orchestratorConfiguration.setValidatorClass((String) ServerSettings.getSetting(OrchestratorConstants.JOB_VALIDATOR)); + orchestratorConfiguration.setValidatorClasses(Arrays.asList(ServerSettings.getSetting(OrchestratorConstants.JOB_VALIDATOR).split(","))); } return orchestratorConfiguration; } http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java index d365c87..1c4eaba 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java @@ -20,13 +20,12 @@ */ package org.apache.airavata.orchestrator.core.validator; +import org.apache.airavata.model.workspace.experiment.Experiment; +import org.apache.airavata.model.workspace.experiment.TaskDetails; +import org.apache.airavata.model.workspace.experiment.WorkflowNodeDetails; import org.apache.airavata.orchestrator.core.exception.OrchestratorException; public interface JobMetadataValidator { - boolean runBasicValidation(String experimentID)throws OrchestratorException; - - boolean runAppSpecificValidation(String experimentID)throws OrchestratorException; - - boolean validate(String experimentID) throws OrchestratorException; + boolean validate(Experiment experiment, WorkflowNodeDetails workflowNodeDetail, TaskDetails taskID) throws OrchestratorException; } http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/AbstractJobMetadataValidator.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/AbstractJobMetadataValidator.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/AbstractJobMetadataValidator.java deleted file mode 100644 index e83d5c9..0000000 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/AbstractJobMetadataValidator.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.orchestrator.core.validator.impl; - -import org.apache.airavata.orchestrator.core.validator.JobMetadataValidator; - -public abstract class AbstractJobMetadataValidator implements JobMetadataValidator { - public boolean runBasicValidation(String experimentID) { - //todo implement basic validation which is not an application specific - return false; - } -} http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java index 53ecfe8..a756a0a 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java @@ -21,13 +21,16 @@ package org.apache.airavata.orchestrator.core.validator.impl; import org.apache.airavata.model.workspace.experiment.Experiment; +import org.apache.airavata.model.workspace.experiment.TaskDetails; +import org.apache.airavata.model.workspace.experiment.WorkflowNodeDetails; import org.apache.airavata.orchestrator.core.exception.OrchestratorException; +import org.apache.airavata.orchestrator.core.validator.JobMetadataValidator; import org.apache.airavata.persistance.registry.jpa.impl.RegistryFactory; import org.apache.airavata.registry.cpi.Registry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class SimpleAppDataValidator extends AbstractJobMetadataValidator { +public class SimpleAppDataValidator implements JobMetadataValidator { private final static Logger logger = LoggerFactory.getLogger(SimpleAppDataValidator.class); private Registry registry; @@ -36,38 +39,11 @@ public class SimpleAppDataValidator extends AbstractJobMetadataValidator { this.registry = RegistryFactory.getDefaultRegistry(); } - public boolean runAppSpecificValidation(String experimentID) throws OrchestratorException{ - // implement simple application specific validator to be used for - // all the applications. - return true; //To change body of implemented methods use File | Settings | File Templates. - } - - public boolean validate(String experimentID) throws OrchestratorException { + public boolean validate(Experiment experiment, WorkflowNodeDetails workflowNodeDetail, TaskDetails taskID) throws OrchestratorException { boolean result = false; - if (super.runBasicValidation(experimentID)) { - - Experiment experiment = null; - try { - experiment = (Experiment) registry.get(org.apache.airavata.registry.cpi.RegistryModelType.EXPERIMENT, experimentID); - } catch (Exception e) { - throw new OrchestratorException(e); - } - if (experiment.getUserConfigurationData().isAiravataAutoSchedule()) { - logger.error("We dont' support auto scheduling at this point, We will simply use user data as it is"); - } - - /* todo like this do more validation and if they are suppose to fail return false otherwise give some - log messages in server side logs - */ - if (runAppSpecificValidation(experimentID)) { - return true; - } - String error = "Application data validation steps failed"; - logger.error(error); - return false; + if (experiment.getUserConfigurationData().isAiravataAutoSchedule()) { + logger.error("We dont' support auto scheduling at this point, We will simply use user data as it is"); } - String error = "Basic validation steps failed"; - logger.error(error); - return false; + return true; } } http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java index d02155e..3a47585 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java @@ -34,6 +34,17 @@ import java.util.List; public interface Orchestrator { /** + * This method can be used to run all custom validators plugged in to the orchestrator and make + * sure the experiment is ready to launch and if its not this will return false + * @param experiment + * @param workflowNodeDetail + * @param taskID + * @return boolean if the experiments are valids after executing all the validators return true otherwise it iwll return false + * @throws OrchestratorException + */ + boolean validateExperiment(Experiment experiment, WorkflowNodeDetails workflowNodeDetail, + TaskDetails taskID) throws OrchestratorException; + /** * After creating the experiment Data user have the * experimentID as the handler to the experiment, during the launchExperiment * We just have to give the experimentID http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java index afd7425..08df639 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java @@ -49,9 +49,6 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{ // this is going to be null unless the thread count is 0 private JobSubmitter jobSubmitter = null; - private JobMetadataValidator jobMetadataValidator = null; - - public SimpleOrchestratorImpl() throws OrchestratorException { try { @@ -60,14 +57,7 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{ Class<? extends JobSubmitter> aClass = Class.forName(submitterClass.trim()).asSubclass(JobSubmitter.class); jobSubmitter = aClass.newInstance(); jobSubmitter.initialize(this.orchestratorContext); - String validatorClzz = this.orchestratorContext.getOrchestratorConfiguration().getValidatorClass(); - if (this.orchestratorConfiguration.isEnableValidation()) { - if (validatorClzz == null) { - logger.error("Job validation class is not properly set, so Validation will be turned off !"); - } - Class<? extends JobMetadataValidator> vClass = Class.forName(validatorClzz.trim()).asSubclass(JobMetadataValidator.class); - jobMetadataValidator = vClass.newInstance(); - } + } catch (Exception e) { String error = "Error creating JobSubmitter in non threaded mode "; logger.error(error); @@ -123,7 +113,35 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{ return tasks; } - public void cancelExperiment(String experimentID) + public boolean validateExperiment(Experiment experiment, WorkflowNodeDetails workflowNodeDetail, TaskDetails taskID) throws OrchestratorException { + if (this.orchestratorConfiguration.isEnableValidation()) { + List<String> validatorClzzez = this.orchestratorContext.getOrchestratorConfiguration().getValidatorClasses(); + for (String validator : validatorClzzez) { + try { + Class<? extends JobMetadataValidator> vClass = Class.forName(validator.trim()).asSubclass(JobMetadataValidator.class); + JobMetadataValidator jobMetadataValidator = vClass.newInstance(); + if(jobMetadataValidator.validate(experiment, workflowNodeDetail, taskID)){ + logger.info("Validation of " + validator + " is SUCCESSFUL"); + }else { + logger.error("Validation of " + validator + " is FAILED"); + return false; + } + } catch (ClassNotFoundException e) { + logger.error("Error loading the validation class: ", validator, e); + return false; + } catch (InstantiationException e) { + logger.error("Error loading the validation class: ", validator, e); + return false; + } catch (IllegalAccessException e) { + logger.error("Error loading the validation class: ", validator, e); + return false; + } + } + } + return true; + } + + public void cancelExperiment(String experimentID) throws OrchestratorException { throw new OrchestratorException(new OperationNotSupportedException()); } @@ -137,14 +155,6 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{ this.executor = executor; } - public JobMetadataValidator getJobMetadataValidator() { - return jobMetadataValidator; - } - - public void setJobMetadataValidator(JobMetadataValidator jobMetadataValidator) { - this.jobMetadataValidator = jobMetadataValidator; - } - public JobSubmitter getJobSubmitter() { return jobSubmitter; } http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-thrift-descriptions/orchestrator.cpi.service.thrift ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-thrift-descriptions/orchestrator.cpi.service.thrift b/modules/orchestrator/orchestrator-thrift-descriptions/orchestrator.cpi.service.thrift index c03e81f..6dc452e 100644 --- a/modules/orchestrator/orchestrator-thrift-descriptions/orchestrator.cpi.service.thrift +++ b/modules/orchestrator/orchestrator-thrift-descriptions/orchestrator.cpi.service.thrift @@ -43,5 +43,7 @@ service OrchestratorService { **/ bool launchExperiment (1: required string experimentId), - bool terminateExperiment (1: required string experimentId) + bool terminateExperiment (1: required string experimentId), + + bool validateExperiment(1: required string experimentId) } \ No newline at end of file
