committing intital gfac app catalog integration
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/8abe8dca Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/8abe8dca Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/8abe8dca Branch: refs/heads/gfac_appcatalog_int Commit: 8abe8dca55f49eeac0ed1416b4565a767922b7a0 Parents: 91f5de5 Author: chathuriw <[email protected]> Authored: Tue Oct 28 16:23:38 2014 -0400 Committer: Chathuri Wimalasena <[email protected]> Committed: Wed Nov 5 11:16:14 2014 -0500 ---------------------------------------------------------------------- .../client/samples/CreateLaunchExperiment.java | 11 +- .../org/apache/airavata/gfac/Scheduler.java | 5 +- .../gfac/core/context/ApplicationContext.java | 44 +-- .../gfac/core/context/JobExecutionContext.java | 47 +++ .../airavata/gfac/core/cpi/BetterGfacImpl.java | 292 +++---------------- .../core/handler/AppDescriptorCheckHandler.java | 61 ++-- .../gfac/core/provider/utils/ProviderUtils.java | 18 +- .../airavata/gfac/core/utils/GFacUtils.java | 16 + 8 files changed, 160 insertions(+), 334 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/8abe8dca/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java index 2845bc6..a96cba7 100644 --- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java +++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java @@ -53,7 +53,7 @@ public class CreateLaunchExperiment { private static final String DEFAULT_GATEWAY = "default.registry.gateway"; private static Airavata.Client airavataClient; - private static String echoAppId = "Echo_6281480a-9887-4a0f-8311-59bbaf738e54"; + private static String echoAppId = "Echo_b6782be4-315b-4cbd-9403-aa7ce564548a"; private static String wrfAppId = "WRF_5f097c9c-7066-49ec-aed7-4e39607b3adc"; private static String amberAppId = "Amber_89906be6-5678-49a6-9d04-a0604fbdef2e"; @@ -70,7 +70,7 @@ public class CreateLaunchExperiment { public static void main(String[] args) throws Exception { airavataClient = AiravataClientFactory.createAiravataClient(THRIFT_SERVER_HOST, THRIFT_SERVER_PORT); System.out.println("API version is " + airavataClient.getAPIVersion()); - registerApplications(); // run this only the first time +// registerApplications(); // run this only the first time createAndLaunchExp(); } @@ -79,12 +79,13 @@ public class CreateLaunchExperiment { public static void createAndLaunchExp() throws TException { - final String expId = createEchoExperimentForFSD(airavataClient); +// final String expId = createEchoExperimentForFSD(airavataClient); try { - for (int i = 0; i < 2; i++) { + for (int i = 0; i < 1; i++) { // final String expId = createExperimentForSSHHost(airavata); // final String expId = createEchoExperimentForFSD(airavataClient); // final String expId = createEchoExperimentForStampede(airavataClient); + final String expId = createEchoExperimentForTrestles(airavataClient); // final String expId = createExperimentEchoForLocalHost(airavataClient); // final String expId = createExperimentWRFTrestles(airavataClient); // final String expId = createExperimentForBR2(airavataClient); @@ -93,7 +94,7 @@ public class CreateLaunchExperiment { // final String expId = createExperimentForStampedeAmber(airavataClient); // final String expId = createExperimentForTrestlesAmber(airavataClient); -// System.out.println("Experiment ID : " + expId); + System.out.println("Experiment ID : " + expId); // updateExperiment(airavata, expId); launchExperiment(airavataClient, expId); } http://git-wip-us.apache.org/repos/asf/airavata/blob/8abe8dca/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/Scheduler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/Scheduler.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/Scheduler.java index 1b8efe0..9b70fae 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/Scheduler.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/Scheduler.java @@ -39,6 +39,7 @@ import org.apache.airavata.gfac.core.context.JobExecutionContext; import org.apache.airavata.gfac.core.provider.GFacProvider; import org.apache.airavata.gfac.core.provider.GFacProviderConfig; import org.apache.airavata.gfac.core.provider.GFacProviderException; +import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; @@ -75,7 +76,7 @@ public class Scheduler { * @return GFacProvider instance. */ private static GFacProvider getProvider(JobExecutionContext jobExecutionContext) throws GFacException { - HostDescription hostDescription = jobExecutionContext.getApplicationContext().getHostDescription(); + ComputeResourceDescription hostDescription = jobExecutionContext.getApplicationContext().getComputeResourceDescription(); String applicationName = jobExecutionContext.getServiceName(); URL resource = Scheduler.class.getClassLoader().getResource(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML); @@ -111,6 +112,8 @@ public class Scheduler { } // We give higher preference to applications specific provider if configured if (provider == null) { + + jobExecutionContext.getApplicationContext().getComputeResourcePreference().getPreferredJobSubmissionProtocol() String hostClass = hostDescription.getType().getClass().getName(); providerClassName = GFacConfiguration.getAttributeValue(GFacConfiguration.getHandlerDoc(), Constants.XPATH_EXPR_PROVIDER_ON_HOST + hostClass + "']", Constants.GFAC_CONFIG_CLASS_ATTRIBUTE); Class<? extends GFacProvider> aClass1 = Class.forName(providerClassName).asSubclass(GFacProvider.class); http://git-wip-us.apache.org/repos/asf/airavata/blob/8abe8dca/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ApplicationContext.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ApplicationContext.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ApplicationContext.java index 4083f29..29197be 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ApplicationContext.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ApplicationContext.java @@ -21,37 +21,47 @@ package org.apache.airavata.gfac.core.context; -import org.apache.airavata.commons.gfac.type.ApplicationDescription; -import org.apache.airavata.commons.gfac.type.HostDescription; -import org.apache.airavata.commons.gfac.type.ServiceDescription; +import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription; +import org.apache.airavata.model.appcatalog.appinterface.ApplicationInterfaceDescription; +import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription; +import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference; +import org.apache.airavata.model.appcatalog.gatewayprofile.GatewayResourceProfile; public class ApplicationContext extends AbstractContext { + private ApplicationDeploymentDescription applicationDeploymentDescription; + private ComputeResourceDescription computeResourceDescription; + private ApplicationInterfaceDescription applicationInterfaceDescription; + private ComputeResourcePreference computeResourcePreference; - private ApplicationDescription applicationDeploymentDescription; - private ServiceDescription serviceDescription; - private HostDescription hostDescription; - - public ApplicationDescription getApplicationDeploymentDescription() { + public ApplicationDeploymentDescription getApplicationDeploymentDescription() { return applicationDeploymentDescription; } - public <T extends ApplicationDescription> void setApplicationDeploymentDescription(T applicationDeploymentDescription) { + public void setApplicationDeploymentDescription(ApplicationDeploymentDescription applicationDeploymentDescription) { this.applicationDeploymentDescription = applicationDeploymentDescription; } - public <T extends ServiceDescription> void setServiceDescription(T serviceDescription) { - this.serviceDescription = serviceDescription; + public ComputeResourceDescription getComputeResourceDescription() { + return computeResourceDescription; + } + + public void setComputeResourceDescription(ComputeResourceDescription computeResourceDescription) { + this.computeResourceDescription = computeResourceDescription; + } + + public ApplicationInterfaceDescription getApplicationInterfaceDescription() { + return applicationInterfaceDescription; } - public <T extends HostDescription> void setHostDescription(T hostDescription) { - this.hostDescription = hostDescription; + public void setApplicationInterfaceDescription(ApplicationInterfaceDescription applicationInterfaceDescription) { + this.applicationInterfaceDescription = applicationInterfaceDescription; } - public ServiceDescription getServiceDescription() { - return serviceDescription; + public ComputeResourcePreference getComputeResourcePreference() { + return computeResourcePreference; } - public HostDescription getHostDescription() { - return hostDescription; + public void setComputeResourcePreference(ComputeResourcePreference computeResourcePreference) { + this.computeResourcePreference = computeResourcePreference; } } http://git-wip-us.apache.org/repos/asf/airavata/blob/8abe8dca/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/JobExecutionContext.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/JobExecutionContext.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/JobExecutionContext.java index 2f94ec5..da716c5 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/JobExecutionContext.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/JobExecutionContext.java @@ -66,6 +66,13 @@ public class JobExecutionContext extends AbstractContext implements Serializable private String credentialStoreToken; + private String workingDir; + + private String inputDir; + private String outputDir; + private String standaredOutput; + private String standaredError; + // private ContextHeaderDocument.ContextHeader contextHeader; // Keep track of the current path of the message. Before hitting provider its in-path. @@ -317,4 +324,44 @@ public class JobExecutionContext extends AbstractContext implements Serializable public void setCredentialStoreToken(String credentialStoreToken) { this.credentialStoreToken = credentialStoreToken; } + + public String getWorkingDir() { + return workingDir; + } + + public void setWorkingDir(String workingDir) { + this.workingDir = workingDir; + } + + public String getInputDir() { + return inputDir; + } + + public void setInputDir(String inputDir) { + this.inputDir = inputDir; + } + + public String getOutputDir() { + return outputDir; + } + + public void setOutputDir(String outputDir) { + this.outputDir = outputDir; + } + + public String getStandaredOutput() { + return standaredOutput; + } + + public void setStandaredOutput(String standaredOutput) { + this.standaredOutput = standaredOutput; + } + + public String getStandaredError() { + return standaredError; + } + + public void setStandaredError(String standaredError) { + this.standaredError = standaredError; + } } http://git-wip-us.apache.org/repos/asf/airavata/blob/8abe8dca/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java index ca7620d..16c49e6 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java @@ -19,14 +19,7 @@ * */ package org.apache.airavata.gfac.core.cpi; -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.xpath.XPathExpressionException; + import org.airavata.appcatalog.cpi.AppCatalog; import org.apache.aiaravata.application.catalog.data.impl.AppCatalogFactory; import org.apache.airavata.common.exception.AiravataException; @@ -35,9 +28,6 @@ import org.apache.airavata.common.utils.AiravataZKUtils; import org.apache.airavata.common.utils.MonitorPublisher; import org.apache.airavata.common.utils.ServerSettings; import org.apache.airavata.common.utils.listener.AbstractActivityListener; -import org.apache.airavata.commons.gfac.type.ApplicationDescription; -import org.apache.airavata.commons.gfac.type.HostDescription; -import org.apache.airavata.commons.gfac.type.ServiceDescription; import org.apache.airavata.gfac.Constants; import org.apache.airavata.gfac.GFacConfiguration; import org.apache.airavata.gfac.GFacException; @@ -57,45 +47,16 @@ import org.apache.airavata.gfac.core.provider.GFacRecoverableProvider; import org.apache.airavata.gfac.core.states.GfacExperimentState; import org.apache.airavata.gfac.core.states.GfacPluginState; import org.apache.airavata.gfac.core.utils.GFacUtils; - import org.apache.airavata.messaging.core.Publisher; - import org.apache.airavata.messaging.core.PublisherFactory; import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription; import org.apache.airavata.model.appcatalog.appinterface.ApplicationInterfaceDescription; -import org.apache.airavata.model.appcatalog.appinterface.InputDataObjectType; -import org.apache.airavata.model.appcatalog.appinterface.OutputDataObjectType; - import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription; -import org.apache.airavata.model.appcatalog.computeresource.JobManagerCommand; -import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionInterface; -import org.apache.airavata.model.appcatalog.computeresource.LOCALSubmission; -import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManager; -import org.apache.airavata.model.appcatalog.computeresource.SSHJobSubmission; -import org.apache.airavata.model.appcatalog.computeresource.UnicoreJobSubmission; - -import org.apache.airavata.model.appcatalog.computeresource.*; - import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference; import org.apache.airavata.model.messaging.event.*; import org.apache.airavata.model.workspace.experiment.*; import org.apache.airavata.registry.cpi.Registry; import org.apache.airavata.registry.cpi.RegistryModelType; -import org.apache.airavata.schemas.gfac.*; -import org.apache.airavata.schemas.gfac.DataType; - -import org.apache.airavata.schemas.gfac.GsisshHostType; -import org.apache.airavata.schemas.gfac.HostDescriptionType; -import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType; -import org.apache.airavata.schemas.gfac.InputParameterType; -import org.apache.airavata.schemas.gfac.JobTypeType; -import org.apache.airavata.schemas.gfac.OutputParameterType; -import org.apache.airavata.schemas.gfac.ParameterType; -import org.apache.airavata.schemas.gfac.ProjectAccountType; -import org.apache.airavata.schemas.gfac.QueueType; -import org.apache.airavata.schemas.gfac.SSHHostType; -import org.apache.airavata.schemas.gfac.ServiceDescriptionType; -import org.apache.airavata.schemas.gfac.UnicoreHostType; import org.apache.zookeeper.*; import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; @@ -111,8 +72,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Properties; -//import org.apache.airavata.api.server.listener.ExperimentStatusChangedEvent; - /** * This is the GFac CPI class for external usage, this simply have a single method to submit a job to * the resource, required data for the job has to be stored in registry prior to invoke this object. @@ -123,13 +82,8 @@ public class BetterGfacImpl implements GFac,Watcher { private Registry registry; -// private AiravataAPI airavataAPI; - -// private AiravataRegistry2 airavataRegistry2; - - private ZooKeeper zk; // we are not storing zk instance in to jobExecution context - - private static Integer mutex = new Integer(-1); + // we are not storing zk instance in to jobExecution context + private ZooKeeper zk; private static List<ThreadedHandler> daemonHandlers = new ArrayList<ThreadedHandler>(); @@ -150,8 +104,6 @@ public class BetterGfacImpl implements GFac,Watcher { public BetterGfacImpl(Registry registry, ZooKeeper zooKeeper, MonitorPublisher publisher) { this.registry = registry; -// this.airavataAPI = airavataAPI; -// this.airavataRegistry2 = airavataRegistry2; monitorPublisher = publisher; // This is a EventBus common for gfac this.zk = zooKeeper; } @@ -186,10 +138,20 @@ public class BetterGfacImpl implements GFac,Watcher { public static void startDaemonHandlers() { List<GFacHandlerConfig> daemonHandlerConfig = null; - URL resource = BetterGfacImpl.class.getClassLoader().getResource(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML); - gfacConfigFile = new File(resource.getPath()); + String className = null; try { + URL resource = BetterGfacImpl.class.getClassLoader().getResource(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML); + if (resource != null) { + gfacConfigFile = new File(resource.getPath()); + } daemonHandlerConfig = GFacConfiguration.getDaemonHandlers(gfacConfigFile); + for (GFacHandlerConfig handlerConfig : daemonHandlerConfig) { + className = handlerConfig.getClassName(); + Class<?> aClass = Class.forName(className).asSubclass(ThreadedHandler.class); + ThreadedHandler threadedHandler = (ThreadedHandler) aClass.newInstance(); + threadedHandler.initProperties(handlerConfig.getProperties()); + daemonHandlers.add(threadedHandler); + } } catch (ParserConfigurationException e) { log.error("Error parsing gfac-config.xml, double check the xml configuration", e); } catch (IOException e) { @@ -198,29 +160,18 @@ public class BetterGfacImpl implements GFac,Watcher { log.error("Error parsing gfac-config.xml, double check the xml configuration", e); } catch (XPathExpressionException e) { log.error("Error parsing gfac-config.xml, double check the xml configuration", e); - } - - for (GFacHandlerConfig handlerConfig : daemonHandlerConfig) { - String className = handlerConfig.getClassName(); - try { - Class<?> aClass = Class.forName(className).asSubclass(ThreadedHandler.class); - ThreadedHandler threadedHandler = (ThreadedHandler) aClass.newInstance(); - threadedHandler.initProperties(handlerConfig.getProperties()); - daemonHandlers.add(threadedHandler); - } catch (ClassNotFoundException e) { - log.error("Error initializing the handler: " + className); - log.error(className + " class has to implement " + ThreadedHandler.class); - } catch (InstantiationException e) { - log.error("Error initializing the handler: " + className); - log.error(className + " class has to implement " + ThreadedHandler.class); - } catch (IllegalAccessException e) { - log.error("Error initializing the handler: " + className); - log.error(className + " class has to implement " + ThreadedHandler.class); - } catch (GFacHandlerException e) { - log.error("Error initializing the handler " + className); - } catch (GFacException e) { - e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. - } + } catch (ClassNotFoundException e) { + log.error("Error initializing the handler: " + className); + log.error(className + " class has to implement " + ThreadedHandler.class); + } catch (InstantiationException e) { + log.error("Error initializing the handler: " + className); + log.error(className + " class has to implement " + ThreadedHandler.class); + } catch (GFacHandlerException e) { + log.error("Error initializing the handler: " + className); + log.error(className + " class has to implement " + ThreadedHandler.class); + } catch (IllegalAccessException e) { + log.error("Error initializing the handler: " + className); + log.error(className + " class has to implement " + ThreadedHandler.class); } for (ThreadedHandler tHandler : daemonHandlers) { (new Thread(tHandler)).start(); @@ -306,173 +257,6 @@ public class BetterGfacImpl implements GFac,Watcher { } } } - //Create the legacy schema docs to fill-in - ServiceDescription legacyServiceDescription = new ServiceDescription(); - ServiceDescriptionType legacyServiceDescType = legacyServiceDescription.getType(); - ApplicationDescription legacyAppDescription = null; - HostDescription legacyHostDescription = null; - - ///////////////SERVICE DESCRIPTOR/////////////////////////////// - //Fetch the application inputs and outputs from the app interface and create the legacy service description. - legacyServiceDescType.setName(applicationInterface.getApplicationName()); - legacyServiceDescType.setDescription(applicationInterface.getApplicationName()); - List<InputParameterType> legacyInputParameters = new ArrayList<InputParameterType>(); - List<OutputParameterType> legacyOutputParameters = new ArrayList<OutputParameterType>(); - List<InputDataObjectType> applicationInputs = applicationInterface.getApplicationInputs(); - for (InputDataObjectType dataObjectType : applicationInputs) { - InputParameterType parameter = InputParameterType.Factory.newInstance(); - parameter.setParameterName(dataObjectType.getName()); - parameter.setParameterDescription(dataObjectType.getUserFriendlyDescription()); - ParameterType parameterType = parameter.addNewParameterType(); - switch (dataObjectType.getType()) { - case FLOAT: - parameterType.setType(DataType.FLOAT); - break; - case INTEGER: - parameterType.setType(DataType.INTEGER); - break; - case STRING: - parameterType.setType(DataType.STRING); - break; - case URI: - parameterType.setType(DataType.URI); - break; - } - parameterType.setName(parameterType.getType().toString()); - parameter.addParameterValue(dataObjectType.getValue()); - legacyInputParameters.add(parameter); - } - - List<OutputDataObjectType> applicationOutputs = applicationInterface.getApplicationOutputs(); - for (OutputDataObjectType dataObjectType : applicationOutputs) { - OutputParameterType parameter = OutputParameterType.Factory.newInstance(); - parameter.setParameterName(dataObjectType.getName()); - parameter.setParameterDescription(dataObjectType.getName()); - ParameterType parameterType = parameter.addNewParameterType(); - switch (dataObjectType.getType()) { - case FLOAT: - parameterType.setType(DataType.FLOAT); - break; - case INTEGER: - parameterType.setType(DataType.INTEGER); - break; - case STRING: - parameterType.setType(DataType.STRING); - break; - case URI: - parameterType.setType(DataType.URI); - break; - } - parameterType.setName(parameterType.getType().toString()); - legacyOutputParameters.add(parameter); - } - - legacyServiceDescType.setInputParametersArray(legacyInputParameters.toArray(new InputParameterType[]{})); - legacyServiceDescType.setOutputParametersArray(legacyOutputParameters.toArray(new OutputParameterType[]{})); - - ////////////////////----------- HOST DESCRIPTOR -----------------////////////////////// - //Fetch the host description details and fill-in legacy doc - ResourceJobManager resourceJobManager = null; - for (JobSubmissionInterface jobSubmissionInterface : computeResource.getJobSubmissionInterfaces()) { - switch (jobSubmissionInterface.getJobSubmissionProtocol()) { - case LOCAL: - legacyHostDescription = new HostDescription(); - LOCALSubmission localSubmission = - appCatalog.getComputeResource().getLocalJobSubmission(jobSubmissionInterface.getJobSubmissionInterfaceId()); - resourceJobManager = localSubmission.getResourceJobManager(); - break; - case SSH: - SSHJobSubmission sshJobSubmission = - appCatalog.getComputeResource().getSSHJobSubmission(jobSubmissionInterface.getJobSubmissionInterfaceId()); - resourceJobManager = sshJobSubmission.getResourceJobManager(); - switch (sshJobSubmission.getSecurityProtocol()) { - case GSI: - legacyHostDescription = new HostDescription(GsisshHostType.type); - ((GsisshHostType) legacyHostDescription.getType()).setJobManager - (resourceJobManager.getResourceJobManagerType().name()); - ((GsisshHostType) legacyHostDescription.getType()).setInstalledPath(resourceJobManager.getJobManagerBinPath()); - // applicationDescription.setInstalledParentPath(resourceJobManager.getJobManagerBinPath()); - ((GsisshHostType) legacyHostDescription.getType()).setPort(sshJobSubmission.getSshPort()); - break; - case SSH_KEYS: - legacyHostDescription = new HostDescription(SSHHostType.type); - ((SSHHostType) legacyHostDescription.getType()).setHpcResource(true); - break; - default: - legacyHostDescription = new HostDescription(SSHHostType.type); - ((SSHHostType) legacyHostDescription.getType()).setHpcResource(true); - break; - } - break; - case UNICORE: - UnicoreJobSubmission ucrSubmission = appCatalog.getComputeResource().getUNICOREJobSubmission(jobSubmissionInterface.getJobSubmissionInterfaceId()); - String unicoreEndpoint = ucrSubmission.getUnicoreEndPointURL(); - legacyHostDescription = new HostDescription(UnicoreHostType.type); - ((UnicoreHostType) legacyHostDescription.getType()).setUnicoreBESEndPointArray(new String[]{unicoreEndpoint}); - break; - default: - break; - } - } - HostDescriptionType legacyHostDescType = legacyHostDescription.getType(); - legacyHostDescType.setHostName(computeResource.getHostName()); - String ipAddress = computeResource.getHostName(); - if (computeResource.getIpAddresses() != null && computeResource.getIpAddresses().size() > 0) { - ipAddress = computeResource.getIpAddresses().iterator().next(); - } else if (computeResource.getHostAliases() != null && computeResource.getHostAliases().size() > 0) { - ipAddress = computeResource.getHostAliases().iterator().next(); - } - legacyHostDescType.setHostAddress(ipAddress); - - /////////////////////---------------- APPLICATION DESCRIPTOR ---------------------///////////////////////// - //Fetch deployment information and fill-in legacy doc - if ((legacyHostDescType instanceof GsisshHostType) - || (legacyHostDescType instanceof SSHHostType) - || (legacyHostDescType instanceof UnicoreHostType)) { - legacyAppDescription = new ApplicationDescription(HpcApplicationDeploymentType.type); - HpcApplicationDeploymentType legacyHPCAppDescType = (HpcApplicationDeploymentType) legacyAppDescription.getType(); - switch (applicationDeployment.getParallelism()) { - case SERIAL: - legacyHPCAppDescType.setJobType(JobTypeType.SERIAL); - break; - case MPI: - legacyHPCAppDescType.setJobType(JobTypeType.MPI); - break; - case OPENMP: - legacyHPCAppDescType.setJobType(JobTypeType.OPEN_MP); - break; - default: - break; - } - //Fetch scheduling information from experiment request - ComputationalResourceScheduling taskSchedule = taskData.getTaskScheduling(); - QueueType queueType = legacyHPCAppDescType.addNewQueue(); - queueType.setQueueName(taskSchedule.getQueueName()); - legacyHPCAppDescType.setCpuCount(taskSchedule.getTotalCPUCount()); - legacyHPCAppDescType.setNodeCount(taskSchedule.getNodeCount()); - legacyHPCAppDescType.setMaxWallTime(taskSchedule.getWallTimeLimit()); - if (resourceJobManager != null) { - legacyHPCAppDescType.setInstalledParentPath(resourceJobManager.getJobManagerBinPath()); - if (resourceJobManager.getJobManagerCommands() != null) { - legacyHPCAppDescType.setJobSubmitterCommand(resourceJobManager.getJobManagerCommands().get(JobManagerCommand.SUBMISSION)); - } - } - ProjectAccountType projectAccountType = legacyHPCAppDescType.addNewProjectAccount(); - if (gatewayResourcePreferences != null) { - projectAccountType.setProjectAccountNumber(gatewayResourcePreferences.getAllocationProjectNumber()); - } - } else { - legacyAppDescription = new ApplicationDescription(); - } - ApplicationDeploymentDescriptionType legacyAppDescType = legacyAppDescription.getType(); - legacyAppDescType.addNewApplicationName().setStringValue(applicationInterface.getApplicationName().replaceAll(" ", "_")); - legacyAppDescType.setExecutableLocation(applicationDeployment.getExecutablePath()); - if (gatewayResourcePreferences != null) { - legacyAppDescType.setScratchWorkingDirectory(gatewayResourcePreferences.getScratchLocation()); - } else { - legacyAppDescType.setScratchWorkingDirectory("/tmp"); - log.warn("Missing gateway resource profile for gateway id '" + gatewayID + "'."); - } URL resource = BetterGfacImpl.class.getClassLoader().getResource(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML); Properties configurationProperties = ServerSettings.getProperties(); @@ -498,19 +282,17 @@ public class BetterGfacImpl implements GFac,Watcher { jobExecutionContext.setRegistry(registry); ApplicationContext applicationContext = new ApplicationContext(); -// applicationContext.setApplicationDeploymentDescription(applicationDescription); - applicationContext.setHostDescription(legacyHostDescription); - applicationContext.setServiceDescription(legacyServiceDescription); - applicationContext.setApplicationDeploymentDescription(legacyAppDescription); + applicationContext.setComputeResourceDescription(computeResource); + applicationContext.setApplicationDeploymentDescription(applicationDeployment); + applicationContext.setApplicationInterfaceDescription(applicationInterface); + applicationContext.setComputeResourcePreference(gatewayResourcePreferences); jobExecutionContext.setApplicationContext(applicationContext); List<DataObjectType> experimentInputs = taskData.getApplicationInputs(); - jobExecutionContext.setInMessageContext(new MessageContext(GFacUtils.getInMessageContext(experimentInputs, - legacyServiceDescType.getInputParametersArray()))); + jobExecutionContext.setInMessageContext(new MessageContext(GFacUtils.getInMessageContext(experimentInputs))); List<DataObjectType> outputData = taskData.getApplicationOutputs(); - jobExecutionContext.setOutMessageContext(new MessageContext(GFacUtils.getOutMessageContext(outputData, - legacyServiceDescType.getOutputParametersArray()))); + jobExecutionContext.setOutMessageContext(new MessageContext(GFacUtils.getOutMessageContext(outputData))); jobExecutionContext.setProperty(Constants.PROP_TOPIC, experimentID); jobExecutionContext.setGfac(this); @@ -1178,14 +960,6 @@ public class BetterGfacImpl implements GFac,Watcher { BetterGfacImpl.monitorPublisher = monitorPublisher; } -// public AiravataAPI getAiravataAPI() { -// return airavataAPI; -// } - -// public AiravataRegistry2 getAiravataRegistry2() { -// return airavataRegistry2; -// } - public static List<ThreadedHandler> getDaemonHandlers() { return daemonHandlers; } http://git-wip-us.apache.org/repos/asf/airavata/blob/8abe8dca/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/handler/AppDescriptorCheckHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/handler/AppDescriptorCheckHandler.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/handler/AppDescriptorCheckHandler.java index 33c32d3..676a15a 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/handler/AppDescriptorCheckHandler.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/handler/AppDescriptorCheckHandler.java @@ -20,12 +20,12 @@ */ package org.apache.airavata.gfac.core.handler; -import org.apache.airavata.commons.gfac.type.ApplicationDescription; import org.apache.airavata.gfac.Constants; import org.apache.airavata.gfac.core.context.JobExecutionContext; import org.apache.airavata.gfac.core.states.GfacPluginState; import org.apache.airavata.gfac.core.utils.GFacUtils; -import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType; +import org.apache.airavata.model.appcatalog.appinterface.ApplicationInterfaceDescription; +import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,47 +43,34 @@ public class AppDescriptorCheckHandler implements GFacRecoverableHandler { logger.info("Error saving plugin status to ZK"); } StringBuffer data = new StringBuffer(); - ApplicationDescription app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription(); - ApplicationDeploymentDescriptionType appDesc = app.getType(); + ApplicationInterfaceDescription appInterface = jobExecutionContext.getApplicationContext().getApplicationInterfaceDescription(); + ComputeResourcePreference computeResourcePreference = jobExecutionContext.getApplicationContext().getComputeResourcePreference(); - if (appDesc.getScratchWorkingDirectory() == null) { - appDesc.setScratchWorkingDirectory("/tmp"); + if (computeResourcePreference.getScratchLocation() == null) { + computeResourcePreference.setScratchLocation("/tmp"); } /* * Working dir */ - if (appDesc.getStaticWorkingDirectory() == null || "null".equals(appDesc.getStaticWorkingDirectory())) { - String tmpDir = appDesc.getScratchWorkingDirectory() + File.separator - + jobExecutionContext.getExperimentID(); - appDesc.setStaticWorkingDirectory(tmpDir); - } - data.append(appDesc.getScratchWorkingDirectory()); - data.append(",").append(appDesc.getStaticWorkingDirectory()); - //FIXME: Move this input/output to application descrpitor + String workingDir = computeResourcePreference.getScratchLocation() + File.separator+ jobExecutionContext.getExperimentID(); + jobExecutionContext.setWorkingDir(workingDir); + data.append(computeResourcePreference.getScratchLocation()); + data.append(",").append(jobExecutionContext.getWorkingDir()); + /* * Input and Output Directory */ - if (appDesc.getInputDataDirectory() == null || "".equals(appDesc.getInputDataDirectory())) { - appDesc.setInputDataDirectory(appDesc.getStaticWorkingDirectory() + File.separator + Constants.INPUT_DATA_DIR_VAR_NAME); - } - if (appDesc.getOutputDataDirectory() == null || "".equals(appDesc.getOutputDataDirectory())) { - appDesc.setOutputDataDirectory(appDesc.getStaticWorkingDirectory() + File.separator + Constants.OUTPUT_DATA_DIR_VAR_NAME); - } + jobExecutionContext.setInputDir(workingDir + File.separator + Constants.INPUT_DATA_DIR_VAR_NAME ); + jobExecutionContext.setOutputDir(workingDir + File.separator + Constants.OUTPUT_DATA_DIR_VAR_NAME); + data.append(",").append(jobExecutionContext.getInputDir()).append(",").append(jobExecutionContext.getOutputDir()); - data.append(",").append(appDesc.getInputDataDirectory()).append(",").append(appDesc.getOutputDataDirectory()); /* * Stdout and Stderr for Shell */ - if (appDesc.getStandardOutput() == null || "".equals(appDesc.getStandardOutput())) { - appDesc.setStandardOutput(appDesc.getStaticWorkingDirectory() + File.separator - + appDesc.getApplicationName().getStringValue().replaceAll("\\s+","") + ".stdout"); - } - if (appDesc.getStandardError() == null || "".equals(appDesc.getStandardError())) { - appDesc.setStandardError(appDesc.getStaticWorkingDirectory() + File.separator - + appDesc.getApplicationName().getStringValue().replaceAll("\\s+","") + ".stderr"); - } - data.append(",").append(appDesc.getStandardOutput()).append(",").append(appDesc.getStandardError()); + jobExecutionContext.setStandaredOutput(workingDir + File.separator + appInterface.getApplicationName().replaceAll("\\s+", "") + ".stdout"); + jobExecutionContext.setStandaredError(workingDir + File.separator + appInterface.getApplicationName().replaceAll("\\s+", "") + ".stderr"); + data.append(",").append(jobExecutionContext.getStandaredOutput()).append(",").append(jobExecutionContext.getStandaredError()); logger.info("Recoverable data is saving to zk: " + data.toString()); @@ -97,17 +84,15 @@ public class AppDescriptorCheckHandler implements GFacRecoverableHandler { } public void recover(JobExecutionContext jobExecutionContext) throws GFacHandlerException { - ApplicationDescription app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription(); - ApplicationDeploymentDescriptionType appDesc = app.getType(); try { String s = GFacUtils.getPluginData(jobExecutionContext, this.getClass().getName()); String[] split = s.split(","); // this is ugly code but nobody else is saving or reading this data, so this is the fastest way - appDesc.setScratchWorkingDirectory(split[0]); - appDesc.setStaticWorkingDirectory(split[1]); - appDesc.setInputDataDirectory(split[2]); - appDesc.setOutputDataDirectory(split[3]); - appDesc.setStandardOutput(split[4]); - appDesc.setStandardError(split[5]); + jobExecutionContext.getApplicationContext().getComputeResourcePreference().setScratchLocation(split[0]); + jobExecutionContext.setWorkingDir(split[1]); + jobExecutionContext.setInputDir(split[2]); + jobExecutionContext.setOutputDir(split[3]); + jobExecutionContext.setStandaredOutput(split[4]); + jobExecutionContext.setStandaredError(split[5]); } catch (Exception e) { throw new GFacHandlerException(e); } http://git-wip-us.apache.org/repos/asf/airavata/blob/8abe8dca/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/provider/utils/ProviderUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/provider/utils/ProviderUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/provider/utils/ProviderUtils.java index c98da92..dc8eb1c 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/provider/utils/ProviderUtils.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/provider/utils/ProviderUtils.java @@ -21,33 +21,23 @@ package org.apache.airavata.gfac.core.provider.utils; -import org.apache.airavata.commons.gfac.type.ActualParameter; -import org.apache.airavata.commons.gfac.type.MappingFactory; import org.apache.airavata.gfac.core.context.JobExecutionContext; import org.apache.airavata.gfac.core.context.MessageContext; import org.apache.airavata.gfac.core.provider.GFacProviderException; -import org.apache.airavata.schemas.gfac.InputParameterType; import java.util.ArrayList; import java.util.List; +import java.util.Map; public class ProviderUtils { public static List<String> getInputParameters(JobExecutionContext jobExecutionContext) throws GFacProviderException { List<String> parameters = new ArrayList<String>(); MessageContext inMessageContext = jobExecutionContext.getInMessageContext(); - InputParameterType[] inputParamDefinitionArray = jobExecutionContext.getApplicationContext(). - getServiceDescription().getType().getInputParametersArray(); - for (InputParameterType inputParam : inputParamDefinitionArray) { - String parameterName = inputParam.getParameterName(); - ActualParameter parameter = (ActualParameter)inMessageContext.getParameter(parameterName); - if(parameter == null){ - throw new GFacProviderException("Cannot find required input parameter " + parameterName + "."); - } - - parameters.add(MappingFactory.toString(parameter)); + Map<String, Object> inputs = inMessageContext.getParameters(); + for (String inputParam : inputs.keySet()) { + parameters.add(inputParam); } - return parameters; } http://git-wip-us.apache.org/repos/asf/airavata/blob/8abe8dca/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java index eef44a4..ce74e4e 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java @@ -819,6 +819,14 @@ public class GFacUtils { return stringObjectHashMap; } + public static Map<String, Object> getInMessageContext(List<DataObjectType> experimentData) throws GFacException { + Map<String, Object> map = new HashMap<String, Object>(); + for (DataObjectType objectType : experimentData) { + map.put(objectType.getKey(), objectType); + } + return map; + } + public static Map<String, Object> getOutMessageContext( List<DataObjectType> experimentData, Parameter[] parameters) throws GFacException { @@ -854,6 +862,14 @@ public class GFacUtils { return stringObjectHashMap; } + public static Map<String, Object> getOutMessageContext(List<DataObjectType> experimentData) throws GFacException { + Map<String, Object> map = new HashMap<String, Object>(); + for (DataObjectType objectType : experimentData) { + map.put(objectType.getKey(), objectType); + } + return map; + } + public static GfacExperimentState getZKExperimentState(ZooKeeper zk, JobExecutionContext jobExecutionContext) throws ApplicationSettingsException, KeeperException,
