Repository: airavata Updated Branches: refs/heads/master 9d80cf298 -> 565b5a3f4
adding lsf support with tests Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/5ff650f4 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/5ff650f4 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/5ff650f4 Branch: refs/heads/master Commit: 5ff650f4c61a9ee7bfcf6f15ea984d74d545ec32 Parents: e9468ca Author: Lahiru Gunathilake <[email protected]> Authored: Sat Mar 7 00:37:06 2015 -0500 Committer: Lahiru Gunathilake <[email protected]> Committed: Sat Mar 7 00:37:06 2015 -0500 ---------------------------------------------------------------------- .../client/samples/CreateLaunchExperiment.java | 73 +++++- .../tools/RegisterSampleApplications.java | 53 +++- .../server/src/main/resources/LSFTemplate.xslt | 43 ++-- .../server/src/main/resources/PBSTemplate.xslt | 3 +- .../credential/store/client/TestSSLClient.java | 2 +- .../store/store/impl/db/SSHCredentialTest.java | 6 +- .../test/resources/airavata-server.properties | 254 +++++++++++++++++++ .../server/src/main/assembly/bin-assembly.xml | 1 + .../airavata/gfac/server/GfacServerHandler.java | 1 - .../org/apache/airavata/gfac/RequestData.java | 2 + .../gfac/gsissh/util/GFACGSISSHUtils.java | 10 +- .../airavata/gfac/ssh/util/GFACSSHUtils.java | 23 +- .../ssh/api/job/JobManagerConfiguration.java | 9 +- .../gsi/ssh/api/job/LSFJobConfiguration.java | 18 +- .../gsi/ssh/api/job/LSFOutputParser.java | 1 + .../gsi/ssh/api/job/PBSJobConfiguration.java | 15 ++ .../gsi/ssh/api/job/SlurmJobConfiguration.java | 15 ++ .../gsi/ssh/impl/GSISSHAbstractCluster.java | 10 +- .../airavata/gsi/ssh/impl/RawCommandInfo.java | 8 +- .../impl/DefaultSSHApiTestWithMyProxyAuth.java | 15 +- 20 files changed, 500 insertions(+), 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java index 3fc74eb..bb2ae6e 100644 --- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java +++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java @@ -58,7 +58,7 @@ public class CreateLaunchExperiment { private static final String DEFAULT_GATEWAY = "php_reference_gateway"; private static Airavata.Client airavataClient; - private static String echoAppId = "Echo_f828a575-7f17-4149-9d45-abe2aa9c6109"; + private static String echoAppId = "Echo_802454e5-6358-4371-9a04-3d5d59cecbc7"; private static String mpiAppId = "HelloMPI_720e159f-198f-4daa-96ca-9f5eafee92c9"; private static String wrfAppId = "WRF_7ad5da38-c08b-417c-a9ea-da9298839762"; private static String amberAppId = "Amber_a56d457c-f239-4c0b-ba00-66bda936f7bc"; @@ -69,15 +69,15 @@ public class CreateLaunchExperiment { private static String trinityAppId = "Trinity_e894acf5-9bca-46e8-a1bd-7e2d5155191a"; private static String autodockAppId = "AutoDock_43d9fdd0-c404-49f4-b913-3abf9080a8c9"; - private static String localHost = "localhost"; private static String trestlesHostName = "trestles.sdsc.xsede.org"; private static String unicoreHostName = "fsd-cloud15.zam.kfa-juelich.de"; private static String stampedeHostName = "stampede.tacc.xsede.org"; private static String br2HostName = "bigred2.uits.iu.edu"; + private static String umassrcHostName = "ghpcc06.umassrc.org"; private static String gatewayId; - + // unicore service endpoint url private static final String unicoreEndPointURL = "https://fsd-cloud15.zam.kfa-juelich.de:7000/INTEROP1/services/BESFactory?res=default_bes_factory";; @@ -88,7 +88,7 @@ public class CreateLaunchExperiment { // createGateway(); // getGateway("testGatewayId"); // registerApplications(); // run this only the first time - createAndLaunchExp(); + createAndLaunchExp(); } private static String fsdResourceId; @@ -154,13 +154,14 @@ public class CreateLaunchExperiment { // final String expId = createExperimentForBR2Amber(airavataClient); // final String expId = createExperimentWRFStampede(airavataClient); // final String expId = createExperimentForStampedeAmber(airavataClient); - final String expId = createExperimentForTrestlesAmber(airavataClient); +// final String expId = createExperimentForTrestlesAmber(airavataClient); // final String expId = createExperimentGROMACSStampede(airavataClient); // final String expId = createExperimentESPRESSOStampede(airavataClient); // final String expId = createExperimentLAMMPSStampede(airavataClient); // final String expId = createExperimentNWCHEMStampede(airavataClient); // final String expId = createExperimentTRINITYStampede(airavataClient); // final String expId = createExperimentAUTODOCKStampede(airavataClient); // this is not working , we need to register AutoDock app on stampede + final String expId = createExperimentForLSF(airavataClient); // final String expId = "Ultrascan_ln_eb029947-391a-4ccf-8ace-9bafebe07cc0"; System.out.println("Experiment ID : " + expId); // updateExperiment(airavata, expId); @@ -185,6 +186,8 @@ public class CreateLaunchExperiment { //Register all compute hosts registerSampleApplications.registerXSEDEHosts(); + registerSampleApplications.registerNonXSEDEHosts(); + //Register Gateway Resource Preferences registerSampleApplications.registerGatewayResourceProfile(); @@ -1343,6 +1346,63 @@ public class CreateLaunchExperiment { return null; } + public static String createExperimentForLSF(Airavata.Client client) throws TException { + try { + List<InputDataObjectType> exInputs = new ArrayList<InputDataObjectType>(); + InputDataObjectType input = new InputDataObjectType(); + input.setName("Input_to_Echo"); + input.setType(DataType.STRING); + input.setValue("Echoed_Output=Hello World"); + input.setRequiredToAddedToCommandLine(true); + exInputs.add(input); + + List<OutputDataObjectType> exOut = new ArrayList<OutputDataObjectType>(); + OutputDataObjectType output = new OutputDataObjectType(); + output.setName("output_file"); + output.setType(DataType.URI); + output.setValue(""); + exOut.add(output); + + Project project = ProjectModelUtil.createProject("default", "lg11w", "test project"); + String projectId = client.createProject(DEFAULT_GATEWAY, project); + + Experiment simpleExperiment = + ExperimentModelUtil.createSimpleExperiment(projectId, "lg11w", "sshEchoExperiment", "StressMem", echoAppId, exInputs); + simpleExperiment.setExperimentOutputs(exOut); + + Map<String, String> computeResources = airavataClient.getAvailableAppInterfaceComputeResources(echoAppId); + if (computeResources != null && computeResources.size() != 0) { + for (String id : computeResources.keySet()) { + String resourceName = computeResources.get(id); + if (resourceName.equals(umassrcHostName)) { + ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling(id, 10, 1, 1, "long", 60, 0, 1000, "airavata"); + UserConfigurationData userConfigurationData = new UserConfigurationData(); + userConfigurationData.setAiravataAutoSchedule(false); + userConfigurationData.setOverrideManualScheduledParams(false); + userConfigurationData.setComputationalResourceScheduling(scheduling); + simpleExperiment.setUserConfigurationData(userConfigurationData); + simpleExperiment.setEmailAddresses(Arrays.asList(new String[]{"[email protected]"})); + return client.createExperiment(DEFAULT_GATEWAY, simpleExperiment); + } + } + } + } catch (AiravataSystemException e) { + logger.error("Error occured while creating the experiment...", e.getMessage()); + throw new AiravataSystemException(e); + } catch (InvalidRequestException e) { + logger.error("Error occured while creating the experiment...", e.getMessage()); + throw new InvalidRequestException(e); + } catch (AiravataClientException e) { + logger.error("Error occured while creating the experiment...", e.getMessage()); + throw new AiravataClientException(e); + } catch (TException e) { + logger.error("Error occured while creating the experiment...", e.getMessage()); + throw new TException(e); + } + return null; + } + + public static String createExperimentForBR2Amber(Airavata.Client client) throws TException { try { List<InputDataObjectType> exInputs = client.getApplicationInputs(amberAppId); @@ -1538,8 +1598,7 @@ public class CreateLaunchExperiment { public static void launchExperiment(Airavata.Client client, String expId) throws TException { try { -// String tokenId = "5f116091-0ad3-4ab6-9df7-6ac909f21f8b"; - String tokenId ="aaaaaa"; + String tokenId ="aa-dcdb-48e3-9cd5-ac90b710d55e"; client.launchExperiment(expId, tokenId); } catch (ExperimentNotFoundException e) { logger.error("Error occured while launching the experiment...", e.getMessage()); http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java index 7f7ac1d..8ae2dd2 100644 --- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java +++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java @@ -54,6 +54,8 @@ public class RegisterSampleApplications { private static String stampedeResourceId = "stampede.tacc.xsede.org_92ac5ed6-35a5-4910-82ef-48f128f9245a"; private static String trestlesResourceId = "trestles.sdsc.xsede.org_db29986e-5a27-4949-ae7f-04a6012d0d35"; private static String bigredResourceId = "bigred2.uits.iu.edu_3eae6e9d-a1a7-44ec-ac85-3796ef726ef1"; + private static String lsfResourceId = "lsf_3eae6e9d-a1a7-44ec-ac85-3796ef726ef1"; + private static String fsdResourceId; // unicore service endpoint url private static final String unicoreEndPointURL = "https://fsd-cloud15.zam.kfa-juelich.de:7000/INTEROP1/services/BESFactory?res=default_bes_factory";; @@ -73,6 +75,7 @@ public class RegisterSampleApplications { private static final String monteXName = "TinkerMonte"; private static final String gaussianName = "Gaussian"; private static final String gamessName = "Gamess"; + private static final String stressMemName = "StressMem"; //Appplication Descriptions private static final String echoDescription = "A Simple Echo Application"; @@ -90,6 +93,7 @@ public class RegisterSampleApplications { private static final String gaussianDescription = "Grid Chem Gaussian Application"; private static final String gamessDescription = "A Gamess Application"; + //App Module Id's private static String echoModuleId; private static String amberModuleId; @@ -136,6 +140,7 @@ public class RegisterSampleApplications { //Register all compute hosts registerSampleApplications.registerXSEDEHosts(); + registerSampleApplications.registerNonXSEDEHosts(); //Register Gateway Resource Preferences registerSampleApplications.registerGatewayResourceProfile(); @@ -200,10 +205,24 @@ public class RegisterSampleApplications { bigredResourceId = registerComputeHost("bigred2.uits.iu.edu", "IU BigRed II Cluster", ResourceJobManagerType.PBS, "push", "/opt/torque/torque-4.2.3.1/bin/", SecurityProtocol.SSH_KEYS, 22, "aprun -n"); System.out.println("BigredII Resource Id is " + bigredResourceId); - + fsdResourceId = registerUnicoreEndpoint("fsd-cloud15.zam.kfa-juelich.de", "interop host", JobSubmissionProtocol.UNICORE, SecurityProtocol.GSI); System.out.println("FSd Resource Id: "+fsdResourceId); - + + } catch (TException e) { + e.printStackTrace(); + } + + } + + public void registerNonXSEDEHosts() { + try { + System.out.println("\n #### Registering Non-XSEDE Computational Resources #### \n"); + + //Register LSF resource + lsfResourceId = registerComputeHost("ghpcc06.umassrc.org", "LSF Cluster", + ResourceJobManagerType.LSF, "push", "source /etc/bashrc;/lsf/9.1/linux2.6-glibc2.3-x86_64/bin", SecurityProtocol.SSH_KEYS, 22, null); + System.out.println("LSF Resource Id is " + lsfResourceId); } catch (TException e) { e.printStackTrace(); @@ -317,7 +336,6 @@ public class RegisterSampleApplications { gamessName, "17May13", gamessDescription)); System.out.println("Gamess Module Id " + gamessModuleId); - } catch (TException e) { e.printStackTrace(); } @@ -341,6 +359,7 @@ public class RegisterSampleApplications { //Registering FSD Apps registerFSDApps(); + registerLSFApps(); } @@ -493,8 +512,8 @@ public class RegisterSampleApplications { e.printStackTrace(); } } - - + + public void registerMPIInterface() { try { System.out.println("#### Registering MPI Interface #### \n"); @@ -1084,6 +1103,22 @@ public class RegisterSampleApplications { } } + public void registerLSFApps() { + try { + System.out.println("#### Registering Application Deployments on Trestles #### \n"); + + //Register Echo + String echoAppDeployId = airavataClient.registerApplicationDeployment(DEFAULT_GATEWAY, + RegisterSampleApplicationsUtils.createApplicationDeployment(echoModuleId, lsfResourceId, + "/home/lg11w/executables/echo.sh", ApplicationParallelismType.SERIAL, + echoDescription, null, null, null)); + System.out.println("Echo on trestles deployment Id " + echoAppDeployId); + + } catch (TException e) { + e.printStackTrace(); + } + } + public void registerBigRedApps() { try { System.out.println("#### Registering Application Deployments on BigRed II #### \n"); @@ -1127,7 +1162,7 @@ public class RegisterSampleApplications { e.printStackTrace(); } } - + public void registerFSDApps() { try { System.out.println("#### Registering Application Deployments on FSD #### \n"); @@ -1207,6 +1242,10 @@ public class RegisterSampleApplications { ComputeResourcePreference bigRedResourcePreferences = RegisterSampleApplicationsUtils. createComputeResourcePreference(bigredResourceId, "TG-STA110014S", false, null, null, null, "/N/dc2/scratch/cgateway/gta-work-dirs"); + + ComputeResourcePreference lsfResourcePreferences = RegisterSampleApplicationsUtils. + createComputeResourcePreference(lsfResourceId, "airavata", false, null, null, null, + "/home/lg11w/mywork"); ComputeResourcePreference fsdResourcePreferences = RegisterSampleApplicationsUtils. createComputeResourcePreference(fsdResourceId, null, false, null, JobSubmissionProtocol.UNICORE, DataMovementProtocol.UNICORE_STORAGE_SERVICE,null); @@ -1217,6 +1256,7 @@ public class RegisterSampleApplications { gatewayResourceProfile.addToComputeResourcePreferences(trestlesResourcePreferences); gatewayResourceProfile.addToComputeResourcePreferences(bigRedResourcePreferences); gatewayResourceProfile.addToComputeResourcePreferences(fsdResourcePreferences); + gatewayResourceProfile.addToComputeResourcePreferences(lsfResourcePreferences); String gatewayProfile = airavataClient.registerGatewayResourceProfile(gatewayResourceProfile); System.out.println("Gateway Profile is registered with Id " + gatewayProfile); @@ -1232,6 +1272,7 @@ public class RegisterSampleApplications { properties.setProperty("stampedeResourceId", stampedeResourceId); properties.setProperty("trestlesResourceId", trestlesResourceId); properties.setProperty("bigredResourceId", bigredResourceId); + properties.setProperty("lsfResourceId", lsfResourceId); properties.setProperty("echoInterfaceId", echoInterfaceId); properties.setProperty("amberInterfaceId", amberInterfaceId); http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/configuration/server/src/main/resources/LSFTemplate.xslt ---------------------------------------------------------------------- diff --git a/modules/configuration/server/src/main/resources/LSFTemplate.xslt b/modules/configuration/server/src/main/resources/LSFTemplate.xslt index ab9fbbd..7081260 100644 --- a/modules/configuration/server/src/main/resources/LSFTemplate.xslt +++ b/modules/configuration/server/src/main/resources/LSFTemplate.xslt @@ -9,47 +9,61 @@ <xsl:output method="text" /> <xsl:template match="/ns:JobDescriptor"> <xsl:param name="quote">"</xsl:param> - #! /bin/bash - # LSF batch job submission script generated by Apache Airavata - # +#! /bin/bash +# LSF batch job submission script generated by Apache Airavata +# + <xsl:choose> + <xsl:when test="ns:shellName"> +#BSUB -L <xsl:value-of select="ns:shellName"/> + </xsl:when></xsl:choose> <xsl:choose> <xsl:when test="ns:queueName"> - #BSUB -n <xsl:value-of select="ns:queueName"/> +#BSUB -q <xsl:value-of select="ns:queueName"/> + </xsl:when> + </xsl:choose> + <xsl:choose> + <xsl:when test="ns:nodes"> +#BSUB -n <xsl:value-of select="ns:nodes"/> </xsl:when> </xsl:choose> <xsl:choose> <xsl:when test="ns:mailAddress"> - #BSUB -u <xsl:value-of select="ns:mailAddress"/> +#BSUB -u <xsl:value-of select="ns:mailAddress"/> </xsl:when> </xsl:choose> <xsl:choose> <xsl:when test="ns:jobName"> - #BSUB -J <xsl:value-of select="ns:jobName"/> +#BSUB -J <xsl:value-of select="ns:jobName"/> </xsl:when> </xsl:choose> <xsl:choose> - <xsl:when test="ns:acountString"> - #BSUB -P <xsl:value-of select="ns:acountString"/> - </xsl:when> - </xsl:choose> + <xsl:when test="ns:acountString"> +#BSUB -P <xsl:value-of select="ns:acountString"/> + </xsl:when> + </xsl:choose> <xsl:choose> <xsl:when test="ns:maxWallTime"> - #BSUB -W<xsl:value-of select="ns:maxWallTime"/> +#BSUB -W <xsl:value-of select="ns:maxWallTime"/> </xsl:when> </xsl:choose> <xsl:choose> <xsl:when test="ns:standardOutFile"> - #BSUB -o <xsl:value-of select="ns:standardOutFile"/> +#BSUB -o "<xsl:value-of select="ns:standardOutFile"/>" </xsl:when> </xsl:choose> <xsl:choose> <xsl:when test="ns:standardOutFile"> - #BSUB -e <xsl:value-of select="ns:standardErrorFile"/> +#BSUB -e "<xsl:value-of select="ns:standardErrorFile"/>" + </xsl:when> + </xsl:choose> + <xsl:choose> + <xsl:when test="ns:chassisName"> +#BSUB -m c<xsl:value-of select="ns:chassisName"/> </xsl:when> </xsl:choose> <xsl:choose> <xsl:when test="ns:usedMem"> - #BSUB -R rusage[mem=<xsl:value-of select="ns:usedMem"/>] +#BSUB -R rusage[mem=<xsl:value-of select="ns:usedMem"/>] </xsl:when> </xsl:choose> @@ -74,7 +88,6 @@ <xsl:value-of select="."/><xsl:text> </xsl:text> </xsl:for-each> <xsl:text>
</xsl:text> - ~/rabbitmq-java-client-bin-3.3.5/runjava.sh com.rabbitmq.examples.SimpleProducer amqp://<xsl:value-of select="ns:callBackIp"/><xsl:text> </xsl:text><xsl:value-of select="ns:userName"/>,<xsl:value-of select="ns:jobName"/><xsl:text> </xsl:text><xsl:value-of select="$quote"/><xsl:value-of select="$quote"/><xsl:text> </xsl:text><xsl:value-of select="ns:callBackPort"/> </xsl:template> </xsl:stylesheet> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/configuration/server/src/main/resources/PBSTemplate.xslt ---------------------------------------------------------------------- diff --git a/modules/configuration/server/src/main/resources/PBSTemplate.xslt b/modules/configuration/server/src/main/resources/PBSTemplate.xslt index aa3cccc..7676883 100644 --- a/modules/configuration/server/src/main/resources/PBSTemplate.xslt +++ b/modules/configuration/server/src/main/resources/PBSTemplate.xslt @@ -96,7 +96,6 @@ cd <xsl:text> </xsl:text><xsl:value-of select="ns:workingDirectory"/><xsl:text <xsl:value-of select="."/><xsl:text> </xsl:text> </xsl:for-each> <xsl:text>
</xsl:text> -~/rabbitmq-java-client-bin-3.3.5/runjava.sh com.rabbitmq.examples.SimpleProducer amqp://<xsl:value-of select="ns:callBackIp"/><xsl:text> </xsl:text><xsl:value-of select="ns:userName"/>,<xsl:value-of select="ns:jobName"/><xsl:text> </xsl:text><xsl:value-of select="$quote"/><xsl:value-of select="$quote"/><xsl:text> </xsl:text><xsl:value-of select="ns:callBackPort"/> -</xsl:template> +R</xsl:template> </xsl:stylesheet> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/credential-store/credential-store-service/src/main/java/org/apache/airavata/credential/store/client/TestSSLClient.java ---------------------------------------------------------------------- diff --git a/modules/credential-store/credential-store-service/src/main/java/org/apache/airavata/credential/store/client/TestSSLClient.java b/modules/credential-store/credential-store-service/src/main/java/org/apache/airavata/credential/store/client/TestSSLClient.java index cc5ebb6..1d9f4d9 100644 --- a/modules/credential-store/credential-store-service/src/main/java/org/apache/airavata/credential/store/client/TestSSLClient.java +++ b/modules/credential-store/credential-store-service/src/main/java/org/apache/airavata/credential/store/client/TestSSLClient.java @@ -63,7 +63,7 @@ public class TestSSLClient { TProtocol protocol = new TBinaryProtocol(transport); CredentialStoreService.Client client = new CredentialStoreService.Client(protocol); -// testSSHCredential(client); + testSSHCredential(client); testCertificateCredential(client); transport.close(); } catch (TTransportException e) { http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/credential-store/credential-store-service/src/test/java/org/apache/airavata/credential/store/store/impl/db/SSHCredentialTest.java ---------------------------------------------------------------------- diff --git a/modules/credential-store/credential-store-service/src/test/java/org/apache/airavata/credential/store/store/impl/db/SSHCredentialTest.java b/modules/credential-store/credential-store-service/src/test/java/org/apache/airavata/credential/store/store/impl/db/SSHCredentialTest.java index 7f44125..4f72b46 100644 --- a/modules/credential-store/credential-store-service/src/test/java/org/apache/airavata/credential/store/store/impl/db/SSHCredentialTest.java +++ b/modules/credential-store/credential-store-service/src/test/java/org/apache/airavata/credential/store/store/impl/db/SSHCredentialTest.java @@ -43,8 +43,8 @@ public class SSHCredentialTest { String userName = "airavata"; String password = "airavata"; String gatewayId = "default"; - String privateKeyPath = "/Users/chathuri/.ssh/id_dsa"; - String pubKeyPath = "/Users/chathuri/.ssh/id_dsa.pub"; + String privateKeyPath = "/Users/lginnali/.ssh/id_dsa"; + String pubKeyPath = "/Users/lginnali/.ssh/id_dsa.pub"; try { AiravataUtils.setExecutionAsServer(); @@ -67,7 +67,7 @@ public class SSHCredentialTest { pubKeyStream.close(); sshCredential.setPrivateKey(bFilePub); sshCredential.setPublicKey(bFilePub); - sshCredential.setPassphrase("test"); + sshCredential.setPassphrase("gjtlmiJdas7wph"); writer.writeCredentials(sshCredential); System.out.println(token); } catch (ClassNotFoundException e) { http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/credential-store/credential-store-service/src/test/resources/airavata-server.properties ---------------------------------------------------------------------- diff --git a/modules/credential-store/credential-store-service/src/test/resources/airavata-server.properties b/modules/credential-store/credential-store-service/src/test/resources/airavata-server.properties new file mode 100644 index 0000000..9bdce87 --- /dev/null +++ b/modules/credential-store/credential-store-service/src/test/resources/airavata-server.properties @@ -0,0 +1,254 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +########################################################################### +# +# This properties file provides configuration for all Airavata Services: +# API Server, Registry, Workflow Interpreter, GFac, Orchestrator +# +########################################################################### + +########################################################################### +# API Server Registry Configuration +########################################################################### + +#for derby [AiravataJPARegistry] +registry.jdbc.driver=org.apache.derby.jdbc.ClientDriver +registry.jdbc.url=jdbc:derby://localhost:1527/persistent_data;create=true;user=airavata;password=airavata +# MySql database configuration +#registry.jdbc.driver=com.mysql.jdbc.Driver +#registry.jdbc.url=jdbc:mysql://localhost:3306/persistent_data +registry.jdbc.user=airavata +registry.jdbc.password=airavata +start.derby.server.mode=true +validationQuery=SELECT 1 from CONFIGURATION +jpa.cache.size=5000 +#jpa.connection.properties=MaxActive=10,MaxIdle=5,MinIdle=2,MaxWait=60000,testWhileIdle=true,testOnBorrow=true + +# Properties for default user mode +default.registry.user=admin +default.registry.password=admin +default.registry.password.hash.method=SHA +default.registry.gateway=php_reference_gateway + +#ip=127.0.0.1 + +########################################################################### +# Application Catalog DB Configuration +########################################################################### +#for derby [AiravataJPARegistry] +appcatalog.jdbc.driver=org.apache.derby.jdbc.ClientDriver +appcatalog.jdbc.url=jdbc:derby://localhost:1527/app_catalog;create=true;user=airavata;password=airavata +# MySql database configuration +#appcatalog.jdbc.driver=com.mysql.jdbc.Driver +#appcatalog.jdbc.url=jdbc:mysql://localhost:3306/app_catalog +appcatalog.jdbc.user=airavata +appcatalog.jdbc.password=airavata +appcatalog.validationQuery=SELECT 1 from CONFIGURATION + +########################################################################### +# Server module Configuration +########################################################################### + +servers=apiserver,orchestrator,gfac,credentialstore +#shutdown.trategy=NONE +shutdown.trategy=SELF_TERMINATE + + +apiserver.server.host=localhost +apiserver.server.port=8930 +apiserver.server.min.threads=50 +orchestrator.server.host=localhost +orchestrator.server.port=8940 +gfac.server.host=localhost +gfac.server.port=8950 +orchestrator.server.min.threads=50 + +########################################################################### +# Job Scheduler can send informative email messages to you about the status of your job. +# Specify a string which consists of either the single character "n" (no mail), or one or more +# of the characters "a" (send mail when job is aborted), "b" (send mail when job begins), +# and "e" (send mail when job terminates). The default is "a" if not specified. +########################################################################### + +job.notification.enable=true +#Provide comma separated email ids as a string if more than one +job.notification.emailids= +job.notification.flags=abe + +########################################################################### +# Credential Store module Configuration +########################################################################### +start.credential.store=false +credential.store.keystore.url=/Users/lginnali/Downloads/airavata_sym.jks +credential.store.keystore.alias=airavata +credential.store.keystore.password=airavata +credential.store.jdbc.url=jdbc:derby://localhost:1527/persistent_data;create=true;user=airavata;password=airavata +credential.store.jdbc.user=airavata +credential.store.jdbc.password=airavata +credential.store.jdbc.driver=org.apache.derby.jdbc.ClientDriver +credential.store.server.host=localhost +credential.store.server.port=8960 +credentialstore=org.apache.airavata.credential.store.server.CredentialStoreServer +credential.store.thrift.server.keystore=/Users/chathuri/dev/airavata/credential-store/oa4mp/airavata.jks +credential.store.thrift.server.keystore.password=airavata + +notifier.enabled=false +#period in milliseconds +notifier.duration=5000 + +email.server=smtp.googlemail.com +email.server.port=465 +email.user=airavata +email.password=xxx +email.ssl=true [email protected] + +########################################################################### +# Airavata GFac MyProxy GSI credentials to access Grid Resources. +########################################################################### +# +# Security Configuration used by Airavata Generic Factory Service +# to interact with Computational Resources. +# +gfac.thread.pool.size=50 +airavata.server.thread.pool.size=50 +gfac=org.apache.airavata.gfac.server.GfacServer +myproxy.server=myproxy.teragrid.org +myproxy.username=ogce +myproxy.password= +myproxy.life=3600 +# XSEDE Trusted certificates can be downloaded from https://software.xsede.org/security/xsede-certs.tar.gz +trusted.cert.location=/Users/lahirugunathilake/Downloads/certificates +gfac.passive=true +# SSH PKI key pair or ssh password can be used SSH based authentication is used. +# if user specify both password authentication gets the higher preference + +################# ---------- For ssh key pair authentication ------------------- ################ +#public.ssh.key=/path to public key for ssh +#private.ssh.key=/path to private key file for ssh +#ssh.keypass=passphrase for the private key +#ssh.username=username for ssh connection +### Incase of password authentication. +#ssh.password=Password for ssh connection + + + +########################################################################### +# Airavata Workflow Interpreter Configurations +########################################################################### + +#runInThread=true +#provenance=true +#provenanceWriterThreadPoolSize=20 +#gfac.embedded=true +#workflowserver=org.apache.airavata.api.server.WorkflowServer + + +########################################################################### +# API Server module Configuration +########################################################################### +apiserver=org.apache.airavata.api.server.AiravataAPIServer + +########################################################################### +# Workflow Server module Configuration +########################################################################### + +workflowserver=org.apache.airavata.api.server.WorkflowServer + +########################################################################### +# Advance configuration to change service implementations +########################################################################### +# If false, disables two phase commit when submitting jobs +TwoPhase=true +# +# Class which implemented HostScheduler interface. It will determine the which host to submit the request +# +host.scheduler=org.apache.airavata.gfac.core.scheduler.impl.SimpleHostScheduler + +########################################################################### +# Monitoring module Configuration +########################################################################### + +#This will be the primary monitoring tool which runs in airavata, in future there will be multiple monitoring +#mechanisms and one would be able to start a monitor +monitors=org.apache.airavata.gfac.monitor.impl.pull.qstat.QstatMonitor,org.apache.airavata.gfac.monitor.impl.LocalJobMonitor + + +########################################################################### +# AMQP Notification Configuration +########################################################################### + + +amqp.notification.enable=1 + +amqp.broker.host=localhost +amqp.broker.port=5672 +amqp.broker.username=guest +amqp.broker.password=guest + +amqp.sender=org.apache.airavata.wsmg.client.amqp.rabbitmq.AMQPSenderImpl +amqp.topic.sender=org.apache.airavata.wsmg.client.amqp.rabbitmq.AMQPTopicSenderImpl +amqp.broadcast.sender=org.apache.airavata.wsmg.client.amqp.rabbitmq.AMQPBroadcastSenderImpl + +#,org.apache.airavata.gfac.monitor.impl.push.amqp.AMQPMonitor +#This is the amqp related configuration and this lists down the Rabbitmq host, this is an xsede specific configuration +amqp.hosts=info1.dyn.teragrid.org,info2.dyn.teragrid.org +proxy.file.path=/Users/lahirugunathilake/Downloads/x509up_u503876 +connection.name=xsede +#publisher +activity.listeners=org.apache.airavata.gfac.core.monitor.AiravataJobStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataTaskStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataWorkflowNodeStatusUpdator,org.apache.airavata.api.server.listener.AiravataExperimentStatusUpdator,org.apache.airavata.gfac.core.monitor.GfacInternalStatusUpdator,org.apache.airavata.workflow.engine.util.ProxyMonitorPublisher +publish.rabbitmq=false +status.publisher=org.apache.airavata.messaging.core.impl.RabbitMQStatusPublisher +task.launch.publisher=org.apache.airavata.messaging.core.impl.RabbitMQTaskLaunchPublisher +rabbitmq.broker.url=amqp://localhost:5672 +rabbitmq.status.exchange.name=airavata_rabbitmq_exchange +rabbitmq.task.launch.exchange.name=airavata_task_launch_rabbitmq_exchange + + +########################################################################### +# Orchestrator module Configuration +########################################################################### + +#job.submitter=org.apache.airavata.orchestrator.core.impl.GFACEmbeddedJobSubmitter +job.submitter=org.apache.airavata.orchestrator.core.impl.GFACPassiveJobSubmitter +#job.submitter=org.apache.airavata.orchestrator.core.impl.GFACRPCJobSubmitter +job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator +submitter.interval=10000 +threadpool.size=10 +start.submitter=true +embedded.mode=true +enable.validation=true +orchestrator=org.apache.airavata.orchestrator.server.OrchestratorServer + +########################################################################### +# Zookeeper Server Configuration +########################################################################### + +embedded.zk=false +zookeeper.server.host=localhost +zookeeper.server.port=2181 +airavata-server=/api-server +orchestrator-server=/orchestrator-server +gfac-server=/gfac-server +gfac-experiments=/gfac-experiments +gfac-server-name=gfac-node0 +orchestrator-server-name=orch-node0 +airavata-server-name=api-node0 http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/distribution/server/src/main/assembly/bin-assembly.xml ---------------------------------------------------------------------- diff --git a/modules/distribution/server/src/main/assembly/bin-assembly.xml b/modules/distribution/server/src/main/assembly/bin-assembly.xml index 4f061e4..18ccad6 100644 --- a/modules/distribution/server/src/main/assembly/bin-assembly.xml +++ b/modules/distribution/server/src/main/assembly/bin-assembly.xml @@ -127,6 +127,7 @@ <include>gfac-config.xml</include> <include>PBSTemplate.xslt</include> <include>SLURMTemplate.xslt</include> + <include>LSFTemplate.xslt</include> <include>SGETemplate.xslt</include> <include>gsissh.properties</include> </includes> http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java index 583ec07..1687462 100644 --- a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java +++ b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java @@ -31,7 +31,6 @@ import org.apache.airavata.common.utils.AiravataZKUtils; import org.apache.airavata.common.utils.Constants; import org.apache.airavata.common.utils.MonitorPublisher; import org.apache.airavata.common.utils.ServerSettings; -import org.apache.airavata.gfac.GFacException; import org.apache.airavata.gfac.core.cpi.BetterGfacImpl; import org.apache.airavata.gfac.core.cpi.GFac; import org.apache.airavata.gfac.core.utils.GFacThreadPoolExecutor; http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/RequestData.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/RequestData.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/RequestData.java index 9f45ce6..000ea7d 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/RequestData.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/RequestData.java @@ -49,6 +49,8 @@ public class RequestData { private int myProxyLifeTime = DEFAULT_LIFE_TIME; + + public RequestData() { } http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java index 7a432e8..631f59c 100644 --- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java +++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java @@ -60,6 +60,8 @@ public class GFACGSISSHUtils { public static final String PBS_JOB_MANAGER = "pbs"; public static final String SLURM_JOB_MANAGER = "slurm"; public static final String SUN_GRID_ENGINE_JOB_MANAGER = "UGE"; + public static final String LSF_JOB_MANAGER = "lsf"; + public static int maxClusterCount = 5; public static Map<String, List<Cluster>> clusters = new HashMap<String, List<Cluster>>(); @@ -138,6 +140,8 @@ public class GFACGSISSHUtils { jConfig = CommonUtils.getSLURMJobManager(installedParentPath); } else if (SUN_GRID_ENGINE_JOB_MANAGER.equalsIgnoreCase(jobManager)) { jConfig = CommonUtils.getSGEJobManager(installedParentPath); + }else if(LSF_JOB_MANAGER.equalsIgnoreCase(jobManager)) { + jConfig = CommonUtils.getLSFJobManager(installedParentPath); } } pbsCluster = new PBSCluster(serverInfo, tokenizedMyProxyAuthInfo, jConfig); @@ -177,7 +181,7 @@ public class GFACGSISSHUtils { if(ServerSettings.getSetting(ServerSettings.JOB_NOTIFICATION_ENABLE).equalsIgnoreCase("true")){ jobDescriptor.setMailOptions(ServerSettings.getSetting(ServerSettings.JOB_NOTIFICATION_FLAGS)); String emailids = ServerSettings.getSetting(ServerSettings.JOB_NOTIFICATION_EMAILIDS); - + if(jobExecutionContext.getTaskData().isSetEmailAddresses()){ List<String> emailList = jobExecutionContext.getTaskData().getEmailAddresses(); String elist = GFacUtils.listToCsv(emailList, ','); @@ -204,6 +208,7 @@ public class GFACGSISSHUtils { jobDescriptor.setStandardOutFile(jobExecutionContext.getStandardOutput()); jobDescriptor.setStandardErrorFile(jobExecutionContext.getStandardError()); String computationalProjectAccount = taskData.getTaskScheduling().getComputationalProjectAccount(); + taskData.getEmailAddresses(); if (computationalProjectAccount == null){ ComputeResourcePreference computeResourcePreference = jobExecutionContext.getApplicationContext().getComputeResourcePreference(); if (computeResourcePreference != null) { @@ -311,6 +316,9 @@ public class GFACGSISSHUtils { } if (taskScheduling.getWallTimeLimit() > 0) { jobDescriptor.setMaxWallTime(String.valueOf(taskScheduling.getWallTimeLimit())); + if(resourceJobManager.getResourceJobManagerType().equals(ResourceJobManagerType.LSF)){ + jobDescriptor.setMaxWallTimeForLSF(String.valueOf(taskScheduling.getWallTimeLimit())); + } } if (taskScheduling.getTotalPhysicalMemory() > 0) { http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java index a9e08d3..c4d25f5 100644 --- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java +++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java @@ -25,6 +25,7 @@ import org.airavata.appcatalog.cpi.AppCatalogException; import org.apache.airavata.common.exception.ApplicationSettingsException; import org.apache.airavata.common.utils.ServerSettings; import org.apache.airavata.credential.store.credential.impl.ssh.SSHCredential; +import org.apache.airavata.gfac.Constants; import org.apache.airavata.gfac.GFacException; import org.apache.airavata.gfac.RequestData; import org.apache.airavata.gfac.core.context.JobExecutionContext; @@ -41,6 +42,7 @@ import org.apache.airavata.gsi.ssh.api.job.JobDescriptor; import org.apache.airavata.gsi.ssh.api.job.JobManagerConfiguration; import org.apache.airavata.gsi.ssh.impl.GSISSHAbstractCluster; import org.apache.airavata.gsi.ssh.impl.PBSCluster; +import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPasswordAuthenticationInfo; import org.apache.airavata.gsi.ssh.util.CommonUtils; import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription; import org.apache.airavata.model.appcatalog.appinterface.DataType; @@ -96,15 +98,21 @@ public class GFACSSHUtils { Cluster pbsCluster = null; try { - TokenizedSSHAuthInfo tokenizedSSHAuthInfo = new TokenizedSSHAuthInfo(requestData); + AuthenticationInfo tokenizedSSHAuthInfo = new TokenizedSSHAuthInfo(requestData); String installedParentPath = jobExecutionContext.getResourceJobManager().getJobManagerBinPath(); if (installedParentPath == null) { installedParentPath = "/"; } - SSHCredential credentials = tokenizedSSHAuthInfo.getCredentials();// this is just a call to get and set credentials in to this object,data will be used + SSHCredential credentials =((TokenizedSSHAuthInfo)tokenizedSSHAuthInfo).getCredentials();// this is just a call to get and set credentials in to this object,data will be used + if(credentials.getPrivateKey()==null || credentials.getPublicKey()==null){ + // now we fall back to username password authentication + Properties configurationProperties = ServerSettings.getProperties(); + tokenizedSSHAuthInfo = new DefaultPasswordAuthenticationInfo(configurationProperties.getProperty(Constants.SSH_PASSWORD)); + } serverInfo.setUserName(credentials.getPortalUserName()); jobExecutionContext.getExperiment().setUserName(credentials.getPortalUserName()); + // inside the pbsCluser object String key = credentials.getPortalUserName() + jobExecutionContext.getHostName() + serverInfo.getPort(); @@ -147,8 +155,11 @@ public class GFACSSHUtils { jConfig = CommonUtils.getSLURMJobManager(installedParentPath); } else if (SUN_GRID_ENGINE_JOB_MANAGER.equalsIgnoreCase(jobManager)) { jConfig = CommonUtils.getSGEJobManager(installedParentPath); + } else if (LSF_JOB_MANAGER.equalsIgnoreCase(jobManager)) { + jConfig = CommonUtils.getLSFJobManager(installedParentPath); } } + pbsCluster = new PBSCluster(serverInfo, tokenizedSSHAuthInfo,jConfig); List<Cluster> pbsClusters = null; if (!(clusters.containsKey(key))) { @@ -322,7 +333,7 @@ public class GFACSSHUtils { List<String> inputValues = new ArrayList<String>(); MessageContext input = jobExecutionContext.getInMessageContext(); - // sort the inputs first and then build the command List + // sort the inputs first and then build the command ListR Comparator<InputDataObjectType> inputOrderComparator = new Comparator<InputDataObjectType>() { @Override public int compare(InputDataObjectType inputDataObjectType, InputDataObjectType t1) { @@ -411,6 +422,12 @@ public class GFACSSHUtils { } if (taskScheduling.getWallTimeLimit() > 0) { jobDescriptor.setMaxWallTime(String.valueOf(taskScheduling.getWallTimeLimit())); + if(resourceJobManager.getResourceJobManagerType().equals(ResourceJobManagerType.LSF)){ + jobDescriptor.setMaxWallTimeForLSF(String.valueOf(taskScheduling.getWallTimeLimit())); + } + } + if (taskScheduling.getTotalPhysicalMemory() > 0) { + jobDescriptor.setUsedMemory(taskScheduling.getTotalPhysicalMemory() + ""); } } else { logger.error("Task scheduling cannot be null at this point.."); http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobManagerConfiguration.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobManagerConfiguration.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobManagerConfiguration.java index f68ba29..85a843e 100644 --- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobManagerConfiguration.java +++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobManagerConfiguration.java @@ -37,6 +37,13 @@ public interface JobManagerConfiguration { public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath); public OutputParser getParser(); - + public String getInstalledPath(); + + public String getBaseCancelCommand(); + + public String getBaseMonitorCommand(); + + public String getBaseSubmitCommand(); + } http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFJobConfiguration.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFJobConfiguration.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFJobConfiguration.java index 018d49d..46fe9ad 100644 --- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFJobConfiguration.java +++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFJobConfiguration.java @@ -80,7 +80,7 @@ public class LSFJobConfiguration implements JobManagerConfiguration { @Override public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath) { - return new RawCommandInfo(this.installedPath + "bsub <" + + return new RawCommandInfo(this.installedPath + "bsub < " + workingDirectory + File.separator + FilenameUtils.getName(pbsFilePath)); } @@ -97,4 +97,20 @@ public class LSFJobConfiguration implements JobManagerConfiguration { public String getInstalledPath() { return installedPath; } + + + @Override + public String getBaseCancelCommand() { + return "bkill"; + } + + @Override + public String getBaseMonitorCommand() { + return "bjobs"; + } + + @Override + public String getBaseSubmitCommand() { + return "bsub"; + } } http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFOutputParser.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFOutputParser.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFOutputParser.java index 5490ff6..bd02e37 100644 --- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFOutputParser.java +++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFOutputParser.java @@ -57,6 +57,7 @@ public class LSFOutputParser implements OutputParser { @Override public void parseJobStatuses(String userName, Map<String, JobStatus> statusMap, String rawOutput) throws SSHApiException { logger.debug(rawOutput); + String[] info = rawOutput.split("\n"); // int lastStop = 0; for (String jobID : statusMap.keySet()) { http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/PBSJobConfiguration.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/PBSJobConfiguration.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/PBSJobConfiguration.java index 18ea772..e935dfb 100644 --- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/PBSJobConfiguration.java +++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/PBSJobConfiguration.java @@ -95,4 +95,19 @@ public class PBSJobConfiguration implements JobManagerConfiguration { public RawCommandInfo getUserBasedMonitorCommand(String userName) { return new RawCommandInfo(this.installedPath + "qstat -u " + userName); } + + @Override + public String getBaseCancelCommand() { + return "qdel"; + } + + @Override + public String getBaseMonitorCommand() { + return "qstat"; + } + + @Override + public String getBaseSubmitCommand() { + return "qsub "; + } } http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/SlurmJobConfiguration.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/SlurmJobConfiguration.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/SlurmJobConfiguration.java index 0b774ed..807ac42 100644 --- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/SlurmJobConfiguration.java +++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/SlurmJobConfiguration.java @@ -94,4 +94,19 @@ public class SlurmJobConfiguration implements JobManagerConfiguration{ public RawCommandInfo getUserBasedMonitorCommand(String userName) { return new RawCommandInfo(this.installedPath + "squeue -u " + userName); } + + @Override + public String getBaseCancelCommand() { + return "scancel"; + } + + @Override + public String getBaseMonitorCommand() { + return "squeue"; + } + + @Override + public String getBaseSubmitCommand() { + return "sbatch"; + } } http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java index dd7f2d9..0420cff 100644 --- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java +++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java @@ -241,7 +241,7 @@ public class GSISSHAbstractCluster implements Cluster { StandardOutReader stdOutReader = new StandardOutReader(); CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader); - String outputifAvailable = getOutputifAvailable(stdOutReader, "Error reading output of job submission",rawCommandInfo.getBaseCommand(jobManagerConfiguration.getInstalledPath())); + String outputifAvailable = getOutputifAvailable(stdOutReader, "Error reading output of job submission",jobManagerConfiguration.getBaseCancelCommand()); // this might not be the case for all teh resources, if so Cluster implementation can override this method // because here after cancelling we try to get the job description and return it back try { @@ -274,7 +274,7 @@ public class GSISSHAbstractCluster implements Cluster { //Check whether pbs submission is successful or not, if it failed throw and exception in submitJob method // with the error thrown in qsub command // - String outputifAvailable = getOutputifAvailable(standardOutReader,"Error reading output of job submission",rawCommandInfo.getBaseCommand(jobManagerConfiguration.getInstalledPath())); + String outputifAvailable = getOutputifAvailable(standardOutReader,"Error reading output of job submission",jobManagerConfiguration.getBaseSubmitCommand()); OutputParser outputParser = jobManagerConfiguration.getParser(); return outputParser.parseJobSubmission(outputifAvailable); } @@ -411,7 +411,7 @@ public class GSISSHAbstractCluster implements Cluster { RawCommandInfo rawCommandInfo = jobManagerConfiguration.getMonitorCommand(jobID); StandardOutReader stdOutReader = new StandardOutReader(); CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader); - String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !",rawCommandInfo.getBaseCommand(jobManagerConfiguration.getInstalledPath())); + String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !",jobManagerConfiguration.getBaseMonitorCommand()); JobDescriptor jobDescriptor = new JobDescriptor(); jobManagerConfiguration.getParser().parseSingleJob(jobDescriptor, result); return jobDescriptor; @@ -421,7 +421,7 @@ public class GSISSHAbstractCluster implements Cluster { RawCommandInfo rawCommandInfo = jobManagerConfiguration.getMonitorCommand(jobID); StandardOutReader stdOutReader = new StandardOutReader(); CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader); - String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !", rawCommandInfo.getBaseCommand(jobManagerConfiguration.getInstalledPath())); + String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !", jobManagerConfiguration.getBaseMonitorCommand()); return jobManagerConfiguration.getParser().parseJobStatus(jobID, result); } @@ -638,7 +638,7 @@ public class GSISSHAbstractCluster implements Cluster { } } } - String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !", rawCommandInfo.getBaseCommand(jobManagerConfiguration.getInstalledPath())); + String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !", jobManagerConfiguration.getBaseMonitorCommand()); jobManagerConfiguration.getParser().parseJobStatuses(userName, jobIDs, result); } http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/RawCommandInfo.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/RawCommandInfo.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/RawCommandInfo.java index ac88fe7..0e9d16e 100644 --- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/RawCommandInfo.java +++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/RawCommandInfo.java @@ -48,13 +48,7 @@ public class RawCommandInfo implements CommandInfo { public String getRawCommand() { return rawCommand; } - /* - * To get command without the path and other parameters. This is required to read errors - */ - public String getBaseCommand(String installPath) { - return rawCommand.substring(rawCommand.lastIndexOf(installPath)+1, rawCommand.indexOf(" ")); - } - + public void setRawCommand(String rawCommand) { this.rawCommand = rawCommand; } http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTestWithMyProxyAuth.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTestWithMyProxyAuth.java b/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTestWithMyProxyAuth.java index 5a12723..0ca7546 100644 --- a/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTestWithMyProxyAuth.java +++ b/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTestWithMyProxyAuth.java @@ -26,6 +26,7 @@ import org.apache.airavata.gsi.ssh.api.*; import org.apache.airavata.gsi.ssh.api.authentication.GSIAuthenticationInfo; import org.apache.airavata.gsi.ssh.api.job.JobDescriptor; import org.apache.airavata.gsi.ssh.config.ConfigReader; +import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPasswordAuthenticationInfo; import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo; import org.apache.airavata.gsi.ssh.util.CommonUtils; import org.slf4j.Logger; @@ -49,17 +50,13 @@ public class DefaultSSHApiTestWithMyProxyAuth { public static void main(String[]ars){ - String myProxyUserName = "ogce"; - String myProxyPassword = "OGCE@xsede14"; - String certificateLocation = "/Users/raminder/.globus/certificates"; + String myProxyUserName = "lg11w"; - - GSIAuthenticationInfo authenticationInfo - = new MyProxyAuthenticationInfo(myProxyUserName, myProxyPassword, "myproxy.teragrid.org", - 7512, 17280000, certificateLocation); + DefaultPasswordAuthenticationInfo authenticationInfo + = new DefaultPasswordAuthenticationInfo(""); // Create command - CommandInfo commandInfo = new RawCommandInfo("/bin/ls"); + CommandInfo commandInfo = new RawCommandInfo("source /etc/bashrc; bsub </home/lg11w/mywork/sshEchoExperiment_9d267072-ca65-4ca8-847a-cd3d130f6050/366787899.lsf"); // Server info //Stampede @@ -68,7 +65,7 @@ public class DefaultSSHApiTestWithMyProxyAuth { // ServerInfo serverInfo = new ServerInfo(myProxyUserName, "trestles.sdsc.xsede.org", 22); //Lonestar - ServerInfo serverInfo = new ServerInfo(myProxyUserName, "lonestar.tacc.utexas.edu", 22); + ServerInfo serverInfo = new ServerInfo(myProxyUserName, "ghpcc06.umassrc.org", 22); // Output CommandOutput commandOutput = new SystemCommandOutput();
