Repository: airavata Updated Branches: refs/heads/master 155c3c753 -> b622683bc
performance test for messaging framework - AIRAVATA-1577 Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/b622683b Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/b622683b Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/b622683b Branch: refs/heads/master Commit: b622683bc8272cf9f1f88d40df1a2e19b10e2ac1 Parents: 155c3c7 Author: Chathuri Wimalasena <[email protected]> Authored: Tue Feb 17 14:58:08 2015 -0500 Committer: Chathuri Wimalasena <[email protected]> Committed: Tue Feb 17 14:58:08 2015 -0500 ---------------------------------------------------------------------- .../client/samples/CreateLaunchExperiment.java | 69 ++++++++------------ .../client/samples/RegisterSampleData.java | 2 +- .../tools/RegisterSampleApplications.java | 13 ++-- .../messaging/core/impl/RabbitMQPublisher.java | 3 + .../messaging/core/stats/StatCounter.java | 69 ++++++++++++++++++++ .../messaging/core/stats/WriterTask.java | 39 +++++++++++ 6 files changed, 148 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/b622683b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java index b90e0ff..200b76e 100644 --- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java +++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java @@ -25,6 +25,7 @@ import org.apache.airavata.api.Airavata; import org.apache.airavata.api.client.AiravataClientFactory; import org.apache.airavata.client.tools.RegisterSampleApplications; import org.apache.airavata.client.tools.RegisterSampleApplicationsUtils; +import org.apache.airavata.messaging.core.stats.StatCounter; import org.apache.airavata.model.appcatalog.appinterface.DataType; import org.apache.airavata.model.appcatalog.appinterface.InputDataObjectType; import org.apache.airavata.model.appcatalog.appinterface.OutputDataObjectType; @@ -42,28 +43,25 @@ import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import java.util.*; public class CreateLaunchExperiment { //FIXME: Read from a config file public static final String THRIFT_SERVER_HOST = "localhost"; public static final int THRIFT_SERVER_PORT = 8930; -// public static final String THRIFT_SERVER_HOST = "gw127.iu.xsede.org"; -// public static final int THRIFT_SERVER_PORT = 9930; +// public static final String THRIFT_SERVER_HOST = "gw111.iu.xsede.org"; +// public static final int THRIFT_SERVER_PORT = 9930; private final static Logger logger = LoggerFactory.getLogger(CreateLaunchExperiment.class); private static final String DEFAULT_USER = "default.registry.user"; private static final String DEFAULT_GATEWAY = "default.registry.gateway"; private static Airavata.Client airavataClient; - private static String echoAppId = "Echo_2e539083-665d-40fd-aaa2-4a751028326b"; + private static String echoAppId = "Echo_7d2a5cde-5b2a-4cad-ae50-f71668f4876d"; private static String mpiAppId = "HelloMPI_720e159f-198f-4daa-96ca-9f5eafee92c9"; private static String wrfAppId = "WRF_7ad5da38-c08b-417c-a9ea-da9298839762"; - private static String amberAppId = "Amber_eda074ea-223d-49d7-a942-6c8742249f36"; + private static String amberAppId = "Amber_42124128-628b-484c-829d-aff8b584eb00"; private static String gromacsAppId = "GROMACS_05622038-9edd-4cb1-824e-0b7cb993364b"; private static String espressoAppId = "ESPRESSO_10cc2820-5d0b-4c63-9546-8a8b595593c1"; private static String lammpsAppId = "LAMMPS_10893eb5-3840-438c-8446-d26c7ecb001f"; @@ -95,19 +93,19 @@ public class CreateLaunchExperiment { public static void createAndLaunchExp() throws TException { // final String expId = createEchoExperimentForFSD(airavataClient); try { - for (int i = 0; i < 1; i++) { + for (int i = 0; i < 100; i++) { // final String expId = createExperimentForSSHHost(airavata); // final String expId = createEchoExperimentForFSD(airavataClient); // final String expId = createMPIExperimentForFSD(airavataClient); // final String expId = createEchoExperimentForStampede(airavataClient); -// final String expId = createEchoExperimentForTrestles(airavataClient); + final String expId = createEchoExperimentForTrestles(airavataClient); // final String expId = createExperimentEchoForLocalHost(airavataClient); // final String expId = createExperimentWRFTrestles(airavataClient); // final String expId = createExperimentForBR2(airavataClient); // final String expId = createExperimentForBR2Amber(airavataClient); // final String expId = createExperimentWRFStampede(airavataClient); // final String expId = createExperimentForStampedeAmber(airavataClient); - final String expId = createExperimentForTrestlesAmber(airavataClient); +// final String expId = createExperimentForTrestlesAmber(airavataClient); // final String expId = createExperimentGROMACSStampede(airavataClient); // final String expId = createExperimentESPRESSOStampede(airavataClient); // final String expId = createExperimentLAMMPSStampede(airavataClient); @@ -115,9 +113,9 @@ public class CreateLaunchExperiment { // final String expId = createExperimentTRINITYStampede(airavataClient); // final String expId = createExperimentAUTODOCKStampede(airavataClient); // this is not working , we need to register AutoDock app on stampede // final String expId = "Ultrascan_ln_eb029947-391a-4ccf-8ace-9bafebe07cc0"; - System.out.println("Experiment ID : " + expId); + System.out.println("Experiment ID : " + expId); // updateExperiment(airavata, expId); - + launchExperiment(airavataClient, expId); } } catch (Exception e) { @@ -173,28 +171,19 @@ public class CreateLaunchExperiment { public static String createEchoExperimentForTrestles(Airavata.Client client) throws TException { try { - List<InputDataObjectType> exInputs = new ArrayList<InputDataObjectType>(); - InputDataObjectType input = new InputDataObjectType(); - input.setName("Echo_Input"); - input.setType(DataType.STRING); - input.setValue("Hello World"); - exInputs.add(input); + List<InputDataObjectType> exInputs = client.getApplicationInputs(echoAppId); + for (InputDataObjectType inputDataObjectType : exInputs) { + if (inputDataObjectType.getName().equalsIgnoreCase("Input_to_Echo")) { + inputDataObjectType.setValue("Hello World"); + } + } + List<OutputDataObjectType> exOut = client.getApplicationOutputs(echoAppId); + + Project project = ProjectModelUtil.createProject("default", "admin", "test project"); + String projectId = client.createProject(project); - List<OutputDataObjectType> exOut = new ArrayList<OutputDataObjectType>(); - OutputDataObjectType output = new OutputDataObjectType(); - output.setName("STDOUT"); - output.setType(DataType.STDOUT); - output.setValue(""); - exOut.add(output); - -// OutputDataObjectType output2 = new OutputDataObjectType(); -// output2.setName("Echoed_Output2"); -// output2.setType(DataType.URI); -// output2.setValue("file:///tmp/test.txt"); -// exOut.add(output2); -// Experiment simpleExperiment = - ExperimentModelUtil.createSimpleExperiment("default", "admin", "echoExperiment", "SimpleEcho2", echoAppId, exInputs); + ExperimentModelUtil.createSimpleExperiment(projectId, "admin", "echoExperiment", "SimpleEcho3", echoAppId, exInputs); simpleExperiment.setExperimentOutputs(exOut); Map<String, String> computeResources = airavataClient.getAvailableAppInterfaceComputeResources(echoAppId); @@ -202,7 +191,7 @@ public class CreateLaunchExperiment { for (String id : computeResources.keySet()) { String resourceName = computeResources.get(id); if (resourceName.equals(trestlesHostName)) { - ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling(id, 1, 1, 1, "normal", 30, 0, 1, "sds128"); + ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling(id, 1, 1, 1, "normal", 30, 0, 1, "TG-STA110014S"); UserConfigurationData userConfigurationData = new UserConfigurationData(); userConfigurationData.setAiravataAutoSchedule(false); userConfigurationData.setOverrideManualScheduledParams(false); @@ -1486,11 +1475,11 @@ public class CreateLaunchExperiment { // } for (InputDataObjectType inputDataObjectType : exInputs) { if (inputDataObjectType.getName().equalsIgnoreCase("Heat_Restart_File")) { - inputDataObjectType.setValue("file://[email protected]:/var/www/experimentData/admin101a290e6330f15a91349159553ae8b6bb1/02_Heat.rst"); + inputDataObjectType.setValue("/Users/chathuri/dev/airavata/source/php/inputs/AMBER_FILES/02_Heat.rst"); } else if (inputDataObjectType.getName().equalsIgnoreCase("Production_Control_File")) { - inputDataObjectType.setValue("file://[email protected]:/var/www/experimentData/admin101a290e6330f15a91349159553ae8b6bb1/03_Prod.in"); + inputDataObjectType.setValue("/Users/chathuri/dev/airavata/source/php/inputs/AMBER_FILES/03_Prod.in"); } else if (inputDataObjectType.getName().equalsIgnoreCase("Parameter_Topology_File")) { - inputDataObjectType.setValue("file://[email protected]:/var/www/experimentData/admin101a290e6330f15a91349159553ae8b6bb1/prmtop"); + inputDataObjectType.setValue("/Users/chathuri/dev/airavata/source/php/inputs/AMBER_FILES/prmtop"); } } @@ -1556,7 +1545,7 @@ public class CreateLaunchExperiment { ExperimentModelUtil.createSimpleExperiment(projectId, "admin", "sshEchoExperiment", "SimpleEchoBR", amberAppId, exInputs); simpleExperiment.setExperimentOutputs(exOut); simpleExperiment.setEnableEmailNotification(true); - simpleExperiment.addToEmailAddresses("[email protected]"); +// simpleExperiment.addToEmailAddresses("[email protected]"); Map<String, String> computeResources = airavataClient.getAvailableAppInterfaceComputeResources(amberAppId); if (computeResources != null && computeResources.size() != 0) { for (String id : computeResources.keySet()) { @@ -1591,8 +1580,8 @@ public class CreateLaunchExperiment { public static void launchExperiment(Airavata.Client client, String expId) throws TException { try { - String tokenId = "5f116091-0ad3-4ab6-9df7-6ac909f21f8b"; -// String tokenId ="aaaaaa"; +// String tokenId = "5f116091-0ad3-4ab6-9df7-6ac909f21f8b"; + String tokenId ="aaaaaa"; client.launchExperiment(expId, tokenId); } catch (ExperimentNotFoundException e) { logger.error("Error occured while launching the experiment...", e.getMessage()); http://git-wip-us.apache.org/repos/asf/airavata/blob/b622683b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/RegisterSampleData.java ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/RegisterSampleData.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/RegisterSampleData.java index 50bf15c..5496cc6 100644 --- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/RegisterSampleData.java +++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/RegisterSampleData.java @@ -78,7 +78,7 @@ public class RegisterSampleData { public void register() throws AiravataClientConnectException, TException { airavataClient = AiravataClientFactory.createAiravataClient(THRIFT_SERVER_HOST, THRIFT_SERVER_PORT); registerLocalhost(); -// registerGatewayProfile(); + registerGatewayProfile(); registerApplicationModules(); registerApplicationDeployments(); registerApplicationInterfaces(); http://git-wip-us.apache.org/repos/asf/airavata/blob/b622683b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java index 3111347..57f7512 100644 --- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java +++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java @@ -466,15 +466,16 @@ public class RegisterSampleApplications { InputDataObjectType input1 = RegisterSampleApplicationsUtils.createAppInput("Input_to_Echo", "", DataType.STRING, null, 1, false, false,false, "A test string to Echo", null); - InputDataObjectType input2 = RegisterSampleApplicationsUtils.createAppInput("Input_to_Echo2", "", - DataType.URI, null, 2, false, false,false, "A sample input remote file", null); - - InputDataObjectType input3 = RegisterSampleApplicationsUtils.createAppInput("Input_to_Echo3", "file:///tmp/test.txt", - DataType.URI, null, 3,false, false, false, "A sample input local file", null); +// InputDataObjectType input2 = RegisterSampleApplicationsUtils.createAppInput("Input_to_Echo2", "", +// DataType.URI, null, 2, false, false,false, "A sample input remote file", null); +// +// InputDataObjectType input3 = RegisterSampleApplicationsUtils.createAppInput("Input_to_Echo3", "file:///tmp/test.txt", +// DataType.URI, null, 3,false, false, false, "A sample input local file", null); List<InputDataObjectType> applicationInputs = new ArrayList<InputDataObjectType>(); - applicationInputs.add(input1); applicationInputs.add(input2); applicationInputs.add(input3); + applicationInputs.add(input1); +// applicationInputs.add(input2); applicationInputs.add(input3); OutputDataObjectType output1 = RegisterSampleApplicationsUtils.createAppOutput("Echoed_Output", "", DataType.STRING, false, false); http://git-wip-us.apache.org/repos/asf/airavata/blob/b622683b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQPublisher.java ---------------------------------------------------------------------- diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQPublisher.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQPublisher.java index 97c0d98..b6e9c98 100644 --- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQPublisher.java +++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQPublisher.java @@ -28,6 +28,7 @@ import org.apache.airavata.common.utils.ThriftUtils; import org.apache.airavata.messaging.core.MessageContext; import org.apache.airavata.messaging.core.MessagingConstants; import org.apache.airavata.messaging.core.Publisher; +import org.apache.airavata.messaging.core.stats.StatCounter; import org.apache.airavata.model.messaging.event.*; import org.apache.thrift.TException; import org.slf4j.Logger; @@ -39,6 +40,7 @@ public class RabbitMQPublisher implements Publisher { private RabbitMQProducer rabbitMQProducer; + StatCounter statCounter = StatCounter.getInstance(); public RabbitMQPublisher() throws Exception { String brokerUrl; @@ -87,6 +89,7 @@ public class RabbitMQPublisher implements Publisher { } byte[] messageBody = ThriftUtils.serializeThriftObject(message); rabbitMQProducer.send(messageBody, routingKey); + statCounter.add(); } catch (TException e) { String msg = "Error while deserializing the object"; log.error(msg, e); http://git-wip-us.apache.org/repos/asf/airavata/blob/b622683b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/stats/StatCounter.java ---------------------------------------------------------------------- diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/stats/StatCounter.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/stats/StatCounter.java new file mode 100644 index 0000000..d6c572d --- /dev/null +++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/stats/StatCounter.java @@ -0,0 +1,69 @@ +package org.apache.airavata.messaging.core.stats; + + +import java.io.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Timer; + +public class StatCounter { + private static StatCounter ourInstance = new StatCounter(); + private long msgCount; + private long period = 10*1000; + private long msgCountForPeriod; + + private long bucketStartTime = 0; + private File file; + private FileOutputStream fos; + private BufferedWriter bw; + + private List<Long> messageContPer10S = new ArrayList<Long>(); + + public static StatCounter getInstance() { + return ourInstance; + } + + public long getMsgCount() { + return msgCount; + } + + public void setMsgCount(long msgCount) { + this.msgCount = msgCount; + } + + public List<Long> getMessageContPer10S() { + return messageContPer10S; + } + + public void setMessageContPer10S(List<Long> messageContPer10S) { + this.messageContPer10S = messageContPer10S; + } + + private StatCounter() { + file = new File("/tmp/results"); + Timer timer = new Timer(); + WriterTask writerTask = new WriterTask(); + writerTask.setFile(file); + timer.scheduleAtFixedRate(writerTask, 0, 60 * 1000); + + } + + public void add () { + if (System.currentTimeMillis() - bucketStartTime < period) { + msgCountForPeriod++; + } else { + messageContPer10S.add(msgCountForPeriod); + bucketStartTime = System.currentTimeMillis(); + msgCountForPeriod = 1; + } + msgCount++; + } + + public long getMsgCountForPeriod() { + return msgCountForPeriod; + } + + public void setMsgCountForPeriod(long msgCountForPeriod) { + this.msgCountForPeriod = msgCountForPeriod; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/b622683b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/stats/WriterTask.java ---------------------------------------------------------------------- diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/stats/WriterTask.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/stats/WriterTask.java new file mode 100644 index 0000000..b154ad7 --- /dev/null +++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/stats/WriterTask.java @@ -0,0 +1,39 @@ +package org.apache.airavata.messaging.core.stats; + +import java.io.*; +import java.util.List; +import java.util.TimerTask; + +/** + * Created by chathuri on 2/16/15. + */ +public class WriterTask extends TimerTask { + + private File file; + private FileOutputStream fos; + private BufferedWriter bw; + + public void setFile(File file) { + this.file = file; + + } + + @Override + public void run() { + try { + System.out.println("########### calling Write Task ############"); + StatCounter statCounter = StatCounter.getInstance(); + List<Long> contPer10S = statCounter.getMessageContPer10S(); + fos = new FileOutputStream(file, false); + bw = new BufferedWriter(new OutputStreamWriter(fos)); + for (int i = 0; i < contPer10S.size(); i++) { + bw.write(String.valueOf(i+1) + " :" + String.valueOf(contPer10S.get(i))); + bw.newLine(); + } + bw.flush(); + } catch (IOException e) { + e.printStackTrace(); + } + + } +}
