http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/action/derived/TimedAction.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/action/derived/TimedAction.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/action/derived/TimedAction.java deleted file mode 100644 index f0e7ad7..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/action/derived/TimedAction.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.action.derived; - -import java.io.OutputStream; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.apache.asterix.experiment.action.base.AbstractAction; -import org.apache.asterix.experiment.action.base.IAction; - -public class TimedAction extends AbstractAction { - - private final Logger LOGGER = Logger.getLogger(TimedAction.class.getName()); - - private final IAction action; - private final OutputStream os; - - public TimedAction(IAction action) { - this.action = action; - os = null; - } - - public TimedAction(IAction action, OutputStream os) { - this.action = action; - this.os = os; - } - - @Override - protected void doPerform() throws Exception { - long start = System.currentTimeMillis(); - action.perform(); - long end = System.currentTimeMillis(); - if (LOGGER.isLoggable(Level.SEVERE)) { - if (os == null) { - System.out.println("Elapsed time = " + (end - start) + " for action " + action); - System.out.flush(); - } else { - os.write(("Elapsed time = " + (end - start) + " for action " + action).getBytes()); - os.flush(); - } - } - } -}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment1Builder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment1Builder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment1Builder.java deleted file mode 100644 index 4461fd7..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment1Builder.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.builder; - -import org.apache.asterix.experiment.action.base.SequentialActionList; -import org.apache.asterix.experiment.action.derived.RunAQLFileAction; -import org.apache.asterix.experiment.client.LSMExperimentConstants; -import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig; - -public abstract class AbstractExperiment1Builder extends AbstractLSMBaseExperimentBuilder { - - public AbstractExperiment1Builder(String name, LSMExperimentSetRunnerConfig config, String clusterConfigFileName, - String ingestFileName, String dgenFileName) { - super(name, config, clusterConfigFileName, ingestFileName, dgenFileName, "count.aql"); - } - - @Override - protected void doBuildDDL(SequentialActionList seq) { - seq.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve( - LSMExperimentConstants.AQL_DIR).resolve("1.aql"))); - } - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment2ABuilder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment2ABuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment2ABuilder.java deleted file mode 100644 index 4dce1af..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment2ABuilder.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.builder; - -import org.apache.asterix.experiment.action.base.SequentialActionList; -import org.apache.asterix.experiment.action.derived.RunAQLFileAction; -import org.apache.asterix.experiment.client.LSMExperimentConstants; -import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig; - -public abstract class AbstractExperiment2ABuilder extends AbstractLSMBaseExperimentBuilder { - - public AbstractExperiment2ABuilder(String name, LSMExperimentSetRunnerConfig config, String clusterConfigFileName, - String ingestFileName, String dgenFileName) { - super(name, config, clusterConfigFileName, ingestFileName, dgenFileName, "count.aql"); - } - - @Override - protected void doBuildDDL(SequentialActionList seq) { - seq.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve( - LSMExperimentConstants.AQL_DIR).resolve("2_a.aql"))); - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment2BBuilder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment2BBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment2BBuilder.java deleted file mode 100644 index 2aeb6f8..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment2BBuilder.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.builder; - -import org.apache.asterix.experiment.action.base.SequentialActionList; -import org.apache.asterix.experiment.action.derived.RunAQLFileAction; -import org.apache.asterix.experiment.client.LSMExperimentConstants; -import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig; - -public abstract class AbstractExperiment2BBuilder extends AbstractLSMBaseExperimentBuilder { - - public AbstractExperiment2BBuilder(String name, LSMExperimentSetRunnerConfig config, String clusterConfigFileName, - String ingestFileName, String dgenFileName) { - super(name, config, clusterConfigFileName, ingestFileName, dgenFileName, "count.aql"); - } - - @Override - protected void doBuildDDL(SequentialActionList seq) { - seq.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve( - LSMExperimentConstants.AQL_DIR).resolve("2_b.aql"))); - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment2CBuilder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment2CBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment2CBuilder.java deleted file mode 100644 index f54c78a..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment2CBuilder.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.builder; - -import org.apache.asterix.experiment.action.base.SequentialActionList; -import org.apache.asterix.experiment.action.derived.RunAQLFileAction; -import org.apache.asterix.experiment.client.LSMExperimentConstants; -import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig; - -public abstract class AbstractExperiment2CBuilder extends AbstractLSMBaseExperimentBuilder { - - public AbstractExperiment2CBuilder(String name, LSMExperimentSetRunnerConfig config, String clusterConfigFileName, - String ingestFileName, String dgenFileName) { - super(name, config, clusterConfigFileName, ingestFileName, dgenFileName, "count.aql"); - } - - @Override - protected void doBuildDDL(SequentialActionList seq) { - seq.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve( - LSMExperimentConstants.AQL_DIR).resolve("2_c.aql"))); - } - - @Override - protected void doPost(SequentialActionList seq) { - seq.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve( - LSMExperimentConstants.AQL_DIR).resolve("avg_gram_tokens.aql"))); - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment2DBuilder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment2DBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment2DBuilder.java deleted file mode 100644 index f54b523..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment2DBuilder.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.builder; - -import org.apache.asterix.experiment.action.base.SequentialActionList; -import org.apache.asterix.experiment.action.derived.RunAQLFileAction; -import org.apache.asterix.experiment.client.LSMExperimentConstants; -import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig; - -public abstract class AbstractExperiment2DBuilder extends AbstractLSMBaseExperimentBuilder { - - public AbstractExperiment2DBuilder(String name, LSMExperimentSetRunnerConfig config, String clusterConfigFileName, - String ingestFileName, String dgenFileName) { - super(name, config, clusterConfigFileName, ingestFileName, dgenFileName, "count.aql"); - } - - @Override - protected void doBuildDDL(SequentialActionList seq) { - seq.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve( - LSMExperimentConstants.AQL_DIR).resolve("2_d.aql"))); - } - - @Override - protected void doPost(SequentialActionList seq) { - seq.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve( - LSMExperimentConstants.AQL_DIR).resolve("avg_keyword_tokens.aql"))); - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment3Builder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment3Builder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment3Builder.java deleted file mode 100644 index f5fb4a4..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment3Builder.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.builder; - -import org.apache.asterix.experiment.action.base.SequentialActionList; -import org.apache.asterix.experiment.action.derived.RunAQLFileAction; -import org.apache.asterix.experiment.client.LSMExperimentConstants; -import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig; - -public abstract class AbstractExperiment3Builder extends AbstractLSMBaseExperimentBuilder { - - public AbstractExperiment3Builder(String name, LSMExperimentSetRunnerConfig config, String clusterConfigFileName, - String ingestFileName, String dgenFileName) { - super(name, config, clusterConfigFileName, ingestFileName, dgenFileName, "count.aql"); - } - - @Override - protected void doBuildDDL(SequentialActionList seq) { - seq.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve( - LSMExperimentConstants.AQL_DIR).resolve("3.aql"))); - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment4Builder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment4Builder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment4Builder.java deleted file mode 100644 index e18975c..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment4Builder.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.builder; - -import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig; - -public abstract class AbstractExperiment4Builder extends AbstractLSMBaseExperimentBuilder { - - public AbstractExperiment4Builder(String name, LSMExperimentSetRunnerConfig config) { - super(name, config, "8node.xml", "base_8_ingest.aql", "8.dgen", "count.aql"); - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment6Builder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment6Builder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment6Builder.java deleted file mode 100644 index a34e089..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment6Builder.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.builder; - -import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig; - -public abstract class AbstractExperiment6Builder extends AbstractLSMBaseExperimentBuilder { - - public AbstractExperiment6Builder(String name, LSMExperimentSetRunnerConfig config) { - super(name, config, "8node.xml", "base_8_ingest.aql", "8.dgen", "count.aql"); - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment7Builder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment7Builder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment7Builder.java deleted file mode 100644 index 4f35ff0..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment7Builder.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.builder; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.Random; - -import org.apache.asterix.experiment.action.base.AbstractAction; -import org.apache.asterix.experiment.action.base.IAction; -import org.apache.asterix.experiment.action.base.ParallelActionSet; -import org.apache.asterix.experiment.action.base.SequentialActionList; -import org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction; -import org.apache.asterix.experiment.action.derived.RunAQLFileAction; -import org.apache.asterix.experiment.action.derived.RunAQLStringAction; -import org.apache.asterix.experiment.action.derived.RunRESTIOWaitAction; -import org.apache.asterix.experiment.action.derived.SleepAction; -import org.apache.asterix.experiment.action.derived.TimedAction; -import org.apache.asterix.experiment.client.LSMExperimentConstants; -import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig; -import org.apache.asterix.experiment.client.OrchestratorServer7; -import org.apache.asterix.experiment.client.OrchestratorServer7.IProtocolActionBuilder; -import org.apache.commons.lang3.StringUtils; - -public abstract class AbstractExperiment7Builder extends AbstractLSMBaseExperimentBuilder { - - private static final long DOMAIN_SIZE = (1L << 32); - - public static final long QUERY_BEGIN_ROUND = 6; - - private static int N_PARTITIONS = 16; - - private final int nIntervals; - - private final String orchHost; - - private final int orchPort; - - protected final long dataInterval; - - protected final int nQueryRuns; - - protected final Random randGen; - - public AbstractExperiment7Builder(String name, LSMExperimentSetRunnerConfig config, String clusterConfigFileName, - String ingestFileName, String dgenFileName) { - super(name, config, clusterConfigFileName, ingestFileName, dgenFileName, null); - nIntervals = config.getNIntervals(); - orchHost = config.getOrchestratorHost(); - orchPort = config.getOrchestratorPort(); - dataInterval = config.getDataInterval(); - this.nQueryRuns = config.getNQueryRuns(); - this.randGen = new Random(); - } - - @Override - protected void doBuildDataGen(SequentialActionList seq, Map<String, List<String>> dgenPairs) throws Exception { - int nDgens = 0; - for (List<String> v : dgenPairs.values()) { - nDgens += v.size(); - } - final OrchestratorServer7 oServer = new OrchestratorServer7(orchPort, nDgens, nIntervals, - new ProtocolActionBuilder(), this.lsAction); - - seq.add(new AbstractAction() { - - @Override - protected void doPerform() throws Exception { - oServer.start(); - } - }); - - ParallelActionSet dgenActions = new ParallelActionSet(); - int partition = 0; - - // run dgen - for (String dgenHost : dgenPairs.keySet()) { - final List<String> rcvrs = dgenPairs.get(dgenHost); - final int p = partition; - dgenActions.add(new AbstractRemoteExecutableAction(dgenHost, username, sshKeyLocation) { - - @Override - protected String getCommand() { - String ipPortPairs = StringUtils.join(rcvrs.iterator(), " "); - String binary = "JAVA_HOME=" + javaHomePath + " " - + localExperimentRoot.resolve("bin").resolve("datagenrunner").toString(); - return StringUtils.join(new String[] { binary, "-si", "" + locationSampleInterval, "-of", - openStreetMapFilePath, "-p", "" + p, "-di", "" + dataInterval, "-ni", "" + nIntervals, - "-oh", orchHost, "-op", "" + orchPort, ipPortPairs }, " "); - } - }); - partition += rcvrs.size(); - } - seq.add(dgenActions); - - // wait until all dgen / queries are done - seq.add(new AbstractAction() { - - @Override - protected void doPerform() throws Exception { - oServer.awaitFinished(); - } - }); - } - - public class ProtocolActionBuilder implements IProtocolActionBuilder { - - private final String rangeQueryTemplate; - - public ProtocolActionBuilder() throws IOException { - this.rangeQueryTemplate = getRangeQueryTemplate(); - } - - private String getRangeQueryTemplate() throws IOException { - Path aqlTemplateFilePath = localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve("8_q2.aql"); - return StandardCharsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(aqlTemplateFilePath))).toString(); - } - - @Override - public IAction buildQueryAction(long cardinality, boolean finalRound) throws IOException { - SequentialActionList protoAction = new SequentialActionList(); - IAction rangeQueryAction = new TimedAction( - new RunAQLStringAction(httpClient, restHost, restPort, getRangeAQL(cardinality, finalRound))); - protoAction.add(rangeQueryAction); - return protoAction; - } - - private String getRangeAQL(long cardinaliry, boolean finalRound) throws IOException { - long round = QUERY_BEGIN_ROUND; - if (finalRound) { - ++round; - } - long numKeys = ((round * dataInterval) / 1000) * N_PARTITIONS; - long rangeSize = (long) ((cardinaliry / (double) numKeys) * DOMAIN_SIZE); - int lowKey = randGen.nextInt(); - long maxLowKey = Integer.MAX_VALUE - rangeSize; - if (lowKey > maxLowKey) { - lowKey = (int) maxLowKey; - } - int highKey = (int) (lowKey + rangeSize); - return rangeQueryTemplate.replaceAll("\\$LKEY\\$", Long.toString(lowKey)).replaceAll("\\$HKEY\\$", - Long.toString(highKey)); - } - - @Override - public IAction buildIOWaitAction() throws IOException { - SequentialActionList ioAction = new SequentialActionList(); - ioAction.add(new SleepAction(10000)); - ioAction.add(new RunRESTIOWaitAction(httpClient, restHost, restPort)); - ioAction.add(new SleepAction(10000)); - return ioAction; - } - - @Override - public IAction buildCompactAction() throws IOException { - return new RunAQLFileAction(httpClient, restHost, restPort, - localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve("8_compact.aql")); - } - } - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment8Builder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment8Builder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment8Builder.java deleted file mode 100644 index 8d4c5ed..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment8Builder.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.builder; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.Random; - -import org.apache.asterix.experiment.action.base.AbstractAction; -import org.apache.asterix.experiment.action.base.ParallelActionSet; -import org.apache.asterix.experiment.action.base.SequentialActionList; -import org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction; -import org.apache.asterix.experiment.action.derived.RunAQLStringAction; -import org.apache.asterix.experiment.action.derived.RunRESTIOWaitAction; -import org.apache.asterix.experiment.action.derived.SleepAction; -import org.apache.asterix.experiment.action.derived.TimedAction; -import org.apache.asterix.experiment.client.LSMExperimentConstants; -import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig; -import org.apache.asterix.experiment.client.OrchestratorServer; -import org.apache.commons.lang3.StringUtils; - -public abstract class AbstractExperiment8Builder extends AbstractLSMBaseExperimentBuilder { - - private static final long DOMAIN_SIZE = (1L << 32); - - private static final long EXPECTED_RANGE_CARDINALITY = 50000; - - private static int N_PARTITIONS = 16; - - private final int nIntervals; - - private final String orchHost; - - private final int orchPort; - - protected final long dataInterval; - - protected final int nQueryRuns; - - protected final Random randGen; - - private final String pointQueryTemplate; - private final String rangeQueryTemplate; - - public AbstractExperiment8Builder(String name, LSMExperimentSetRunnerConfig config, String clusterConfigFileName, - String ingestFileName, String dgenFileName) throws IOException { - super(name, config, clusterConfigFileName, ingestFileName, dgenFileName, null); - nIntervals = config.getNIntervals(); - orchHost = config.getOrchestratorHost(); - orchPort = config.getOrchestratorPort(); - dataInterval = config.getDataInterval(); - this.nQueryRuns = config.getNQueryRuns(); - this.randGen = new Random(); - this.pointQueryTemplate = getPointQueryTemplate(); - this.rangeQueryTemplate = getRangeQueryTemplate(); - } - - protected void doBuildProtocolAction(SequentialActionList seq, int round) throws Exception { - for (int i = 0; i < nQueryRuns; ++i) { - // String aql = getPointLookUpAQL(round); - // seq.add(new TimedAction(new RunAQLStringAction(httpClient, restHost, restPort, aql))); - String aql = getRangeAQL(round); - seq.add(new TimedAction(new RunAQLStringAction(httpClient, restHost, restPort, aql))); - } - } - - @Override - protected void doBuildDataGen(SequentialActionList seq, Map<String, List<String>> dgenPairs) throws Exception { - //start datagen - SequentialActionList[] protocolActions = new SequentialActionList[nIntervals]; - for (int i = 0; i < nIntervals; i++) { - protocolActions[i] = new SequentialActionList(); - protocolActions[i].add(new SleepAction(10000)); - protocolActions[i].add(new RunRESTIOWaitAction(httpClient, restHost, restPort)); - protocolActions[i].add(new SleepAction(10000)); - doBuildProtocolAction(protocolActions[i], i); - } - - int nDgens = 0; - for (List<String> v : dgenPairs.values()) { - nDgens += v.size(); - } - final OrchestratorServer oServer = new OrchestratorServer(orchPort, nDgens, nIntervals, protocolActions); - - seq.add(new AbstractAction() { - - @Override - protected void doPerform() throws Exception { - oServer.start(); - } - }); - - ParallelActionSet dgenActions = new ParallelActionSet(); - int partition = 0; - - // run dgen - for (String dgenHost : dgenPairs.keySet()) { - final List<String> rcvrs = dgenPairs.get(dgenHost); - final int p = partition; - dgenActions.add(new AbstractRemoteExecutableAction(dgenHost, username, sshKeyLocation) { - - @Override - protected String getCommand() { - String ipPortPairs = StringUtils.join(rcvrs.iterator(), " "); - String binary = "JAVA_HOME=" + javaHomePath + " " - + localExperimentRoot.resolve("bin").resolve("datagenrunner").toString(); - return StringUtils.join(new String[] { binary, "-si", "" + locationSampleInterval, "-of", - openStreetMapFilePath, "-p", "" + p, "-di", "" + dataInterval, "-ni", "" + nIntervals, - "-oh", orchHost, "-op", "" + orchPort, ipPortPairs }, " "); - } - }); - partition += rcvrs.size(); - } - seq.add(dgenActions); - - // wait until all dgen / queries are done - seq.add(new AbstractAction() { - - @Override - protected void doPerform() throws Exception { - oServer.awaitFinished(); - } - }); - } - - private String getRangeQueryTemplate() throws IOException { - Path aqlTemplateFilePath = localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve("8_q2.aql"); - return StandardCharsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(aqlTemplateFilePath))).toString(); - } - - private String getPointQueryTemplate() throws IOException { - Path aqlTemplateFilePath = localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve("8_q1.aql"); - return StandardCharsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(aqlTemplateFilePath))).toString(); - } - - protected String getPointLookUpAQL(int round) { - ByteBuffer bb = ByteBuffer.allocate(8); - bb.put((byte) 0); - bb.put((byte) randGen.nextInt(N_PARTITIONS)); - bb.putShort((short) 0); - bb.putInt(randGen.nextInt((int) (((1 + round) * dataInterval) / 1000))); - bb.flip(); - long key = bb.getLong(); - return pointQueryTemplate.replaceAll("\\$KEY\\$", Long.toString(key)); - } - - protected String getRangeAQL(int round) { - long numKeys = (((1 + round) * dataInterval) / 1000) * N_PARTITIONS; - long rangeSize = (long) ((EXPECTED_RANGE_CARDINALITY / (double) numKeys) * DOMAIN_SIZE); - int lowKey = randGen.nextInt(); - long maxLowKey = Integer.MAX_VALUE - rangeSize; - if (lowKey > maxLowKey) { - lowKey = (int) maxLowKey; - } - int highKey = (int) (lowKey + rangeSize); - return rangeQueryTemplate.replaceAll("\\$LKEY\\$", Long.toString(lowKey)).replaceAll("\\$HKEY\\$", - Long.toString(highKey)); - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment9Builder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment9Builder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment9Builder.java deleted file mode 100644 index bd3067d..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment9Builder.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.builder; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.Random; - -import org.apache.asterix.experiment.action.base.AbstractAction; -import org.apache.asterix.experiment.action.base.IAction; -import org.apache.asterix.experiment.action.base.ParallelActionSet; -import org.apache.asterix.experiment.action.base.SequentialActionList; -import org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction; -import org.apache.asterix.experiment.action.derived.RunAQLStringAction; -import org.apache.asterix.experiment.action.derived.TimedAction; -import org.apache.asterix.experiment.client.LSMExperimentConstants; -import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig; -import org.apache.asterix.experiment.client.OrchestratorServer9; -import org.apache.asterix.experiment.client.OrchestratorServer9.IProtocolActionBuilder; -import org.apache.commons.lang3.StringUtils; - -public abstract class AbstractExperiment9Builder extends AbstractLSMBaseExperimentBuilder { - - private static final long DOMAIN_SIZE = (1L << 32); - - private static final long EXPECTED_RANGE_CARDINALITY = 1000; - - private static int N_PARTITIONS = 16; - - private final int nIntervals; - - private final String orchHost; - - private final int orchPort; - - protected final long dataInterval; - - protected final int nQueryRuns; - - protected final Random randGen; - - public AbstractExperiment9Builder(String name, LSMExperimentSetRunnerConfig config, String clusterConfigFileName, - String ingestFileName, String dgenFileName) { - super(name, config, clusterConfigFileName, ingestFileName, dgenFileName, null); - nIntervals = config.getNIntervals(); - orchHost = config.getOrchestratorHost(); - orchPort = config.getOrchestratorPort(); - dataInterval = config.getDataInterval(); - this.nQueryRuns = config.getNQueryRuns(); - this.randGen = new Random(); - } - - @Override - protected void doBuildDataGen(SequentialActionList seq, Map<String, List<String>> dgenPairs) throws IOException { - int nDgens = 0; - for (List<String> v : dgenPairs.values()) { - nDgens += v.size(); - } - final OrchestratorServer9 oServer = new OrchestratorServer9(orchPort, nDgens, nIntervals, - new ProtocolActionBuilder()); - - seq.add(new AbstractAction() { - - @Override - protected void doPerform() throws Exception { - oServer.start(); - } - }); - - ParallelActionSet dgenActions = new ParallelActionSet(); - int partition = 0; - - // run dgen - for (String dgenHost : dgenPairs.keySet()) { - final List<String> rcvrs = dgenPairs.get(dgenHost); - final int p = partition; - dgenActions.add(new AbstractRemoteExecutableAction(dgenHost, username, sshKeyLocation) { - - @Override - protected String getCommand() { - String ipPortPairs = StringUtils.join(rcvrs.iterator(), " "); - String binary = "JAVA_HOME=" + javaHomePath + " " - + localExperimentRoot.resolve("bin").resolve("datagenrunner").toString(); - return StringUtils.join(new String[] { binary, "-si", "" + locationSampleInterval, "-of", - openStreetMapFilePath, "-p", "" + p, "-di", "" + dataInterval, "-ni", "" + nIntervals, - "-oh", orchHost, "-op", "" + orchPort, ipPortPairs }, " "); - } - }); - partition += rcvrs.size(); - } - seq.add(dgenActions); - - // wait until all dgen / queries are done - seq.add(new AbstractAction() { - - @Override - protected void doPerform() throws Exception { - oServer.awaitFinished(); - } - }); - } - - public class ProtocolActionBuilder implements IProtocolActionBuilder { - - private final String pointQueryTemplate; - - private final String rangeQueryTemplate; - - public ProtocolActionBuilder() throws IOException { - this.pointQueryTemplate = getPointQueryTemplate(); - this.rangeQueryTemplate = getRangeQueryTemplate(); - } - - private String getRangeQueryTemplate() throws IOException { - Path aqlTemplateFilePath = localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve("8_q2.aql"); - return StandardCharsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(aqlTemplateFilePath))).toString(); - } - - private String getPointQueryTemplate() throws IOException { - Path aqlTemplateFilePath = localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve("8_q1.aql"); - return StandardCharsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(aqlTemplateFilePath))).toString(); - } - - @Override - public IAction buildAction(int round) throws Exception { - SequentialActionList protoAction = new SequentialActionList(); - IAction pointQueryAction = new TimedAction( - new RunAQLStringAction(httpClient, restHost, restPort, getPointLookUpAQL(round))); - IAction rangeQueryAction = new TimedAction( - new RunAQLStringAction(httpClient, restHost, restPort, getRangeAQL(round))); - protoAction.add(pointQueryAction); - protoAction.add(rangeQueryAction); - return protoAction; - } - - private String getPointLookUpAQL(int round) throws Exception { - ByteBuffer bb = ByteBuffer.allocate(8); - bb.put((byte) 0); - bb.put((byte) randGen.nextInt(N_PARTITIONS)); - bb.putShort((short) 0); - bb.putInt(randGen.nextInt((int) (((1 + round) * dataInterval) / 1000))); - bb.flip(); - long key = bb.getLong(); - return pointQueryTemplate.replaceAll("\\$KEY\\$", Long.toString(key)); - } - - private String getRangeAQL(int round) throws Exception { - long numKeys = (((1 + round) * dataInterval) / 1000) * N_PARTITIONS; - long rangeSize = (long) ((EXPECTED_RANGE_CARDINALITY / (double) numKeys) * DOMAIN_SIZE); - int lowKey = randGen.nextInt(); - long maxLowKey = Integer.MAX_VALUE - rangeSize; - if (lowKey > maxLowKey) { - lowKey = (int) maxLowKey; - } - int highKey = (int) (lowKey + rangeSize); - return rangeQueryTemplate.replaceAll("\\$LKEY\\$", Long.toString(lowKey)).replaceAll("\\$HKEY\\$", - Long.toString(highKey)); - } - - } - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperimentBuilder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperimentBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperimentBuilder.java deleted file mode 100644 index 1717652..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperimentBuilder.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.builder; - -public abstract class AbstractExperimentBuilder { - private final String name; - - protected AbstractExperimentBuilder(String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public final Experiment build() throws Exception { - Experiment e = new Experiment(name); - doBuild(e); - return e; - } - - protected abstract void doBuild(Experiment e) throws Exception; -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractLSMBaseExperimentBuilder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractLSMBaseExperimentBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractLSMBaseExperimentBuilder.java deleted file mode 100644 index 1a57480..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractLSMBaseExperimentBuilder.java +++ /dev/null @@ -1,357 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.builder; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Scanner; -import java.util.Set; - -import javax.xml.bind.JAXBContext; -import javax.xml.bind.Unmarshaller; - -import org.apache.asterix.event.schema.cluster.Cluster; -import org.apache.asterix.experiment.action.base.ParallelActionSet; -import org.apache.asterix.experiment.action.base.SequentialActionList; -import org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction; -import org.apache.asterix.experiment.action.derived.ManagixActions.CreateAsterixManagixAction; -import org.apache.asterix.experiment.action.derived.ManagixActions.DeleteAsterixManagixAction; -import org.apache.asterix.experiment.action.derived.ManagixActions.LogAsterixManagixAction; -import org.apache.asterix.experiment.action.derived.ManagixActions.StopAsterixManagixAction; -import org.apache.asterix.experiment.action.derived.RemoteAsterixDriverKill; -import org.apache.asterix.experiment.action.derived.RunAQLFileAction; -import org.apache.asterix.experiment.action.derived.SleepAction; -import org.apache.asterix.experiment.client.LSMExperimentConstants; -import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig; -import org.apache.commons.lang3.StringUtils; -import org.apache.http.client.HttpClient; -import org.apache.http.impl.client.DefaultHttpClient; - -public abstract class AbstractLSMBaseExperimentBuilder extends AbstractExperimentBuilder { - - private static final String ASTERIX_INSTANCE_NAME = "a1"; - - private final String logDirSuffix; - - protected final HttpClient httpClient; - - protected final String restHost; - - protected final int restPort; - - private final String managixHomePath; - - protected final String javaHomePath; - - protected final Path localExperimentRoot; - - protected final String username; - - protected final String sshKeyLocation; - - private final int duration; - - private final String clusterConfigFileName; - - private final String ingestFileName; - - protected final String dgenFileName; - - private final String countFileName; - - private final String statFile; - - protected final SequentialActionList lsAction; - - protected final String openStreetMapFilePath; - protected final int locationSampleInterval; - - protected final int recordCountPerBatchDuringIngestionOnly; - protected final int recordCountPerBatchDuringQuery; - protected final long dataGenSleepTimeDuringIngestionOnly; - protected final long dataGenSleepTimeDuringQuery; - - public AbstractLSMBaseExperimentBuilder(String name, LSMExperimentSetRunnerConfig config, - String clusterConfigFileName, String ingestFileName, String dgenFileName, String countFileName) { - super(name); - this.logDirSuffix = config.getLogDirSuffix(); - this.httpClient = new DefaultHttpClient(); - this.restHost = config.getRESTHost(); - this.restPort = config.getRESTPort(); - this.managixHomePath = config.getManagixHome(); - this.javaHomePath = config.getJavaHome(); - this.localExperimentRoot = Paths.get(config.getLocalExperimentRoot()); - this.username = config.getUsername(); - this.sshKeyLocation = config.getSSHKeyLocation(); - this.duration = config.getDuration(); - this.clusterConfigFileName = clusterConfigFileName; - this.ingestFileName = ingestFileName; - this.dgenFileName = dgenFileName; - this.countFileName = countFileName; - this.statFile = config.getStatFile(); - this.lsAction = new SequentialActionList(); - this.openStreetMapFilePath = config.getOpenStreetMapFilePath(); - this.locationSampleInterval = config.getLocationSampleInterval(); - recordCountPerBatchDuringIngestionOnly = config.getRecordCountPerBatchDuringIngestionOnly(); - recordCountPerBatchDuringQuery = config.getRecordCountPerBatchDuringQuery(); - dataGenSleepTimeDuringIngestionOnly = config.getDataGenSleepTimeDuringIngestionOnly(); - dataGenSleepTimeDuringQuery = config.getDataGenSleepTimeDuringQuery(); - } - - protected abstract void doBuildDDL(SequentialActionList seq); - - protected void doPost(SequentialActionList seq) { - } - - protected void doBuildDataGen(SequentialActionList seq, final Map<String, List<String>> dgenPairs) throws Exception { - - //start datagen - ParallelActionSet dgenActions = new ParallelActionSet(); - int partition = 0; - for (String dgenHost : dgenPairs.keySet()) { - final List<String> rcvrs = dgenPairs.get(dgenHost); - final int p = partition; - dgenActions.add(new AbstractRemoteExecutableAction(dgenHost, username, sshKeyLocation) { - - @Override - protected String getCommand() { - String ipPortPairs = StringUtils.join(rcvrs.iterator(), " "); - String binary = "JAVA_HOME=" + javaHomePath + " " - + localExperimentRoot.resolve("bin").resolve("datagenrunner").toString(); - if (openStreetMapFilePath == null) { - return StringUtils.join(new String[] { binary, "-rcbi", - "" + recordCountPerBatchDuringIngestionOnly, "-rcbq", - "" + recordCountPerBatchDuringQuery, "-dsti", "" + dataGenSleepTimeDuringIngestionOnly, - "-dstq", "" + dataGenSleepTimeDuringQuery, "-si", "" + locationSampleInterval, "-p", - "" + p, "-d", "" + duration, ipPortPairs }, " "); - } else { - return StringUtils.join(new String[] { binary, "-rcbi", - "" + recordCountPerBatchDuringIngestionOnly, "-rcbq", - "" + recordCountPerBatchDuringQuery, "-dsti", "" + dataGenSleepTimeDuringIngestionOnly, - "-dstq", "" + dataGenSleepTimeDuringQuery, "-si", "" + locationSampleInterval, "-of", - openStreetMapFilePath, "-p", "" + p, "-d", "" + duration, ipPortPairs }, " "); - } - } - }); - partition += rcvrs.size(); - } - seq.add(dgenActions); - } - - @Override - protected void doBuild(Experiment e) throws Exception { - SequentialActionList execs = new SequentialActionList(); - - String clusterConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR) - .resolve(clusterConfigFileName).toString(); - String asterixConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR) - .resolve(LSMExperimentConstants.ASTERIX_CONFIGURATION).toString(); - - //create instance - execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME)); - execs.add(new DeleteAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME)); - execs.add(new SleepAction(30000)); - execs.add(new CreateAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, clusterConfigPath, - asterixConfigPath)); - - //run ddl statements - execs.add(new SleepAction(15000)); - // TODO: implement retry handler - execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve( - LSMExperimentConstants.AQL_DIR).resolve(LSMExperimentConstants.BASE_TYPES))); - doBuildDDL(execs); - execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot - .resolve(LSMExperimentConstants.AQL_DIR).resolve(LSMExperimentConstants.BASE_DIR) - .resolve(ingestFileName))); - - Map<String, List<String>> dgenPairs = readDatagenPairs(localExperimentRoot.resolve( - LSMExperimentConstants.DGEN_DIR).resolve(dgenFileName)); - final Set<String> ncHosts = new HashSet<>(); - for (List<String> ncHostList : dgenPairs.values()) { - for (String ncHost : ncHostList) { - ncHosts.add(ncHost.split(":")[0]); - } - } - - if (statFile != null) { - ParallelActionSet ioCountActions = new ParallelActionSet(); - for (String ncHost : ncHosts) { - ioCountActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) { - - @Override - protected String getCommand() { - String cmd = "screen -d -m sh -c \"sar -b -u 1 > " + statFile + "\""; - return cmd; - } - }); - } - execs.add(ioCountActions); - } - - SequentialActionList postLSAction = new SequentialActionList(); - File file = new File(clusterConfigPath); - JAXBContext ctx = JAXBContext.newInstance(Cluster.class); - Unmarshaller unmarshaller = ctx.createUnmarshaller(); - final Cluster cluster = (Cluster) unmarshaller.unmarshal(file); - String[] storageRoots = cluster.getIodevices().split(","); - for (String ncHost : ncHosts) { - for (final String sRoot : storageRoots) { - lsAction.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) { - @Override - protected String getCommand() { - return "ls -Rl " + sRoot; - } - }); - postLSAction.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) { - @Override - protected String getCommand() { - return "ls -Rl " + sRoot; - } - }); - - } - } - - // main exp - doBuildDataGen(execs, dgenPairs); - - // if (statFile != null) { - // ParallelActionSet ioCountKillActions = new ParallelActionSet(); - // for (String ncHost : ncHosts) { - // ioCountKillActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) { - // - // @Override - // protected String getCommand() { - // String cmd = "screen -X -S `screen -list | grep Detached | awk '{print $1}'` quit"; - // return cmd; - // } - // }); - // } - // execs.add(ioCountKillActions); - // } - - execs.add(new SleepAction(10000)); - if (countFileName != null) { - execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve( - LSMExperimentConstants.AQL_DIR).resolve(countFileName))); - } - - execs.add(postLSAction); - doPost(execs); - ParallelActionSet killCmds = new ParallelActionSet(); - for (String ncHost : ncHosts) { - killCmds.add(new RemoteAsterixDriverKill(ncHost, username, sshKeyLocation)); - } - //killCmds.add(new RemoteAsterixDriverKill(restHost, username, sshKeyLocation)); - execs.add(killCmds); - execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME)); - if (statFile != null) { - ParallelActionSet collectIOActions = new ParallelActionSet(); - for (String ncHost : ncHosts) { - collectIOActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) { - - @Override - protected String getCommand() { - String cmd = "cp " + statFile + " " + cluster.getLogDir(); - return cmd; - } - }); - } - execs.add(collectIOActions); - } - - //collect profile information - // if (ExperimentProfiler.PROFILE_MODE) { - // if (!SpatialIndexProfiler.PROFILE_HOME_DIR.contentEquals(cluster.getLogDir())) { - // ParallelActionSet collectProfileInfo = new ParallelActionSet(); - // for (String ncHost : ncHosts) { - // collectProfileInfo.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) { - // @Override - // protected String getCommand() { - // String cmd = "mv " + SpatialIndexProfiler.PROFILE_HOME_DIR + "*.txt " + cluster.getLogDir(); - // return cmd; - // } - // }); - // } - // execs.add(collectProfileInfo); - // } - // } - - execs.add(new LogAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, localExperimentRoot - .resolve(LSMExperimentConstants.LOG_DIR + "-" + logDirSuffix).resolve(getName()).toString())); - - if (getName().contains("SpatialIndexExperiment2") || getName().contains("SpatialIndexExperiment5")) { - //get query result file - SequentialActionList getQueryResultFileActions = new SequentialActionList(); - final String queryResultFilePath = openStreetMapFilePath.substring(0, - openStreetMapFilePath.lastIndexOf(File.separator)) - + File.separator + "QueryGenResult-*.txt"; - for (final String qgenHost : dgenPairs.keySet()) { - getQueryResultFileActions.add(new AbstractRemoteExecutableAction(restHost, username, sshKeyLocation) { - - @Override - protected String getCommand() { - String cmd = "scp " - + username - + "@" - + qgenHost - + ":" - + queryResultFilePath - + " " - + localExperimentRoot.resolve(LSMExperimentConstants.LOG_DIR + "-" + logDirSuffix) - .resolve(getName()).toString(); - return cmd; - } - }); - } - execs.add(getQueryResultFileActions); - } - - e.addBody(execs); - } - - protected Map<String, List<String>> readDatagenPairs(Path p) throws IOException { - Map<String, List<String>> dgenPairs = new HashMap<>(); - Scanner s = new Scanner(p, StandardCharsets.UTF_8.name()); - try { - while (s.hasNextLine()) { - String line = s.nextLine(); - String[] pair = line.split("\\s+"); - List<String> vals = dgenPairs.get(pair[0]); - if (vals == null) { - vals = new ArrayList<>(); - dgenPairs.put(pair[0], vals); - } - vals.add(pair[1]); - } - } finally { - s.close(); - } - return dgenPairs; - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractLocalExperimentBuilder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractLocalExperimentBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractLocalExperimentBuilder.java deleted file mode 100644 index 315ad9c..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractLocalExperimentBuilder.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.builder; - -import org.apache.asterix.experiment.action.base.SequentialActionList; - -public abstract class AbstractLocalExperimentBuilder extends AbstractExperimentBuilder { - - private final int nNodeControllers; - - protected AbstractLocalExperimentBuilder(String name, int nNodeControllers) { - super(name); - this.nNodeControllers = nNodeControllers; - } - - protected abstract void addPre(SequentialActionList pre); - - protected abstract void addPost(SequentialActionList post); - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractPerfLoadBuilder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractPerfLoadBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractPerfLoadBuilder.java deleted file mode 100644 index 472d610..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractPerfLoadBuilder.java +++ /dev/null @@ -1,251 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.builder; - -import org.apache.asterix.event.schema.cluster.Cluster; -import org.apache.asterix.experiment.action.base.ParallelActionSet; -import org.apache.asterix.experiment.action.base.SequentialActionList; -import org.apache.asterix.experiment.action.derived.*; -import org.apache.asterix.experiment.action.derived.ManagixActions.CreateAsterixManagixAction; -import org.apache.asterix.experiment.action.derived.ManagixActions.DeleteAsterixManagixAction; -import org.apache.asterix.experiment.action.derived.ManagixActions.LogAsterixManagixAction; -import org.apache.asterix.experiment.action.derived.ManagixActions.StopAsterixManagixAction; -import org.apache.asterix.experiment.client.LSMExperimentConstants; -import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig; -import org.apache.asterix.experiment.client.LSMPerfConstants; -import org.apache.http.client.HttpClient; -import org.apache.http.impl.client.DefaultHttpClient; - -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Unmarshaller; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; - -/** - * This class is used to create experiments for spatial index static data evaluation, that is, no ingestion is involved. - * Also, there is no orchestration server involved in this experiment builder. - */ -public abstract class AbstractPerfLoadBuilder extends AbstractExperimentBuilder { - - private static final String ASTERIX_INSTANCE_NAME = "a1"; - - private final String logDirSuffix; - - protected final HttpClient httpClient; - - protected final String restHost; - - protected final int restPort; - - private final String managixHomePath; - - protected final String javaHomePath; - - protected final Path localExperimentRoot; - - protected final String username; - - protected final String sshKeyLocation; - - private final String clusterConfigFileName; - - protected final String dgenFileName; - - private final String countFileName; - - private final String statFile; - - protected final SequentialActionList lsAction; - - protected final String openStreetMapFilePath; - - protected final int locationSampleInterval; - - protected final String loadAQLFilePath; - - protected final String querySQLPPFileName; - - public AbstractPerfLoadBuilder(String name, LSMExperimentSetRunnerConfig config, - String clusterConfigFileName, String dgenFileName, - String countFileName, String loadAQLFileName, String querySQLPPFileName) { - super(name); - this.logDirSuffix = config.getLogDirSuffix(); - this.httpClient = new DefaultHttpClient(); - this.restHost = config.getRESTHost(); - this.restPort = config.getRESTPort(); - this.managixHomePath = config.getManagixHome(); - this.javaHomePath = config.getJavaHome(); - this.localExperimentRoot = Paths.get(config.getLocalExperimentRoot()); - this.username = config.getUsername(); - this.sshKeyLocation = config.getSSHKeyLocation(); - this.clusterConfigFileName = clusterConfigFileName; - this.dgenFileName = dgenFileName; - this.countFileName = countFileName; - this.statFile = config.getStatFile(); - this.lsAction = new SequentialActionList(); - this.openStreetMapFilePath = config.getOpenStreetMapFilePath(); - this.locationSampleInterval = config.getLocationSampleInterval(); - this.loadAQLFilePath = loadAQLFileName; - this.querySQLPPFileName = querySQLPPFileName; - } - - protected abstract void doBuildDDL(SequentialActionList seq); - - @Override - protected void doBuild(Experiment e) throws IOException, JAXBException { - SequentialActionList execs = new SequentialActionList(); - - String clusterConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR) - .resolve(clusterConfigFileName).toString(); - String asterixConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR) - .resolve(LSMExperimentConstants.ASTERIX_CONFIGURATION).toString(); - - //stop/delete/create instance - execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME)); - execs.add(new DeleteAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME)); - execs.add(new SleepAction(30000)); - execs.add(new CreateAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, clusterConfigPath, - asterixConfigPath)); - - //ddl statements - execs.add(new SleepAction(15000)); - // TODO: implement retry handler - execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve( - LSMExperimentConstants.AQL_DIR).resolve(LSMPerfConstants.BASE_TYPES))); - doBuildDDL(execs); - - //prepare io state action in NC node(s) - Map<String, List<String>> dgenPairs = readDatagenPairs(localExperimentRoot.resolve( - LSMExperimentConstants.DGEN_DIR).resolve(dgenFileName)); - final Set<String> ncHosts = new HashSet<>(); - for (List<String> ncHostList : dgenPairs.values()) { - for (String ncHost : ncHostList) { - ncHosts.add(ncHost.split(":")[0]); - } - } - if (statFile != null) { - ParallelActionSet ioCountActions = new ParallelActionSet(); - for (String ncHost : ncHosts) { - ioCountActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) { - - @Override - protected String getCommand() { - String cmd = "screen -d -m sh -c \"sar -b -u 1 > " + statFile + "\""; - return cmd; - } - }); - } - execs.add(ioCountActions); - } - - //prepare post ls action - File file = new File(clusterConfigPath); - JAXBContext ctx = JAXBContext.newInstance(Cluster.class); - Unmarshaller unmarshaller = ctx.createUnmarshaller(); - final Cluster cluster = (Cluster) unmarshaller.unmarshal(file); - String[] storageRoots = cluster.getIodevices().split(","); - - //---------- main experiment body begins ----------- - - //run DDL + Load - execs.add(new TimedAction(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve( - LSMExperimentConstants.AQL_DIR).resolve(loadAQLFilePath)))); - - //execute SQL++ Queries - execs.add(new TimedAction(new RunSQLPPFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve( - LSMExperimentConstants.AQL_DIR).resolve(querySQLPPFileName), - localExperimentRoot.resolve(LSMPerfConstants.RESULT_FILE)))); - - //---------- main experiment body ends ----------- - - //kill io state action - if (statFile != null) { - ParallelActionSet ioCountKillActions = new ParallelActionSet(); - for (String ncHost : ncHosts) { - ioCountKillActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) { - - @Override - protected String getCommand() { - String cmd = "screen -X -S `screen -list | grep Detached | awk '{print $1}'` quit"; - return cmd; - } - }); - } - execs.add(ioCountKillActions); - } - - //total record count - execs.add(new SleepAction(10000)); - if (countFileName != null) { - execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve( - LSMExperimentConstants.AQL_DIR).resolve(countFileName))); - } - - - execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME)); - - //prepare to collect io state by putting the state file into asterix log dir - if (statFile != null) { - ParallelActionSet collectIOActions = new ParallelActionSet(); - for (String ncHost : ncHosts) { - collectIOActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) { - - @Override - protected String getCommand() { - String cmd = "cp " + statFile + " " + cluster.getLogDir(); - return cmd; - } - }); - } - execs.add(collectIOActions); - } - - //collect cc and nc logs - execs.add(new LogAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, localExperimentRoot - .resolve(LSMExperimentConstants.LOG_DIR + "-" + logDirSuffix).resolve(getName()).toString())); - - e.addBody(execs); - } - - protected Map<String, List<String>> readDatagenPairs(Path p) throws IOException { - Map<String, List<String>> dgenPairs = new HashMap<>(); - Scanner s = new Scanner(p, StandardCharsets.UTF_8.name()); - try { - while (s.hasNextLine()) { - String line = s.nextLine(); - String[] pair = line.split("\\s+"); - List<String> vals = dgenPairs.get(pair[0]); - if (vals == null) { - vals = new ArrayList<>(); - dgenPairs.put(pair[0], vals); - } - vals.add(pair[1]); - } - } finally { - s.close(); - } - return dgenPairs; - } -}