Github user codymarcel commented on a diff in the pull request:
https://github.com/apache/phoenix/pull/41#discussion_r26231562
--- Diff:
phoenix-pherf/src/main/java/org/apache/phoenix/pherf/loaddata/DataLoader.java
---
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.loaddata;
+
+import java.math.BigDecimal;
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.phoenix.pherf.result.ResultUtil;
+import org.apache.phoenix.pherf.util.ResourceList;
+import org.apache.phoenix.pherf.util.RowCalculator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.phoenix.pherf.PherfConstants;
+import org.apache.phoenix.pherf.configuration.Column;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.configuration.XMLConfigParser;
+import org.apache.phoenix.pherf.exception.PherfException;
+import org.apache.phoenix.pherf.result.DataLoadThreadTime;
+import org.apache.phoenix.pherf.result.DataLoadTimeSummary;
+import org.apache.phoenix.pherf.rules.DataValue;
+import org.apache.phoenix.pherf.rules.RulesApplier;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+
+public class DataLoader {
+ private static final Logger logger =
LoggerFactory.getLogger(DataLoader.class);
+ private final PhoenixUtil pUtil = new PhoenixUtil();
+ private final XMLConfigParser parser;
+ private final RulesApplier rulesApplier;
+ private final ResultUtil resultUtil;
+ private final ExecutorService pool;
+ private final Properties properties;
+
+ private final int threadPoolSize;
+ private final int batchSize;
+
+ public DataLoader(XMLConfigParser parser) throws Exception {
+ this(new ResourceList().getProperties(), parser);
+ }
+
+ /**
+ * Default the writers to use up all available cores for threads.
+ *
+ * @param parser
+ * @throws Exception
+ */
+ public DataLoader(Properties properties, XMLConfigParser parser)
throws Exception {
+ this.parser = parser;
+ this.properties = properties;
+ this.rulesApplier = new RulesApplier(this.parser);
+ this.resultUtil = new ResultUtil();
+ int size =
Integer.parseInt(properties.getProperty("pherf.default.dataloader.threadpool"));
+ this.threadPoolSize = (size == 0) ?
Runtime.getRuntime().availableProcessors() : size;
+ this.pool = Executors.newFixedThreadPool(this.threadPoolSize);
+ String bSize =
properties.getProperty("pherf.default.dataloader.batchsize");
+ this.batchSize = (bSize == null) ?
PherfConstants.DEFAULT_BATCH_SIZE : Integer.parseInt(bSize);
+ }
+
+ public void execute() throws Exception {
+ try {
+ DataModel model = getParser().getDataModels().get(0);
+ DataLoadTimeSummary dataLoadTimeSummary = new
DataLoadTimeSummary();
+ DataLoadThreadTime dataLoadThreadTime = new
DataLoadThreadTime();
+
+ for (Scenario scenario : getParser().getScenarios()) {
+ List<Future> writeBatches = new ArrayList<Future>();
+ logger.info("\nLoading " + scenario.getRowCount()
+ + " rows for " + scenario.getTableName());
+ long start = System.currentTimeMillis();
+
+ RowCalculator rowCalculator = new
RowCalculator(getThreadPoolSize(), scenario.getRowCount());
+ for (int i = 0; i < getThreadPoolSize(); i++) {
+ List<Column> phxMetaCols = pUtil.getColumnsFromPhoenix(
+ scenario.getSchemaName(),
+ scenario.getTableNameWithoutSchemaName(),
+ pUtil.getConnection());
+ int threadRowCount = rowCalculator.getNext();
+ logger.info("Kick off thread (#" + i + ")for upsert
with (" + threadRowCount + ") rows.");
+ Future<Info> write = upsertData(scenario, phxMetaCols,
+ scenario.getTableName(), threadRowCount,
dataLoadThreadTime);
+ writeBatches.add(write);
+ }
+
+ if (writeBatches.isEmpty()) {
+ throw new PherfException(
+ "Holy shit snacks! Throwing up hands in
disbelief and exiting. Could not write data for some unknown reason.");
+ }
+
+ int sumRows = 0, sumDuration = 0;
+ // Wait for all the batch threads to complete
+ for (Future<Info> write : writeBatches) {
+ Info writeInfo = write.get();
+ sumRows += writeInfo.getRowCount();
+ sumDuration += writeInfo.getDuration();
+ System.out
--- End diff --
Yep
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
