This is an automated email from the ASF dual-hosted git repository. gjacoby pushed a commit to branch 4.x-HBase-1.5 in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x-HBase-1.5 by this push: new 38b1018 PHOENIX-4285 Add PHERF.LOG_PER_NROWS constraint to PHERF.properties file 38b1018 is described below commit 38b1018230a627392ac077509d969f1c4d3d2d94 Author: Xinyi <x...@salesforce.com> AuthorDate: Tue Sep 10 16:26:22 2019 -0700 PHOENIX-4285 Add PHERF.LOG_PER_NROWS constraint to PHERF.properties file Signed-off-by: Geoffrey Jacoby <gjac...@apache.org> --- phoenix-pherf/config/pherf.properties | 3 ++ .../main/java/org/apache/phoenix/pherf/Pherf.java | 26 ++++++++++++++ .../org/apache/phoenix/pherf/PherfConstants.java | 1 + .../phoenix/pherf/workload/WriteWorkload.java | 8 ++++- .../java/org/apache/phoenix/pherf/PherfTest.java | 42 ++++++++++++++++++++++ .../org/apache/phoenix/pherf/ResourceTest.java | 1 + 6 files changed, 80 insertions(+), 1 deletion(-) diff --git a/phoenix-pherf/config/pherf.properties b/phoenix-pherf/config/pherf.properties index bfce8bc..9892e2e 100644 --- a/phoenix-pherf/config/pherf.properties +++ b/phoenix-pherf/config/pherf.properties @@ -21,6 +21,9 @@ pherf.default.threadpool=10 # Interval in Ms that the JMX monitors will take a snapshot and dump to log pherf.default.monitorFrequency=30000 +# Default value to display log line after every 'N' row load +pherf.default.log_per_nrows=1000000 + # Default number of writers to use when loading data # 0 - Set the number of writers to use all available cores # 1-N - ANy integer value for the number of threads to use diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java index 05e747a..2572a34 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java @@ -75,6 +75,8 @@ public class Pherf { options.addOption("rowCountOverride", true, "Row count override to use instead of one specified in scenario."); options.addOption("hint", true, "Executes all queries with specified hint. Example SMALL"); + options.addOption("log_per_nrows", true, + "Default value to display log line after every 'N' row load"); options.addOption("diff", false, "Run pherf in verification mode and diff with exported results"); options.addOption("export", false, @@ -143,6 +145,8 @@ public class Pherf { LOGGER.debug("Using Monitor: " + monitor); LOGGER.debug("Monitor Frequency Ms:" + monitorFrequency); + properties.setProperty(PherfConstants.LOG_PER_NROWS_NAME, getLogPerNRow(command)); + preLoadData = command.hasOption("l"); executeQuerySets = command.hasOption("q"); zookeeper = command.getOptionValue("z", "localhost"); @@ -193,6 +197,28 @@ public class Pherf { ResultUtil.setFileSuffix(label); } + private String getLogPerNRow(CommandLine command) { + try { + String logPerNRows = (command.hasOption("log_per_nrows")) ? + command.getOptionValue("log_per_nrows") : + properties.getProperty( + PherfConstants.LOG_PER_NROWS_NAME, + String.valueOf(PherfConstants.LOG_PER_NROWS) + ); + if (Integer.valueOf(logPerNRows) > 0) { + return logPerNRows; + } + } catch (Exception e) { + LOGGER.warn("Invalid Log per N rows value. Phoenix will pick the default value."); + } + + return String.valueOf(PherfConstants.LOG_PER_NROWS); + } + + public Properties getProperties() { + return this.properties; + } + public static void main(String[] args) { try { new Pherf(args).run(); diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java index e7ba056..2a7a9b9 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java @@ -67,6 +67,7 @@ public class PherfConstants { // TODO MOve to properties // log out data load per n rows + public static final String LOG_PER_NROWS_NAME = "pherf.default.log_per_nrows"; public static final int LOG_PER_NROWS = 1000000; public static final String COMBINED_FILE_NAME = "COMBINED"; diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java index 98029c0..a498bfe 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java @@ -266,6 +266,12 @@ public class WriteWorkload implements Workload { long maxDuration = (WriteWorkload.this.writeParams == null) ? Long.MAX_VALUE : WriteWorkload.this.writeParams.getExecutionDurationInMs(); + int logPerNRows = PherfConstants.LOG_PER_NROWS; + String customizedLogPerNRows = connection.getClientInfo(). + getProperty(PherfConstants.LOG_PER_NROWS_NAME); + if (customizedLogPerNRows!= null) { + logPerNRows = Integer.valueOf(customizedLogPerNRows); + } last = start = System.currentTimeMillis(); String sql = buildSql(columns, tableName); stmt = connection.prepareStatement(sql); @@ -298,7 +304,7 @@ public class WriteWorkload implements Workload { + " rows for this thread (" + this.hashCode() + ") in (" + duration + ") Ms"); - if (i % PherfConstants.LOG_PER_NROWS == 0 && i != 0) { + if (i % logPerNRows == 0 && i != 0) { dataLoadThreadTime.add(tableName, Thread.currentThread().getName(), i, System.currentTimeMillis() - logStartTime); diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java index 80ab971..6a24ced 100644 --- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java +++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java @@ -22,6 +22,10 @@ import org.junit.Rule; import org.junit.Test; import org.junit.contrib.java.lang.system.ExpectedSystemExit; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; + public class PherfTest { @Rule public final ExpectedSystemExit exit = ExpectedSystemExit.none(); @@ -40,4 +44,42 @@ public class PherfTest { exit.expectSystemExitWithStatus(1); Pherf.main(args); } + + @Test + public void testDefaultLogPerNRowsArgument() throws Exception { + String[] args = {"-listFiles"}; + assertEquals(Long.valueOf(PherfConstants.LOG_PER_NROWS), + getLogPerNRowsValue(new Pherf(args).getProperties())); + } + + @Test + public void testCustomizedLogPerNRowsArgument() throws Exception { + Long customizedPerNRows = 15l; + String[] args = {"-listFiles", "-log_per_nrows", customizedPerNRows.toString()}; + assertEquals(customizedPerNRows, + getLogPerNRowsValue(new Pherf(args).getProperties())); + } + + @Test + public void testInvalidLogPerNRowsArgument() throws Exception { + Long zero = 0l; + Long negativeOne = -1l; + String invaildNum = "abc"; + + String[] args = {"-listFiles", "-log_per_nrows", zero.toString()}; + assertEquals(Long.valueOf(PherfConstants.LOG_PER_NROWS), + getLogPerNRowsValue(new Pherf(args).getProperties())); + + String[] args2 = {"-listFiles", "-log_per_nrows", negativeOne.toString()}; + assertEquals(Long.valueOf(PherfConstants.LOG_PER_NROWS), + getLogPerNRowsValue(new Pherf(args2).getProperties())); + + String[] args3 = {"-listFiles", "-log_per_nrows", invaildNum}; + assertEquals(Long.valueOf(PherfConstants.LOG_PER_NROWS), + getLogPerNRowsValue(new Pherf(args3).getProperties())); + } + + private Long getLogPerNRowsValue(Properties prop) { + return Long.valueOf(prop.getProperty(PherfConstants.LOG_PER_NROWS_NAME)); + } } diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ResourceTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ResourceTest.java index d8cad48..5920025 100644 --- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ResourceTest.java +++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ResourceTest.java @@ -51,6 +51,7 @@ public class ResourceTest { PherfConstants constants = PherfConstants.create(); assertNotNull(constants.getProperty("pherf.default.dataloader.threadpool")); assertNotNull(constants.getProperty("pherf.default.results.dir")); + assertNotNull(constants.getProperty(PherfConstants.LOG_PER_NROWS_NAME)); } private Collection<Path> assertResources(String pattern, String rootDir, String assertStr) throws Exception {