PHOENIX-2299 Added support for current date generation
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ddf24b5a Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ddf24b5a Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ddf24b5a Branch: refs/heads/master Commit: ddf24b5a06de0f9ec2a5cfb0aad8199306906aa4 Parents: 9357017 Author: Karan Singhal <karan24...@gmail.com> Authored: Wed Dec 23 19:10:01 2015 +0530 Committer: Cody Marcel <cmar...@salesforce.com> Committed: Wed Jan 6 10:55:08 2016 -0800 ---------------------------------------------------------------------- .../phoenix/pherf/configuration/Column.java | 10 +++ .../apache/phoenix/pherf/rules/DataValue.java | 11 ++++ .../phoenix/pherf/rules/RulesApplier.java | 21 ++++++- .../phoenix/pherf/ConfigurationParserTest.java | 16 +++++ .../apache/phoenix/pherf/RuleGeneratorTest.java | 66 ++++++++++++++++++++ .../test/resources/datamodel/test_schema.sql | 2 + .../test/resources/scenario/test_scenario.xml | 27 ++++++++ 7 files changed, 151 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf24b5a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Column.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Column.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Column.java index a6b9d26..7c9e180 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Column.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Column.java @@ -33,6 +33,7 @@ public class Column { private boolean userDefined; private List<DataValue> dataValues; private DataTypeMapping type; + private boolean useCurrentDate; public Column() { super(); @@ -44,6 +45,7 @@ public class Column { this.precision = Integer.MIN_VALUE; this.nullChance = Integer.MIN_VALUE; this.userDefined = false; + this.useCurrentDate = false; } public Column(Column column) { @@ -119,6 +121,10 @@ public class Column { this.precision = precision; } + public void setUseCurrentDate(boolean useCurrentDate) { this.useCurrentDate = useCurrentDate; } + + public boolean getUseCurrentDate() { return useCurrentDate; } + /** * Changes fields of this object to match existing fields from the passed Column * null object members are ignored. @@ -167,6 +173,10 @@ public class Column { if (column.dataValues != null) { setDataValues(column.getDataValues()); } + + if(column.getUseCurrentDate()) { + setUseCurrentDate(column.getUseCurrentDate()); + } } public int getNullChance() { http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf24b5a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/DataValue.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/DataValue.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/DataValue.java index 9bfc0dd..8f5506f 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/DataValue.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/DataValue.java @@ -28,6 +28,7 @@ public class DataValue { private String maxValue; private String minValue; private int distribution; + private boolean useCurrentDate; public DataValue() { super(); @@ -37,6 +38,7 @@ public class DataValue { this.type = type; this.value = value; this.distribution = Integer.MIN_VALUE; + this.useCurrentDate = false; } public DataValue(DataValue dataValue) { @@ -44,6 +46,7 @@ public class DataValue { this.setDistribution(dataValue.getDistribution()); this.setMinValue(dataValue.getMinValue()); this.setMaxValue(dataValue.getMaxValue()); + this.setUseCurrentDate(dataValue.getUseCurrentDate()); } public String getValue() { @@ -86,4 +89,12 @@ public class DataValue { public void setMaxValue(String maxValue) { this.maxValue = maxValue; } + + public boolean getUseCurrentDate() { + return useCurrentDate; + } + + public void setUseCurrentDate(boolean useCurrentDate) { + this.useCurrentDate = useCurrentDate; + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf24b5a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java index 68791ce..202c8b5 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java @@ -198,7 +198,7 @@ public class RulesApplier { data = pickDataValueFromList(dataValues); // Check if date has right format or not data.setValue(checkDatePattern(data.getValue())); - } else { + } else if (column.getUseCurrentDate() != true){ int minYear = column.getMinValue(); int maxYear = column.getMaxValue(); Preconditions.checkArgument((minYear > 0) && (maxYear > 0), "min and max values need to be set in configuration"); @@ -207,6 +207,9 @@ public class RulesApplier { data = new DataValue(column.getType(), dt); data.setMaxValue(String.valueOf(minYear)); data.setMinValue(String.valueOf(maxYear)); + } else { + String dt = getCurrentDate(); + data = new DataValue(column.getType(), dt); } break; default: @@ -242,6 +245,12 @@ public class RulesApplier { return fmtr.print(dt); } + public String getCurrentDate() { + DateTimeFormatter fmtr = DateTimeFormat.forPattern(PherfConstants.DEFAULT_DATE_PATTERN); + DateTime dt = new DateTime(PherfConstants.DEFAULT_TIME_ZONE); + return fmtr.print(dt); + } + /** * Given an int chance [0-100] inclusive, this method will return true if a winner is selected, otherwise false. * @@ -261,7 +270,8 @@ public class RulesApplier { int dist = value.getDistribution(); sum += dist; } - Preconditions.checkArgument((sum == 100) || (sum == 0), "Distributions need to add up to 100 or not exist."); + Preconditions.checkArgument((sum == 100) || (sum == 0), + "Distributions need to add up to 100 or not exist."); // Spin the wheel until we get a value. while (generatedDataValue == null) { @@ -293,6 +303,13 @@ public class RulesApplier { return (rndVal.nextInt(100) <= chance) ? retValue : null; } + // Path taken when configuration specifies to use current date + if (valueRule.getUseCurrentDate() == true) { + int chance = (valueRule.getDistribution() == 0) ? 100 : valueRule.getDistribution(); + retValue.setValue(getCurrentDate()); + return (rndVal.nextInt(100) <= chance) ? retValue : null; + } + // Later we can add support fo other data types if needed.Right now, we just do this for dates Preconditions.checkArgument((retValue.getMinValue() != null) || (retValue.getMaxValue() != null), "Both min/maxValue tags must be set if value tag is not used"); Preconditions.checkArgument((retValue.getType() == DataTypeMapping.DATE), "Currently on DATE is supported for ranged random values"); http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf24b5a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java index 6f25fbd..2f08bc0 100644 --- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java +++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java @@ -76,6 +76,7 @@ public class ConfigurationParserTest extends ResultBaseTest { && (dataMappingColumns.get(6).getDataValues().size() > 0)); assertDateValue(dataMappingColumns); + assertCurrentDateValue(dataMappingColumns); // Validate column mappings for (Column column : dataMappingColumns) { @@ -161,6 +162,21 @@ public class ConfigurationParserTest extends ResultBaseTest { fail("We should have found a Rule value that matched."); } + private void assertCurrentDateValue(List<Column> dataMappingColumns) { + for (Column dataMapping : dataMappingColumns) { + if ((dataMapping.getType() == DataTypeMapping.DATE) && (dataMapping.getName() + .equals("PRESENT_DATE"))) { + //First rule should have use current date value set + assertNotNull(dataMapping.getDataValues().get(0).getUseCurrentDate()); + + //Second rule should have use current date value set + assertNotNull(dataMapping.getDataValues().get(1).getUseCurrentDate()); + return; + } + } + fail("We should have found a Rule value that matched."); + } + /* Used for debugging to dump out a simple xml filed based on the bound objects. */ http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf24b5a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java index 37bfd47..659c381 100644 --- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java +++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java @@ -71,6 +71,64 @@ public class RuleGeneratorTest { } } + //Test to check the current date is generated correctly between the timestamps at column level and datavalue level + @Test + public void testCurrentDateGenerator() throws Exception { + XMLConfigParser parser = new XMLConfigParser(matcherScenario); + DataModel model = parser.getDataModels().get(0); + WriteWorkload loader = new WriteWorkload(parser); + RulesApplier rulesApplier = loader.getRulesApplier(); + + // Time before generating the date + String timeStamp1 = rulesApplier.getCurrentDate(); + sleep(2); //sleep for few mili-sec + + for (Column dataMapping : model.getDataMappingColumns()) { + if ((dataMapping.getType() == DataTypeMapping.DATE) + && (dataMapping.getUseCurrentDate() == true)) { + + // Generate the date using rules + DataValue value = rulesApplier.getDataValue(dataMapping); + assertNotNull("Could not retrieve DataValue for random DATE.", value); + assertNotNull("Could not retrieve a value in DataValue for random DATE.", + value.getValue()); + + sleep(2); + // Time after generating the date + String timeStamp2 = rulesApplier.getCurrentDate(); + + // Check that dates are between timestamp1 & timestamp2 + value.setMinValue(timeStamp1); + value.setMaxValue(timeStamp2); + assertDateBetween(value); + } + + // Check at list level + if ((dataMapping.getType() == DataTypeMapping.DATE) + && (dataMapping.getName().equals("PRESENT_DATE"))) { + // Do this 20 times and we should and every time generated data should be between + // timestamps + for (int i = 0; i < 1; i++) { + DataValue value = rulesApplier.getDataValue(dataMapping); + assertNotNull("Could not retrieve DataValue for random DATE.", value); + assertNotNull("Could not retrieve a value in DataValue for random DATE.", + value.getValue()); + + sleep(2); + // Time after generating the date + String timeStamp2 = rulesApplier.getCurrentDate(); + + // Check generated date is between timestamp1 & timestamp2 + value.setMinValue(timeStamp1); + value.setMaxValue(timeStamp2); + assertDateBetween(value); + + } + } + } + + } + @Test public void testNullChance() throws Exception { XMLConfigParser parser = new XMLConfigParser(matcherScenario); @@ -259,4 +317,12 @@ public class RuleGeneratorTest { assertTrue("Value " + dt + " is not after minValue", dt.isAfter(min)); assertTrue("Value " + dt + " is not before maxValue", dt.isBefore(max)); } + + private void sleep(int time) { + try { + Thread.sleep(time); + } catch(InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf24b5a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql b/phoenix-pherf/src/test/resources/datamodel/test_schema.sql index 2cf8e9b..8e0da1e 100644 --- a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql +++ b/phoenix-pherf/src/test/resources/datamodel/test_schema.sql @@ -19,6 +19,8 @@ CREATE TABLE IF NOT EXISTS PHERF.TEST_TABLE ( TENANT_ID CHAR(15) NOT NULL, PARENT_ID CHAR(15) NOT NULL, CREATED_DATE DATE NOT NULL, + NOW_DATE DATE NOT NULL, + PRESENT_DATE NOT NULL, OTHER_ID CHAR(15), FIELD VARCHAR, OLDVAL_STRING VARCHAR, http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf24b5a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml index 3438db4..47aa4a5 100644 --- a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml +++ b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml @@ -45,6 +45,17 @@ <name>GENERAL_DATE</name> </column> <column> + <type>DATE</type> + <!--SEQUENTIAL is unsupported for DATE --> + <dataSequence>RANDOM</dataSequence> + <!-- Number [0-100] that represents the probability of creating a null value --> + <!-- The higher the number, the more like the value will returned will be null --> + <!-- Leaving this tag out is equivalent to having a 0 probability. i.e. never null --> + <nullChance>0</nullChance> + <useCurrentDate>true</useCurrentDate> + <name>NOW_DATE</name> + </column> + <column> <type>DECIMAL</type> <dataSequence>RANDOM</dataSequence> <minValue>0</minValue> @@ -91,6 +102,22 @@ </valuelist> </column> <column> + <type>DATE</type> + <name>PRESENT_DATE</name> + <minValue>1975</minValue> + <maxValue>2025</maxValue> + <valuelist> + <!-- Distributes randomly with equal chance of being picked --> + <datavalue distribution="80"> + <!-- Joda time format: yyyy-MM-dd HH:mm:ss.SSS z --> + <useCurrentDate>true</useCurrentDate> + </datavalue> + <datavalue distribution="20"> + <useCurrentDate>true</useCurrentDate> + </datavalue> + </valuelist> + </column> + <column> <type>CHAR</type> <userDefined>true</userDefined> <dataSequence>LIST</dataSequence>
