Repository: lens Updated Branches: refs/heads/master ae23e9c97 -> 53c342bd2
LENS-1496 : Look ahead Time Partitions to be made configurable Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/53c342bd Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/53c342bd Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/53c342bd Branch: refs/heads/master Commit: 53c342bd20dc35f75b80da78ac799c48cc2e04b1 Parents: ae23e9c Author: Rajitha R <rajitha....@gmail.com> Authored: Wed Dec 27 13:43:09 2017 +0530 Committer: sushilmohanty <sushilmoha...@apache.org> Committed: Wed Dec 27 13:43:09 2017 +0530 ---------------------------------------------------------------------- .../lens/cube/parse/CubeQueryConfUtil.java | 6 +++ .../lens/cube/parse/StorageCandidate.java | 27 +++++++++--- .../src/main/resources/olap-query-conf.xml | 8 ++++ .../apache/lens/cube/metadata/DateFactory.java | 2 + .../lens/cube/parse/TestCubeRewriter.java | 43 ++++++++++++++++++++ 5 files changed, 80 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/53c342bd/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java index eeaa3af..4768fb8 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java @@ -48,6 +48,7 @@ public final class CubeQueryConfUtil { public static final String PROCESS_TIME_PART_COL = "lens.cube.query.process.time" + ".partition.column"; public static final String COMPLETENESS_CHECK_PART_COL = "lens.cube.query.completeness.check.partition.column"; public static final String LOOK_AHEAD_PT_PARTS_PFX = "lens.cube.query.lookahead.ptparts.forinterval."; + public static final String LOOK_AHEAD_TIME_PARTS_PFX = "lens.cube.query.lookahead.timeparts.forinterval."; public static final String ENABLE_GROUP_BY_TO_SELECT = "lens.cube.query.promote.groupby.toselect"; public static final String ENABLE_SELECT_TO_GROUPBY = "lens.cube.query.promote.select.togroupby"; public static final String ENABLE_ATTRFIELDS_ADD_DISTINCT = "lens.cube.query.enable.attrfields.add.distinct"; @@ -58,6 +59,7 @@ public final class CubeQueryConfUtil { public static final String REPLACE_TIMEDIM_WITH_PART_COL = "lens.cube.query.replace.timedim"; public static final boolean DEFAULT_MULTI_TABLE_SELECT = true; public static final int DEFAULT_LOOK_AHEAD_PT_PARTS = 1; + public static final int DEFAULT_LOOK_AHEAD_TIME_PARTS = 1; public static final boolean DEFAULT_ENABLE_GROUP_BY_TO_SELECT = false; public static final boolean DEFAULT_ENABLE_SELECT_TO_GROUPBY = false; public static final boolean DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL = true; @@ -66,6 +68,10 @@ public final class CubeQueryConfUtil { return LOOK_AHEAD_PT_PARTS_PFX + interval.name().toLowerCase(); } + public static String getLookAheadTimePartsKey(UpdatePeriod interval) { + return LOOK_AHEAD_TIME_PARTS_PFX + interval.name().toLowerCase(); + } + private static String getValidKeyCubePFX(String cubeName) { return VALID_PFX + cubeName.toLowerCase(); } http://git-wip-us.apache.org/repos/asf/lens/blob/53c342bd/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index 3ee66ee..b48be7f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -456,6 +456,10 @@ public class StorageCandidate implements Candidate, CandidateTable { int lookAheadNumParts = getConf() .getInt(CubeQueryConfUtil.getLookAheadPTPartsKey(maxInterval), CubeQueryConfUtil.DEFAULT_LOOK_AHEAD_PT_PARTS); + + int lookAheadNumTimeParts = getConf() + .getInt(CubeQueryConfUtil.getLookAheadTimePartsKey(maxInterval), CubeQueryConfUtil.DEFAULT_LOOK_AHEAD_TIME_PARTS); + TimeRange.Iterable.Iterator iter = TimeRange.iterable(ceilFromDate, floorToDate, maxInterval, 1).iterator(); // add partitions from ceilFrom to floorTo while (iter.hasNext()) { @@ -475,7 +479,7 @@ public class StorageCandidate implements Candidate, CandidateTable { log.debug("part column is process time col"); } else if (updatePeriods.first().equals(maxInterval)) { log.debug("Update period is the least update period"); - } else if ((iter.getNumIters() - iter.getCounter()) > lookAheadNumParts) { + } else if ((iter.getNumIters() - iter.getCounter()) > lookAheadNumTimeParts) { // see if this is the part of the last-n look ahead partitions log.debug("Not a look ahead partition"); } else { @@ -486,16 +490,27 @@ public class StorageCandidate implements Candidate, CandidateTable { // process time are present TimeRange.Iterable.Iterator processTimeIter = TimeRange.iterable(nextDt, lookAheadNumParts, maxInterval, 1) .iterator(); + TimeRange.Iterable.Iterator timeIter = TimeRange.iterable(nextDt, lookAheadNumTimeParts, maxInterval, 1) + .iterator(); while (processTimeIter.hasNext()) { Date pdt = processTimeIter.next(); Date nextPdt = processTimeIter.peekNext(); - FactPartition processTimePartition = new FactPartition(processTimePartCol, pdt, maxInterval, null, - partWhereClauseFormat); - updatePartitionStorage(processTimePartition); - if (processTimePartition.isFound()) { + FactPartition currFactPartition; + boolean allProcessTimePartitionsFound = true; + while (timeIter.hasNext()){ + Date date = timeIter.next(); + currFactPartition = new FactPartition(processTimePartCol, date, maxInterval, null, + partWhereClauseFormat); + updatePartitionStorage(currFactPartition); + if (!currFactPartition.isFound()) { + log.debug("Looked ahead process time partition {} is not found : " + currFactPartition); + allProcessTimePartitionsFound = false; + break; + } + } + if (allProcessTimePartitionsFound) { log.debug("Finer parts not required for look-ahead partition :{}", part); } else { - log.debug("Looked ahead process time partition {} is not found", processTimePartition); TreeSet<UpdatePeriod> newset = new TreeSet<UpdatePeriod>(); newset.addAll(updatePeriods); newset.remove(maxInterval); http://git-wip-us.apache.org/repos/asf/lens/blob/53c342bd/lens-cube/src/main/resources/olap-query-conf.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/resources/olap-query-conf.xml b/lens-cube/src/main/resources/olap-query-conf.xml index b389d6a..d0e5eb4 100644 --- a/lens-cube/src/main/resources/olap-query-conf.xml +++ b/lens-cube/src/main/resources/olap-query-conf.xml @@ -153,6 +153,14 @@ </property> <property> + <name>lens.cube.query.lookahead.timeparts.forinterval.${interval}</name> + <value>1</value> + <description>The number of time partitions for interval specified where look ahead is applied. + Interval can be any Update period. + </description> + </property> + + <property> <name>lens.cube.query.replace.timedim</name> <value>true</value> <description>Tells whether timedim attribute queried in the time range should be replaced with its corresponding http://git-wip-us.apache.org/repos/asf/lens/blob/53c342bd/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java index 75f2abd..c4404f1 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java @@ -168,6 +168,7 @@ public class DateFactory { public static final String TWO_DAYS_RANGE_TTD_BEFORE_4_DAYS; public static final String TWO_DAYS_RANGE_TTD2; public static final String TWO_DAYS_RANGE_TTD2_BEFORE_4_DAYS; + public static final String THREE_DAYS_RANGE_IT; public static final String TWO_DAYS_RANGE_IT; public static final String ONE_DAY_RANGE_IT; public static final String THIS_YEAR_RANGE; @@ -218,6 +219,7 @@ public class DateFactory { TWO_DAYS_RANGE_TTD_BEFORE_4_DAYS = getTimeRangeString("test_time_dim", DAILY, -6, -4, HOURLY); TWO_DAYS_RANGE_TTD2 = getTimeRangeString("test_time_dim2", DAILY, -2, 0, HOURLY); TWO_DAYS_RANGE_TTD2_BEFORE_4_DAYS = getTimeRangeString("test_time_dim2", DAILY, -6, -4, HOURLY); + THREE_DAYS_RANGE_IT = getTimeRangeString("it", DAILY, -3, 0, HOURLY); TWO_DAYS_RANGE_IT = getTimeRangeString("it", DAILY, -2, 0, HOURLY); ONE_DAY_RANGE_IT = getTimeRangeString("it", DAILY, -1, 0, DAILY); THIS_YEAR_RANGE = getTimeRangeString(YEARLY, 0, 1); http://git-wip-us.apache.org/repos/asf/lens/blob/53c342bd/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 9b54957..e83eacb 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -1467,6 +1467,49 @@ public class TestCubeRewriter extends TestQueryRewrite { .getParticipatingPartitions()); // pt does not exist beyond 1 day. So in this test, max look ahead possible is 3 assertEquals(partsQueried, expectedPartsQueried); + + //New test case for testing look ahead time parts + expectedPartsQueried = Sets.newTreeSet(); + conf.setInt(CubeQueryConfUtil.LOOK_AHEAD_TIME_PARTS_PFX + "daily", 2); + + ceilDay = DAILY.getCeilDate(getDateWithOffset(DAILY, -3)); + nextDay = DateUtils.addDays(ceilDay, 1); + nextToNextDay = DateUtils.addDays(nextDay, 1); + + for (TimePartition p : Iterables.concat( + TimePartition.of(HOURLY, getDateWithOffset(DAILY, -3)).rangeUpto(TimePartition.of(HOURLY, ceilDay)), + TimePartition.of(DAILY, ceilDay).rangeUpto(TimePartition.of(DAILY, nextDay)), + TimePartition.of(DAILY, nextDay).rangeUpto(TimePartition.of(DAILY, nextToNextDay)), + TimePartition.of(HOURLY, nextToNextDay).rangeUpto(TimePartition.of(HOURLY, NOW)))) { + FactPartition fp = new FactPartition("it", p, null, storageTables); + expectedPartsQueried.add(fp); + } + for (TimePartition it : TimePartition.of(HOURLY, ceilDay).rangeUpto(TimePartition.of(HOURLY, nextDay))) { + for (TimePartition pt : TimePartition.of(HOURLY, nextDay).rangeUpto(TimePartition.of(HOURLY, nextToNextDay))) { + FactPartition ptPartition = new FactPartition("pt", pt, null, storageTables); + FactPartition itPartition = new FactPartition("it", it, ptPartition, storageTables); + expectedPartsQueried.add(itPartition); + } + } + + ceilDay = DateUtils.addDays(ceilDay, 1); + nextDay = DateUtils.addDays(nextDay, 1); + nextToNextDay = DateUtils.addDays(nextToNextDay, 1); + + for (TimePartition it : TimePartition.of(HOURLY, ceilDay).rangeUpto(TimePartition.of(HOURLY, nextDay))) { + for (TimePartition pt : TimePartition.of(HOURLY, nextDay).rangeUpto(TimePartition.of(HOURLY, nextToNextDay))) { + FactPartition ptPartition = new FactPartition("pt", pt, null, storageTables); + FactPartition itPartition = new FactPartition("it", it, ptPartition, storageTables); + expectedPartsQueried.add(itPartition); + } + } + + ctx = rewriteCtx("select dim1, max(msr3)," + " msr2 from testCube" + " where " + THREE_DAYS_RANGE_IT, + conf); + partsQueried = new TreeSet<>(((StorageCandidate)ctx.getCandidates().iterator().next()) + .getParticipatingPartitions()); + // pt does not exist beyond 1 day. So in this test, max look ahead possible is 3 + assertEquals(partsQueried, expectedPartsQueried); } @Test