Repository: lens Updated Branches: refs/heads/master e39dec5f8 -> 5a73df0d2
LENS-1525 : Adding start and end times in cube Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c4cad0af Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c4cad0af Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c4cad0af Branch: refs/heads/master Commit: c4cad0af68f180daaf4e0abffb4961aa109ca038 Parents: e39dec5 Author: Rajitha R <[email protected]> Authored: Thu Aug 16 12:19:14 2018 +0530 Committer: Rajitha.R <[email protected]> Committed: Thu Aug 16 12:19:14 2018 +0530 ---------------------------------------------------------------------- lens-api/src/main/resources/cube-0.1.xsd | 15 +++++++ .../lens/cube/error/LensCubeErrorCode.java | 1 + .../org/apache/lens/cube/metadata/Cube.java | 36 ++++++++++++++++ .../lens/cube/metadata/CubeInterface.java | 44 ++++++++++++++++++++ .../apache/lens/cube/metadata/DerivedCube.java | 31 ++++++++++++++ .../lens/cube/metadata/MetastoreConstants.java | 4 ++ .../lens/cube/parse/TimerangeResolver.java | 7 +++- .../apache/lens/cube/metadata/DateFactory.java | 2 + .../lens/cube/parse/TestTimeRangeResolver.java | 11 +++++ .../resources/schema/cubes/base/testcube.xml | 2 + 10 files changed, 152 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-api/src/main/resources/cube-0.1.xsd ---------------------------------------------------------------------- diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd index 4d10d24..65bd6ea 100644 --- a/lens-api/src/main/resources/cube-0.1.xsd +++ b/lens-api/src/main/resources/cube-0.1.xsd @@ -102,6 +102,21 @@ corresponding partition column different in the underlying storage tables. This base cube's property gets propagated to its derived cubes. + 3. cube.absolute.start.time: start time of the cube. For queries that ask for time before this, + this cube throws an exception. Time format can be as you would specify in the time_range_in clause. + i.e. yyyy[-mm[-dd[-hh[:MM[:ss[,SSS]]]]]] + + 4. cube.relative.start.time: Here you can specify cube's relative validity relative to current time. + Useful if you want to specify e.g. this cube is valid for today - 90 days. Can be specified as just + a time difference e.g. "-90 days". Or can be specified in relative syntax. + e.g. now.year or now.day - 6 hour etc. + + 5. cube.absolute.end.time: This is to restrict the end date users can query on this cube. + Format same as absolute start time. + + 6. cube.relative.end.time: You can specify the end date for cube + relative to current date. This can be specified as just a time difference e.g. "+90 days" + The properties that are over-ridable are 1. cube.allfields.queriable - (defaults to true) - If set to true, all the fields (measures, attributes) are http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java index ed076e2..49343a7 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java @@ -40,6 +40,7 @@ public enum LensCubeErrorCode { FROM_AFTER_TO(3015, 0), JOIN_TARGET_NOT_CUBE_TABLE(3016, 0), ORDERBY_ALIAS_CONTAINING_WHITESPACE(3036, 0), + QUERY_OUT_OF_ALLOWED_RANGE(3037, 0), // Error codes different for drivers CANNOT_USE_TIMERANGE_WRITER(3017, 100), NO_DEFAULT_AGGREGATE(3018, 200), http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java index 7c3da2c..1d85fbd 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java @@ -24,6 +24,7 @@ import java.util.*; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.ql.metadata.Table; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -414,4 +415,39 @@ public class Cube extends AbstractBaseTable implements CubeInterface { return partCol; } + @Override + public Date getAbsoluteStartTime() { + return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.CUBE_ABSOLUTE_START_TIME), + false, true); + } + + @Override + public Date getRelativeStartTime() { + return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.CUBE_RELATIVE_START_TIME), + true, true); + } + + @Override + public Date getStartTime() { + return Collections.max(Lists.newArrayList(getRelativeStartTime(), getAbsoluteStartTime())); + } + + @Override + public Date getAbsoluteEndTime() { + return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.CUBE_ABSOLUTE_END_TIME), + false, false); + } + + @Override + public Date getRelativeEndTime() { + return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.CUBE_RELATIVE_END_TIME), + true, false); + } + + @Override + public Date getEndTime() { + return Collections.min(Lists.newArrayList(getRelativeEndTime(), getAbsoluteEndTime())); + } + + } http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java index 6477a6b..3f2b670 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java @@ -18,6 +18,7 @@ */ package org.apache.lens.cube.metadata; +import java.util.Date; import java.util.Set; /** @@ -155,4 +156,47 @@ public interface CubeInterface extends Named { * @return true or false */ boolean allFieldsQueriable(); + + /** + * Absolute start time of the cube + * + * @return Absolute Start time of the cube {@link Date} + */ + Date getAbsoluteStartTime(); + + /** + * Relative start time of the cube + * + * @return Relative Start time of the cube {@link Date} + */ + Date getRelativeStartTime(); + + /** + * Start time of the cube + * + * @return Start time of the cube {@link Date} + */ + Date getStartTime(); + + /** + * Absolute end time of the cube + * + * @return Absolute End time of the cube {@link Date} + */ + Date getAbsoluteEndTime(); + + /** + * Relative End time of the cube + * + * @return Relative end time of the cube {@link Date} + */ + Date getRelativeEndTime(); + + /** + * End time of the cube + * + * @return End time of the cube {@link Date} + */ + Date getEndTime(); + } http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java index 4c73785..2ff4ab1 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java @@ -338,4 +338,35 @@ public class DerivedCube extends AbstractCubeTable implements CubeInterface { // TODO Auto-generated method stub return null; } + + @Override + public Date getAbsoluteStartTime() { + return parent.getAbsoluteStartTime(); + } + + @Override + public Date getRelativeStartTime() { + return parent.getRelativeStartTime(); + } + + @Override + public Date getStartTime() { + return parent.getStartTime(); + } + + @Override + public Date getAbsoluteEndTime() { + return parent.getAbsoluteEndTime(); + } + + @Override + public Date getRelativeEndTime() { + return parent.getRelativeEndTime(); + } + + @Override + public Date getEndTime() { + return parent.getEndTime(); + } + } http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java index 160addb..5bdfea4 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java @@ -39,6 +39,10 @@ public final class MetastoreConstants { public static final String TIMED_DIMENSIONS_LIST_SFX = ".timed.dimensions.list"; public static final String PARENT_CUBE_SFX = ".parent.cube"; public static final String CUBE_ALL_FIELDS_QUERIABLE = "cube.allfields.queriable"; + public static final String CUBE_ABSOLUTE_START_TIME = "cube.absolute.start.time"; + public static final String CUBE_RELATIVE_START_TIME = "cube.relative.start.time"; + public static final String CUBE_ABSOLUTE_END_TIME = "cube.absolute.end.time"; + public static final String CUBE_RELATIVE_END_TIME = "cube.relative.end.time"; // Uber dimension constants public static final String DIMENSION_PFX = "dimension."; http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java index 8e77a82..de5214f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java @@ -45,9 +45,14 @@ class TimerangeResolver implements ContextRewriter { return; } extractTimeRange(cubeql); + CubeInterface cube = cubeql.getCube(); + for(TimeRange timeRange : cubeql.getTimeRanges()) { + if (!(timeRange.getFromDate().after(cube.getStartTime()) && timeRange.getToDate().before(cube.getEndTime()))) { + throw new LensException(LensCubeErrorCode.QUERY_OUT_OF_ALLOWED_RANGE.getLensErrorInfo()); + } + } } - private void extractTimeRange(CubeQueryContext cubeql) throws LensException { // get time range - // Time range should be direct child of where condition http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java index c4404f1..99c5961 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java @@ -173,6 +173,7 @@ public class DateFactory { public static final String ONE_DAY_RANGE_IT; public static final String THIS_YEAR_RANGE; public static final String LAST_YEAR_RANGE; + public static final String PREV_FIVE_TO_NEXT_FIVE_YEAR_RANGE; public static final String TWO_MONTHS_RANGE_UPTO_MONTH; public static final String TWO_MONTHS_RANGE_UPTO_DAYS; public static final String TWO_MONTHS_RANGE_UPTO_HOURS; @@ -224,6 +225,7 @@ public class DateFactory { ONE_DAY_RANGE_IT = getTimeRangeString("it", DAILY, -1, 0, DAILY); THIS_YEAR_RANGE = getTimeRangeString(YEARLY, 0, 1); LAST_YEAR_RANGE = getTimeRangeString(YEARLY, -1, 0); + PREV_FIVE_TO_NEXT_FIVE_YEAR_RANGE = getTimeRangeString(YEARLY, -5, 5); TWO_MONTHS_RANGE_UPTO_MONTH = getTimeRangeString(MONTHLY, -2, 0); TWO_MONTHS_RANGE_UPTO_DAYS = getTimeRangeString(MONTHLY, -2, 0, DAILY); TWO_MONTHS_RANGE_UPTO_HOURS = getTimeRangeString(MONTHLY, -2, 0, HOURLY); http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java index 181608f..632d941 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java @@ -42,6 +42,7 @@ import org.testng.annotations.Test; public class TestTimeRangeResolver extends TestQueryRewrite { private Configuration conf; + Date now = new Date(); @BeforeTest public void setupDriver() throws Exception { @@ -102,6 +103,14 @@ public class TestTimeRangeResolver extends TestQueryRewrite { assertEquals(e.getMessage(), "NO_CANDIDATE_FACT_AVAILABLE[Range not answerable]"); } + @Test + public void testQueryTimeRange() throws LensException { + String query = "select msr2 from " + CubeTestSetup.TEST_CUBE_NAME + " where " + PREV_FIVE_TO_NEXT_FIVE_YEAR_RANGE; + + LensException e = getLensExceptionInRewrite(query, getConf()); + assertEquals(e.getErrorInfo().getErrorName(), "QUERY_OUT_OF_ALLOWED_RANGE"); + } + /** * * @param stoargeName storageName_factName @@ -119,4 +128,6 @@ public class TestTimeRangeResolver extends TestQueryRewrite { } return new ArrayList<>(); } + + } http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-cube/src/test/resources/schema/cubes/base/testcube.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml index 4cafd31..3d517ee 100644 --- a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml +++ b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml @@ -21,6 +21,8 @@ --> <x_base_cube name="testcube" xmlns="uri:lens:cube:0.1"> <properties> + <property name="cube.relative.start.time" value="now.year - 2 years"/> + <property name="cube.relative.end.time" value="now.day + 2 years"/> <property name="cube.timedim.partition.et" value="et"/> <property name="cube.timedim.partition.it" value="it"/> <property name="cube.timedim.partition.d_time" value="dt"/>
