Repository: lens Updated Branches: refs/heads/master d0b78e81b -> 9e7012ed9
LENS-1431 Data-availability feature fixes Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/419e1904 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/419e1904 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/419e1904 Branch: refs/heads/master Commit: 419e1904ec971ff1c944f7543ea6bfc6607555e7 Parents: 60e1277 Author: Sushrut Meghshyam Ikhar <sushrutikha...@gmail.com> Authored: Tue Jun 6 17:53:27 2017 +0530 Committer: rajub <raju.bairishe...@lazada.com> Committed: Sat Jun 10 13:30:01 2017 +0800 ---------------------------------------------------------------------- .../lens/cube/parse/ExpressionResolver.java | 4 -- .../lens/cube/parse/StorageCandidate.java | 8 +++- .../cube/parse/MockCompletenessChecker.java | 10 ++-- .../lens/cube/parse/TestCubeRewriter.java | 49 +++++++++++++++++++- .../resources/schema/cubes/base/basecube.xml | 5 ++ .../resources/schema/cubes/derived/der1.xml | 1 + .../schema/facts/testfact2_raw_base.xml | 1 + .../schema/facts/virtual/virtualfact.xml | 27 +++++++++++ .../test/resources/schema/facts/virtualfact.xml | 27 ----------- 9 files changed, 95 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/419e1904/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index ea6d5c7..b1654d1 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -556,12 +556,8 @@ class ExpressionResolver implements ContextRewriter { log.info("Removing expression {} as all tables have non reachable fields", esc); iterator.remove(); removedEsc.add(esc); - removed = true; break; } - if (removed) { - continue; - } //remove expressions which are not valid in the timerange queried // If an expression is defined as // ex = a + b // from t1 to t2; http://git-wip-us.apache.org/repos/asf/lens/blob/419e1904/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index 3f5e2e5..7980797 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -651,6 +651,10 @@ public class StorageCandidate implements Candidate, CandidateTable { } private boolean evaluateMeasuresCompleteness(TimeRange timeRange) throws LensException { + if (getCubeMetastoreClient() == null || !getCubeMetastoreClient().isDataCompletenessCheckEnabled()) { + log.info("Skipping availability check for the fact table: {} as dataCompleteness check is not enabled", fact); + return true; + } String factDataCompletenessTag = fact.getDataCompletenessTag(); if (factDataCompletenessTag == null) { log.info("Not checking completeness for the fact table:{} as the dataCompletenessTag is not set", fact); @@ -673,7 +677,8 @@ public class StorageCandidate implements Candidate, CandidateTable { log.info("No Queried measures with the dataCompletenessTag, hence skipping the availability check"); return true; } - boolean isDataComplete = false; + // default completenessTag will be true + boolean isDataComplete = true; DataCompletenessChecker completenessChecker = getCubeMetastoreClient().getCompletenessChecker(); DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); formatter.setTimeZone(TimeZone.getTimeZone("UTC")); @@ -695,6 +700,7 @@ public class StorageCandidate implements Candidate, CandidateTable { String measureorExprFromTag = tagToMeasureOrExprMap.get(tag); dataCompletenessMap.computeIfAbsent(measureorExprFromTag, k -> new HashMap<>()) .put(formatter.format(completenessResult.getKey()), completenessResult.getValue()); + // set completeness to false if availability for measure is below threshold isDataComplete = false; } } http://git-wip-us.apache.org/repos/asf/lens/blob/419e1904/lens-cube/src/test/java/org/apache/lens/cube/parse/MockCompletenessChecker.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/MockCompletenessChecker.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/MockCompletenessChecker.java index 76e81d5..c1936eb 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/MockCompletenessChecker.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/MockCompletenessChecker.java @@ -33,10 +33,12 @@ public class MockCompletenessChecker implements DataCompletenessChecker { Calendar cal = Calendar.getInstance(); cal.setTimeZone(TimeZone.getTimeZone("GMT")); cal.add(Calendar.DATE, -1); - if (factTag.equals("f1")) { - partitionCompleteness.put(cal.getTime(), 80f); - } else { - partitionCompleteness.put(cal.getTime(), 90f); + if (start.before(cal.getTime()) && end.after(cal.getTime())) { + if (factTag.equals("f1")) { + partitionCompleteness.put(cal.getTime(), 80f); + } else { + partitionCompleteness.put(cal.getTime(), 90f); + } } result.put("tag1", partitionCompleteness); return result; http://git-wip-us.apache.org/repos/asf/lens/blob/419e1904/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 65ca861..d8c7335 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -981,7 +981,7 @@ public class TestCubeRewriter extends TestQueryRewrite { conf.setStrings(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL, "dt"); String hqlQuery = rewrite("select SUM(msr9) from basecube where " + TWO_DAYS_RANGE, conf); String expected = getExpectedQuery("basecube", "select sum(basecube.msr9) as `sum(msr9)` FROM ", null, null, - getWhereForHourly2days("basecube", "c1_testfact5_raw_base")); + getWhereForHourly2days("basecube", "c1_testfact5_raw_base")); compareQueries(hqlQuery, expected); } @@ -1007,6 +1007,53 @@ public class TestCubeRewriter extends TestQueryRewrite { } } + /* + * The test is to check that query is rewritten successfully if there is missing entry in + * dataavailability service for the measure's tag + */ + @Test + public void testQueryWithMeasureWithDataCompletenessTagWithDataAvailiability() throws ParseException, + LensException { + NoCandidateFactAvailableException ne; + PruneCauses.BriefAndDetailedError pruneCauses; + String hqlQuery; + String expected; + + Configuration conf = getConf(); + conf.setStrings(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL, "dt"); + + conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, true); + + // 1. data completeness check applicable + ne = getLensExceptionInRewrite("select SUM(msr16) from basecube" + " where " + TWO_DAYS_RANGE, conf); + pruneCauses = ne.getJsonMessage(); + assertEquals(pruneCauses.getBrief().substring(0, 10), INCOMPLETE_PARTITION.errorFormat.substring(0, 10), + pruneCauses.getBrief()); + + // 2. time outside data completeness check but missing partitions + ne = getLensExceptionInRewrite("select SUM(msr16) from basecube where " + TWO_DAYS_RANGE_BEFORE_4_DAYS, conf); + pruneCauses = ne.getJsonMessage(); + assertEquals(pruneCauses.getBrief().substring(0, 10), MISSING_PARTITIONS.errorFormat.substring(0, 10), + pruneCauses.getBrief()); + + + conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false); + + // 3. query allowed on partial data although data incomplete + hqlQuery = rewrite("select SUM(msr16) from basecube" + " where " + TWO_DAYS_RANGE, conf); + expected = getExpectedQuery("basecube", "select sum(basecube.msr16) as `sum(msr16)` FROM ", null, null, + getWhereForHourly2days("basecube", "c1_testfact2_raw_base")); + compareQueries(hqlQuery.toLowerCase(), expected.toLowerCase()); + + // 4. query allowed on partial data with missing partitions but outside data availability window + hqlQuery = rewrite("select SUM(msr16) from basecube" + " where " + TWO_DAYS_RANGE_BEFORE_4_DAYS, conf); + expected = getExpectedQuery("basecube", "select sum(basecube.msr16) as `sum(msr16)` FROM ", null, null, + getWhereForUpdatePeriods("basecube", "c1_testfact2_raw_base", + DateUtils.addHours(getDateWithOffset(UpdatePeriod.DAILY, -6), -1), + getDateWithOffset(UpdatePeriod.DAILY, -4), Sets.newHashSet(UpdatePeriod.HOURLY))); + compareQueries(hqlQuery.toLowerCase(), expected.toLowerCase()); + } + @Test public void testCubeWhereQueryForMonthWithNoPartialData() throws Exception { Configuration conf = getConf(); http://git-wip-us.apache.org/repos/asf/lens/blob/419e1904/lens-cube/src/test/resources/schema/cubes/base/basecube.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/cubes/base/basecube.xml b/lens-cube/src/test/resources/schema/cubes/base/basecube.xml index 708b510..bcea938 100644 --- a/lens-cube/src/test/resources/schema/cubes/base/basecube.xml +++ b/lens-cube/src/test/resources/schema/cubes/base/basecube.xml @@ -90,6 +90,11 @@ </measure> <measure _type="INT" name="segmsr1" default_aggr="SUM" description="segmentation measure 1" display_string="segmentation measure 1"/> + <measure _type="BIGINT" name="msr16" description="sixteenth measure"> + <tags> + <property name="cube.measure.datacompleteness.tag" value="tag1"/> + </tags> + </measure> </measures> <dim_attributes> <dim_attribute _type="string" name="business" display_string="business unit" http://git-wip-us.apache.org/repos/asf/lens/blob/419e1904/lens-cube/src/test/resources/schema/cubes/derived/der1.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/cubes/derived/der1.xml b/lens-cube/src/test/resources/schema/cubes/derived/der1.xml index cf36cc7..01bec03 100644 --- a/lens-cube/src/test/resources/schema/cubes/derived/der1.xml +++ b/lens-cube/src/test/resources/schema/cubes/derived/der1.xml @@ -33,6 +33,7 @@ <measure_name>msr1</measure_name> <measure_name>msr11</measure_name> <measure_name>msr9</measure_name> + <measure_name>msr16</measure_name> </measure_names> <dim_attr_names> <attr_name>dim1</attr_name> http://git-wip-us.apache.org/repos/asf/lens/blob/419e1904/lens-cube/src/test/resources/schema/facts/testfact2_raw_base.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/facts/testfact2_raw_base.xml b/lens-cube/src/test/resources/schema/facts/testfact2_raw_base.xml index 0bab461..69fd8d9 100644 --- a/lens-cube/src/test/resources/schema/facts/testfact2_raw_base.xml +++ b/lens-cube/src/test/resources/schema/facts/testfact2_raw_base.xml @@ -24,6 +24,7 @@ <column name="msr11" _type="int" comment="first measure"/> <column name="msr12" _type="float" comment="second measure"/> <column name="msr9" _type="bigint" comment="ninth measure"/> + <column name="msr16" _type="bigint" comment="sixteenth measure"/> <column name="d_time" _type="timestamp" comment="event time"/> <column name="processing_time" _type="timestamp" comment="processing time"/> <column name="dim1" _type="string" comment="base dim"/> http://git-wip-us.apache.org/repos/asf/lens/blob/419e1904/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml b/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml new file mode 100644 index 0000000..860965f --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml @@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<x_virtual_fact_table source_fact_name="summary1" cube_name="virtualcube" name="virtualfact1" xmlns="uri:lens:cube:0.1" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> + <properties> + <property name="cube.fact.query.where.filter" value=" dim1 = 10 "/> + </properties> +</x_virtual_fact_table> http://git-wip-us.apache.org/repos/asf/lens/blob/419e1904/lens-cube/src/test/resources/schema/facts/virtualfact.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/facts/virtualfact.xml b/lens-cube/src/test/resources/schema/facts/virtualfact.xml deleted file mode 100644 index 860965f..0000000 --- a/lens-cube/src/test/resources/schema/facts/virtualfact.xml +++ /dev/null @@ -1,27 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - ---> -<x_virtual_fact_table source_fact_name="summary1" cube_name="virtualcube" name="virtualfact1" xmlns="uri:lens:cube:0.1" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> - <properties> - <property name="cube.fact.query.where.filter" value=" dim1 = 10 "/> - </properties> -</x_virtual_fact_table>
