Repository: lens Updated Branches: refs/heads/master 42c4dfcb6 -> a736f4697
LENS-1208 : Fix join condition for outer join of multi fact queries for expression attributes Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a736f469 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a736f469 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a736f469 Branch: refs/heads/master Commit: a736f4697ee91fa7949e4e895407dbcbf3d5e62b Parents: 42c4dfc Author: Amareshwari Sriramadasu <amareshw...@gmail.com> Authored: Tue Jul 5 15:04:05 2016 +0530 Committer: Rajat Khandelwal <rajatgupt...@gmail.com> Committed: Tue Jul 5 15:04:05 2016 +0530 ---------------------------------------------------------------------- .../apache/lens/cube/parse/CandidateFact.java | 19 +++++++++-- .../lens/cube/parse/ExpressionResolver.java | 2 ++ .../apache/lens/cube/parse/CubeTestSetup.java | 34 +++++++++++++++++++- .../lens/cube/parse/TestBaseCubeQueries.java | 30 +++++++++++++++-- 4 files changed, 79 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/a736f469/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java index 187d98d..01265a5 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java @@ -180,8 +180,6 @@ public class CandidateFact implements CandidateTable, QueryAST { * @throws LensException */ public void updateASTs(CubeQueryContext cubeql) throws LensException { - Set<String> cubeCols = cubeql.getCube().getAllFieldNames(); - // update select AST with selected fields int currentChild = 0; for (int i = 0; i < cubeql.getSelectAST().getChildCount(); i++) { @@ -189,7 +187,9 @@ public class CandidateFact implements CandidateTable, QueryAST { Set<String> exprCols = HQLParser.getColsInExpr(cubeql.getAliasForTableName(cubeql.getCube()), selectExpr); if (getColumns().containsAll(exprCols)) { selectIndices.add(i); - if (cubeql.getCube().getDimAttributeNames().containsAll(exprCols)) { + if (exprCols.isEmpty() // no direct fact columns + // does not have measure names + || (!containsAny(cubeql.getCube().getMeasureNames(), exprCols))) { dimFieldIndices.add(i); } ASTNode aliasNode = HQLParser.findNodeByPath(selectExpr, Identifier); @@ -220,6 +220,19 @@ public class CandidateFact implements CandidateTable, QueryAST { // push down of having clauses happens just after this call in cubequerycontext } + // The source set contains atleast one column in the colSet + static boolean containsAny(Collection<String> srcSet, Collection<String> colSet) { + if (colSet == null || colSet.isEmpty()) { + return true; + } + for (String column : colSet) { + if (srcSet.contains(column)) { + return true; + } + } + return false; + } + @Override public String getStorageString(String alias) { return StringUtils.join(storageTables, ",") + " " + alias; http://git-wip-us.apache.org/repos/asf/lens/blob/a736f469/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index 898c438..5adea6c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -356,7 +356,9 @@ class ExpressionResolver implements ContextRewriter { ExpressionContext ec = getExpressionContext(expr, alias); if (cTable.getColumns().contains(expr)) { // expression is directly materialized in candidate table + log.debug("{} is directly evaluable in {}", expr, cTable); ec.addDirectlyAvailable(cTable); + return; } for (ExprSpecContext esc : ec.allExprs) { if (esc.getTblAliasToColumns().get(alias) == null) { http://git-wip-us.apache.org/repos/asf/lens/blob/a736f469/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index ea6b4a1..48652f2 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -1302,10 +1302,26 @@ public class CubeTestSetup { factColumns.add(new FieldSchema("dim11", "string", "base dim")); factColumns.add(new FieldSchema("test_time_dim_hour_id", "int", "time id")); - // create cube fact + // create cube fact with materialized expressions client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L, factValidityProperties, storageTables); + factName = "testFact5_BASE"; + factColumns = new ArrayList<>(cubeMeasures.size()); + for (CubeMeasure measure : cubeMeasures) { + factColumns.add(measure.getColumn()); + } + + // add dimensions of the cube + factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); + factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); + factColumns.add(new FieldSchema("dim1", "string", "base dim")); + factColumns.add(new FieldSchema("booleancut", "boolean", "expr dim")); + + // create cube fact + client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 150L, + factValidityProperties, storageTables); + // create fact only with extra measures factName = "testFact2_BASE"; factColumns = new ArrayList<FieldSchema>(); @@ -1345,6 +1361,22 @@ public class CubeTestSetup { client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L, factValidityProperties, storageTables); + // create fact with materialized expression + factName = "testFact6_BASE"; + factColumns = new ArrayList<>(); + factColumns.add(new FieldSchema("msr13", "double", "third measure")); + factColumns.add(new FieldSchema("msr14", "bigint", "fourth measure")); + + // add dimensions of the cube + factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); + factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); + factColumns.add(new FieldSchema("dim1", "string", "base dim")); + factColumns.add(new FieldSchema("booleancut", "boolean", "expr dim")); + + // create cube fact + client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 150L, + factValidityProperties, storageTables); + // create raw fact only with extra measures factName = "testFact2_RAW_BASE"; factColumns = new ArrayList<FieldSchema>(); http://git-wip-us.apache.org/repos/asf/lens/blob/a736f469/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java index d17c18f..d42d494 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java @@ -112,7 +112,8 @@ public class TestBaseCubeQueries extends TestQueryRewrite { * */ boolean columnNotFound = false; - List<String> testTimeDimFactTables = Arrays.asList("testfact3_base", "testfact1_raw_base", "testfact3_raw_base"); + List<String> testTimeDimFactTables = Arrays.asList("testfact3_base", "testfact1_raw_base", "testfact3_raw_base", + "testfact5_base", "testfact6_base"); List<String> factTablesForMeasures = Arrays.asList("testfact_deprecated", "testfact2_raw_base", "testfact2_base"); for (Map.Entry<String, List<CandidateTablePruneCause>> entry : pruneCauses.getDetails().entrySet()) { if (entry.getValue().contains(CandidateTablePruneCause.columnNotFound("test_time_dim"))) { @@ -213,7 +214,7 @@ public class TestBaseCubeQueries extends TestQueryRewrite { String lower = hqlQuery.toLowerCase(); assertTrue( lower.startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, mq2.roundedmsr2 roundedmsr2, mq1.msr12 msr12 from ") - || lower.startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, mq1.roundedmsr2 roundedmsr2, mq2.msr12 msr12" + || lower.startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, mq1.roundedmsr2 roundedmsr2, mq2.msr12 msr12" + " from "), hqlQuery); assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.dim1 <=> mq2.dim1"), @@ -663,6 +664,31 @@ public class TestBaseCubeQueries extends TestQueryRewrite { && hqlQuery.endsWith("mq2 on mq1.booleancut <=> mq2.booleancut"), hqlQuery); } + + @Test + public void testMultiFactQueryWithMaterializedExpressions() throws Exception { + Configuration tconf = new Configuration(conf); + tconf.set(CubeQueryConfUtil.getValidFactTablesKey("basecube"), "testfact5_base,testfact6_base"); + String hqlQuery = + rewrite( + "select booleancut, round(sum(msr2)/1000), msr13 from basecube where " + TWO_DAYS_RANGE, tconf); + String expected1 = + getExpectedQuery(cubeName, "select basecube.booleancut as `booleancut`,max(basecube.msr13) as `msr13` FROM ", + null, " group by basecube.booleancut", getWhereForDailyAndHourly2days(cubeName, "C1_testfact6_base")); + String expected2 = + getExpectedQuery(cubeName, "select basecube.booleancut as `booleancut`," + + " round(sum(basecube.msr2)/1000) as `expr2` FROM ", null, " group by basecube.booleancut", + getWhereForDailyAndHourly2days(cubeName, "C1_testfact5_base")); + compareContains(expected1, hqlQuery); + compareContains(expected2, hqlQuery); + assertTrue(hqlQuery.toLowerCase().startsWith("select coalesce(mq1.booleancut, mq2.booleancut) booleancut, " + + "mq2.expr2 `round((sum(msr2) / 1000))`, mq1.msr13 msr13 from ") + || hqlQuery.toLowerCase().startsWith("select coalesce(mq1.booleancut, mq2.booleancut) booleancut, " + + "mq1.expr2 `round((sum(msr2) / 1000))`, mq2.msr13 msr13 from "), hqlQuery); + assertTrue(hqlQuery.contains("mq1 full outer join ") + && hqlQuery.endsWith("mq2 on mq1.booleancut <=> mq2.booleancut"), + hqlQuery); + } @Test public void testFallbackPartCol() throws Exception { Configuration conf = getConfWithStorages("C1");
