Repository: lens Updated Branches: refs/heads/master bee5cfd8d -> 0b3203d04
LENS-946 : Fix filter clauses in multi fact or multi storage queries Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/0b3203d0 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/0b3203d0 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/0b3203d0 Branch: refs/heads/master Commit: 0b3203d047d9738141d153e9f7894fcc6ac64f79 Parents: bee5cfd Author: Amareshwari Sriramadasu <amareshw...@apache.org> Authored: Mon Feb 8 12:00:53 2016 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Mon Feb 8 12:00:53 2016 +0530 ---------------------------------------------------------------------- .../apache/lens/cube/parse/CandidateFact.java | 22 ++----- .../lens/cube/parse/CubeQueryContext.java | 9 ++- .../apache/lens/cube/parse/DefaultQueryAST.java | 2 +- .../cube/parse/DenormalizationResolver.java | 13 +++- .../lens/cube/parse/ExpressionResolver.java | 12 +++- .../parse/SingleFactMultiStorageHQLContext.java | 10 --- .../apache/lens/cube/parse/CubeTestSetup.java | 24 +++++-- .../lens/cube/parse/TestBaseCubeQueries.java | 68 ++++++++++++++++++-- .../lens/cube/parse/TestCubeRewriter.java | 25 ++++++- 9 files changed, 132 insertions(+), 53 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/0b3203d0/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java index 4faebe1..1b0de34 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java @@ -76,12 +76,11 @@ public class CandidateFact implements CandidateTable, QueryAST { @Getter @Setter private Integer limitValue; - private List<TimeRangeNode> timenodes = Lists.newArrayList(); private final List<Integer> selectIndices = Lists.newArrayList(); private final List<Integer> dimFieldIndices = Lists.newArrayList(); private Collection<String> columns; @Getter - private final Map<String, String> storgeWhereClauseMap = new HashMap<>(); + private final Map<String, ASTNode> storgeWhereClauseMap = new HashMap<>(); @Getter private final Map<TimeRange, Map<String, LinkedHashSet<FactPartition>>> rangeToStoragePartMap = new HashMap<>(); @Getter @@ -145,18 +144,6 @@ public class CandidateFact implements CandidateTable, QueryAST { return alias; } - static class TimeRangeNode { - ASTNode timenode; - ASTNode parent; - int childIndex; - - TimeRangeNode(ASTNode timenode, ASTNode parent, int childIndex) { - this.timenode = timenode; - this.parent = parent; - this.childIndex = childIndex; - } - } - void incrementPartsQueried(int incr) { numQueriedParts += incr; } @@ -174,8 +161,8 @@ public class CandidateFact implements CandidateTable, QueryAST { } - public String getWhereClause(String storageTable) { - return getStorgeWhereClauseMap().get(storageTable); + public ASTNode getStorageWhereClause(String storageTable) { + return storgeWhereClauseMap.get(storageTable); } public boolean isExpressionAnswerable(ASTNode node, CubeQueryContext context) throws LensException { @@ -231,7 +218,7 @@ public class CandidateFact implements CandidateTable, QueryAST { private Set<String> getColsInExpr(final CubeQueryContext cubeql, final Set<String> cubeCols, ASTNode expr) throws LensException { - final Set<String> cubeColsInExpr = new HashSet<String>(); + final Set<String> cubeColsInExpr = new HashSet<>(); HQLParser.bft(expr, new ASTNodeVisitor() { @Override public void visit(TreeNode visited) { @@ -346,7 +333,6 @@ public class CandidateFact implements CandidateTable, QueryAST { return null; } - /** * @return the selectIndices */ http://git-wip-us.apache.org/repos/asf/lens/blob/0b3203d0/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java index ebf8875..b8b6db9 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java @@ -875,7 +875,7 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { ASTNode rangeAST = HQLParser.parseExpr(rangeWhere); range.getParent().setChild(range.getChildIndex(), rangeAST); } - fact.getStorgeWhereClauseMap().put(table, getWhereTree()); + fact.getStorgeWhereClauseMap().put(table, HQLParser.parseExpr(getWhereTree())); } } } @@ -905,7 +905,7 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { } // pick dimension tables required during expression expansion for the picked fact and dimensions - Set<Dimension> exprDimensions = new HashSet<Dimension>(); + Set<Dimension> exprDimensions = new HashSet<>(); if (cfacts != null) { for (CandidateFact cfact : cfacts) { Set<Dimension> factExprDimTables = exprCtx.rewriteExprCtx(cfact, dimsToQuery, cfacts.size() > 1 ? cfact : this); @@ -970,6 +970,7 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { whereAST = MultiFactHQLContext.convertHavingToWhere(havingAST, this, cfacts, new DefaultAliasDecider()); } } + hqlContext = createHQLContext(cfacts, dimsToQuery, factDimMap); return hqlContext.toHQL(); } @@ -982,8 +983,10 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { //create single fact with multiple storage context return new SingleFactMultiStorageHQLContext(facts.iterator().next(), dimsToQuery, this, this); } else if (facts.size() == 1 && facts.iterator().next().getStorageTables().size() == 1) { + CandidateFact fact = facts.iterator().next(); // create single fact context - return new SingleFactSingleStorageHQLContext(facts.iterator().next(), dimsToQuery, this, this); + return new SingleFactSingleStorageHQLContext(fact, null, + dimsToQuery, this, DefaultQueryAST.fromCandidateFact(fact, fact.getStorageTables().iterator().next(), this)); } else { return new MultiFactHQLContext(facts, dimsToQuery, factDimMap, this); } http://git-wip-us.apache.org/repos/asf/lens/blob/0b3203d0/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java index 0997f37..a403e36 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java @@ -68,7 +68,7 @@ public class DefaultQueryAST implements QueryAST { public static DefaultQueryAST fromCandidateFact(CandidateFact fact, String storageTable, QueryAST ast) throws LensException { return new DefaultQueryAST(ast.getSelectAST(), - HQLParser.parseExpr(fact.getWhereClause(storageTable.substring(storageTable.indexOf(".") + 1))), + fact.getStorageWhereClause(storageTable.substring(storageTable.indexOf(".") + 1)), ast.getGroupByAST(), ast.getHavingAST(), ast.getJoinAST(), ast.getOrderByAST(), ast.getLimitValue()); } } http://git-wip-us.apache.org/repos/asf/lens/blob/0b3203d0/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java index f2dc2e5..ab1710d 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java @@ -239,12 +239,19 @@ public class DenormalizationResolver implements ContextRewriter { private void replaceReferencedColumns(CandidateFact cfact, boolean replaceFact) throws LensException { QueryAST ast = cubeql; - if (replaceFact - && (tableToRefCols.get(cfact.getName()) != null && !tableToRefCols.get(cfact.getName()).isEmpty())) { + boolean factRefExists = cfact != null && tableToRefCols.get(cfact.getName()) != null && !tableToRefCols.get(cfact + .getName()).isEmpty(); + if (replaceFact && factRefExists) { ast = cfact; } resolveClause(cubeql, ast.getSelectAST()); - resolveClause(cubeql, ast.getWhereAST()); + if (factRefExists) { + for (ASTNode storageWhereClauseAST : cfact.getStorgeWhereClauseMap().values()) { + resolveClause(cubeql, storageWhereClauseAST); + } + } else { + resolveClause(cubeql, ast.getWhereAST()); + } resolveClause(cubeql, ast.getGroupByAST()); resolveClause(cubeql, ast.getHavingAST()); resolveClause(cubeql, cubeql.getOrderByAST()); http://git-wip-us.apache.org/repos/asf/lens/blob/0b3203d0/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index fa81831..04c3c45 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -447,7 +447,7 @@ class ExpressionResolver implements ContextRewriter { } } // Replace picked expressions in all the base trees - replacePickedExpressions(queryAST); + replacePickedExpressions(cfact, queryAST); log.debug("Picked expressions: {}", pickedExpressions); for (Set<PickedExpression> peSet : pickedExpressions.values()) { for (PickedExpression pe : peSet) { @@ -459,10 +459,16 @@ class ExpressionResolver implements ContextRewriter { return exprDims; } - private void replacePickedExpressions(QueryAST queryAST) + private void replacePickedExpressions(CandidateFact cfact, QueryAST queryAST) throws LensException { replaceAST(cubeql, queryAST.getSelectAST()); - replaceAST(cubeql, queryAST.getWhereAST()); + if (cfact != null) { + for (ASTNode storageWhereClauseAST : cfact.getStorgeWhereClauseMap().values()) { + replaceAST(cubeql, storageWhereClauseAST); + } + } else { + replaceAST(cubeql, queryAST.getWhereAST()); + } replaceAST(cubeql, queryAST.getJoinAST()); replaceAST(cubeql, queryAST.getGroupByAST()); // Having AST is resolved by each fact, so that all facts can expand their expressions. http://git-wip-us.apache.org/repos/asf/lens/blob/0b3203d0/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java index 9f16c5a..63cb388 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java @@ -56,7 +56,6 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { this.ast = ast; processSelectAST(); processGroupByAST(); - processWhereAST(); processHavingAST(); processOrderByAST(); processLimit(); @@ -76,15 +75,6 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { } } - private void processWhereAST() throws LensException { - for (String storageTable : fact.getStorgeWhereClauseMap().keySet()) { - ASTNode tree = parseExpr(fact.getStorgeWhereClauseMap().get(storageTable)); - ASTNode replaced = replaceAST(tree); - //TODO: optimize parse/unparse cycle - fact.getStorgeWhereClauseMap().put(storageTable, getString(replaced)); - } - } - private void processHavingAST() throws LensException { if (ast.getHavingAST() != null) { setHaving(getString(processExpression(ast.getHavingAST()))); http://git-wip-us.apache.org/repos/asf/lens/blob/0b3203d0/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 100d7c9..42decc6 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -119,7 +119,7 @@ public class CubeTestSetup { } public static String getExpectedUnionQuery(String cubeName, List<String> storages, StoragePartitionProvider provider, - String outerSelectPart, String outerWhere, String outerPostWhere, String innerQuerySelectPart, + String outerSelectPart, String outerWhere, String outerPostWhere, String innerQuerySelectPart, String innerJoin, String innerWhere, String innerPostWhere) { if (!innerQuerySelectPart.trim().toLowerCase().endsWith("from")) { innerQuerySelectPart += " from "; @@ -132,13 +132,19 @@ public class CubeTestSetup { sb.append(" ("); String sep = ""; for (String storage : storages) { - sb.append(sep).append(getExpectedQuery(cubeName, innerQuerySelectPart + " ", - innerWhere, innerPostWhere, provider.providePartitionsForStorage(storage))); + sb.append(sep).append(getExpectedQuery(cubeName, innerQuerySelectPart + " ", innerJoin, + innerWhere, innerPostWhere, null, provider.providePartitionsForStorage(storage))); sep = " UNION ALL "; } return sb.append(") ").append(cubeName).append(" ").append(outerWhere == null ? "" : outerWhere) .append(" ").append(outerPostWhere == null ? "" : outerPostWhere).toString(); } + public static String getExpectedUnionQuery(String cubeName, List<String> storages, StoragePartitionProvider provider, + String outerSelectPart, String outerWhere, String outerPostWhere, String innerQuerySelectPart, + String innerWhere, String innerPostWhere) { + return getExpectedUnionQuery(cubeName, storages, provider, outerSelectPart, outerWhere, outerPostWhere, + innerQuerySelectPart, null, innerWhere, innerPostWhere); + } public static String getExpectedQuery(String cubeName, String selExpr, String whereExpr, String postWhereExpr, Map<String, String> storageTableToWhereClause) { @@ -206,8 +212,11 @@ public class CubeTestSetup { assertEquals(1, numTabs); for (Map.Entry<String, String> entry : storageTableToWhereClause.entrySet()) { String storageTable = entry.getKey(); - expected.append(selExpr).append(storageTable).append(" ").append(cubeName).append(joinExpr) - .append(" WHERE ").append("("); + expected.append(selExpr).append(storageTable).append(" ").append(cubeName); + if (joinExpr != null) { + expected.append(joinExpr); + } + expected.append(" WHERE ").append("("); if (notLatestConditions != null) { for (String cond : notLatestConditions) { expected.append(cond).append(" AND "); @@ -553,6 +562,8 @@ public class CubeTestSetup { cubeDimensions.add(new BaseDimAttribute(new FieldSchema("test_time_dim_day_id2", "int", "ref dim"))); cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("testDim3id", "string", "direct id to testdim3"), "dim3 refer", "dim3chain", "id", null, null, 0.0)); + cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("cityname", "string", "city name"), + "city name", "cubecity", "name", null, null, 0.0)); List<ChainRefCol> references = new ArrayList<>(); references.add(new ChainRefCol("timedatechain1", "full_date")); references.add(new ChainRefCol("timehourchain1", "full_hour")); @@ -623,7 +634,8 @@ public class CubeTestSetup { "substr(cubestate.name, 5)")); exprs.add(new ExprColumn(new FieldSchema("substrdim2big1", "String", "substr of dim2big1"), "dim2big1 substr", "substr(dim2big1, 5)")); - + exprs.add(new ExprColumn(new FieldSchema("asciicity", "String", "ascii cityname"), "ascii cityname substr", + "ascii(cityname)")); Map<String, String> cubeProperties = new HashMap<String, String>(); cubeProperties.put(MetastoreUtil.getCubeTimedDimensionListKey(TEST_CUBE_NAME), "d_time,pt,it,et,test_time_dim,test_time_dim2"); http://git-wip-us.apache.org/repos/asf/lens/blob/0b3203d0/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java index 8aab777..1d27558 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java @@ -195,6 +195,7 @@ public class TestBaseCubeQueries extends TestQueryRewrite { assertTrue(lower.startsWith("select mq2.roundedmsr2 roundedmsr2, mq1.msr12 msr12 from ") || lower.startsWith("select mq1.roundedmsr2 roundedmsr2, mq2.msr12 msr12 from "), hqlQuery); assertTrue(lower.contains("mq1 full outer join") && lower.endsWith("mq2"), hqlQuery); + assertFalse(lower.contains("mq2 on"), hqlQuery); assertFalse(lower.contains("<=>"), hqlQuery); } @@ -205,8 +206,8 @@ public class TestBaseCubeQueries extends TestQueryRewrite { getExpectedQuery(cubeName, "select basecube.dim1 as `dim1`, sum(basecube.msr12) as `msr12` FROM ", null, " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); String expected2 = getExpectedQuery(cubeName, - "select basecube.dim1 as `dim1`, round(sum(basecube.msr2)/1000) as `roundedmsr2` FROM ", null, - " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); + "select basecube.dim1 as `dim1`, round(sum(basecube.msr2)/1000) as `roundedmsr2` FROM ", null, + " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); String lower = hqlQuery.toLowerCase(); @@ -359,7 +360,7 @@ public class TestBaseCubeQueries extends TestQueryRewrite { @Test public void testMultiFactQueryWithNoAggregates() throws Exception { // no aggregates in the query - String hqlQuery = rewrite("select dim1, msr11, roundedmsr2 from basecube" + " where " + TWO_DAYS_RANGE, conf); + String hqlQuery = rewrite("select dim1, msr11, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, conf); String expected1 = getExpectedQuery(cubeName, "select basecube.dim1 as `dim1`, basecube.msr11 as `msr11` FROM ", null, null, getWhereForHourly2days(cubeName, "C1_testfact2_raw_base")); @@ -381,7 +382,7 @@ public class TestBaseCubeQueries extends TestQueryRewrite { public void testMultiFactQueryWithColumnAliases() throws Exception { // query with aliases passed String hqlQuery = - rewrite("select dim1 d1, msr12 `my msr12`, roundedmsr2 m2 from basecube" + " where " + TWO_DAYS_RANGE, conf); + rewrite("select dim1 d1, msr12 `my msr12`, roundedmsr2 m2 from basecube where " + TWO_DAYS_RANGE, conf); String expected1 = getExpectedQuery(cubeName, "select basecube.dim1 as `d1`, sum(basecube.msr12) as `expr2` FROM ", null, " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); @@ -401,7 +402,7 @@ public class TestBaseCubeQueries extends TestQueryRewrite { @Test public void testMultiFactQueryWithNoDefaultAggregates() throws Exception { // query with non default aggregate - String hqlQuery = rewrite("select dim1, avg(msr12), avg(msr2) from basecube" + " where " + TWO_DAYS_RANGE, conf); + String hqlQuery = rewrite("select dim1, avg(msr12), avg(msr2) from basecube where " + TWO_DAYS_RANGE, conf); String expected1 = getExpectedQuery(cubeName, "select basecube.dim1 as `dim1`, avg(basecube.msr12) as `msr12` FROM ", null, " group by basecube.dim1", getWhereForHourly2days(cubeName, "C1_testfact2_raw_base")); @@ -444,7 +445,7 @@ public class TestBaseCubeQueries extends TestQueryRewrite { @Test public void testMultiFactQueryWithDenormColumn() throws Exception { // query with denorm variable - String hqlQuery = rewrite("select dim2, msr13, roundedmsr2 from basecube" + " where " + TWO_DAYS_RANGE, conf); + String hqlQuery = rewrite("select dim2, msr13, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, conf); String expected1 = getExpectedQuery(cubeName, "select dim2chain.id as `dim2`, max(basecube.msr13) as `msr13` FROM ", " JOIN " + getDbName() + "c1_testdim2tbl dim2chain ON basecube.dim12 = " + " dim2chain.id and (dim2chain.dt = 'latest') ", null, " group by dim2chain.id", null, @@ -463,12 +464,33 @@ public class TestBaseCubeQueries extends TestQueryRewrite { } @Test + public void testMultiFactQueryWithDenormColumnInWhere() throws Exception { + // query with denorm variable + String hqlQuery = rewrite("select dim2, msr13, roundedmsr2 from basecube where dim2 == 10 and " + TWO_DAYS_RANGE, + conf); + String expected1 = getExpectedQuery(cubeName, "select dim2chain.id as `dim2`, max(basecube.msr13) as `msr13` FROM ", + " JOIN " + getDbName() + "c1_testdim2tbl dim2chain ON basecube.dim12 = " + + " dim2chain.id and (dim2chain.dt = 'latest') ", "dim2chain.id == 10", " group by dim2chain.id", null, + getWhereForHourly2days(cubeName, "C1_testFact3_RAW_BASE")); + String expected2 = getExpectedQuery(cubeName, + "select basecube.dim2 as `dim2`, round(sum(basecube.msr2)/1000) as `roundedmsr2` FROM ", "basecube.dim2 == 10", + " group by basecube.dim2", getWhereForHourly2days(cubeName, "C1_testfact1_raw_base")); + compareContains(expected1, hqlQuery); + compareContains(expected2, hqlQuery); + assertTrue(hqlQuery.toLowerCase().startsWith( + "select coalesce(mq1.dim2, mq2.dim2) dim2, mq2.msr13 msr13, mq1.roundedmsr2 roundedmsr2 from ") + || hqlQuery.toLowerCase().startsWith( + "select coalesce(mq1.dim2, mq2.dim2) dim2, mq1.msr13 msr13, mq2.roundedmsr2 roundedmsr2 from "), hqlQuery); + assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.dim2 <=> mq2.dim2"), + hqlQuery); + } + @Test public void testMultiFactQueryWithExpressionInvolvingDenormVariable() throws Exception { // query with expression // The expression to be answered from denorm columns String hqlQuery = rewrite( - "select booleancut, round(sum(msr2)/1000), avg(msr13 + msr14) from basecube" + " where " + TWO_DAYS_RANGE, + "select booleancut, round(sum(msr2)/1000), avg(msr13 + msr14) from basecube where " + TWO_DAYS_RANGE, conf); String expected1 = getExpectedQuery(cubeName, "select basecube.dim1 != 'x' AND dim2chain.id != 10 as `booleancut`," @@ -495,6 +517,38 @@ public class TestBaseCubeQueries extends TestQueryRewrite { } @Test + public void testMultiFactQueryWithExpressionInvolvingDenormVariableInWhereClause() throws Exception { + // query with expression + // The expression to be answered from denorm columns + String hqlQuery = + rewrite( + "select booleancut, round(sum(msr2)/1000), avg(msr13 + msr14) from basecube where booleancut == 'true' and " + + TWO_DAYS_RANGE, conf); + String expected1 = + getExpectedQuery(cubeName, "select basecube.dim1 != 'x' AND dim2chain.id != 10 as `booleancut`," + + " avg(basecube.msr13 + basecube.msr14) as `expr3` FROM ", " JOIN " + getDbName() + + "c1_testdim2tbl dim2chain ON basecube.dim12 = " + " dim2chain.id and (dim2chain.dt = 'latest') ", + "(basecube.dim1 != 'x' AND dim2chain.id != 10) == true", + " group by basecube.dim1 != 'x' AND dim2chain.id != 10", null, + getWhereForHourly2days(cubeName, "C1_testfact3_raw_base")); + String expected2 = + getExpectedQuery(cubeName, "select basecube.dim1 != 'x' AND basecube.dim2 != 10 as `booleancut`," + + " round(sum(basecube.msr2)/1000) as `msr2` FROM ", "(basecube.dim1 != 'x' AND basecube.dim2 != 10) == true", + " group by basecube.dim1 != 'x' AND basecube.dim2 != 10", + getWhereForHourly2days(cubeName, "C1_testfact1_raw_base")); + compareContains(expected1, hqlQuery); + compareContains(expected2, hqlQuery); + assertTrue(hqlQuery.toLowerCase() + .startsWith("select coalesce(mq1.booleancut, mq2.booleancut) booleancut, mq2.msr2 msr2," + + " mq1.expr3 expr3 from ") + || hqlQuery.toLowerCase() + .startsWith("select coalesce(mq1.booleancut, mq2.booleancut) booleancut, mq1.msr2 msr2," + + " mq2.expr3 expr3 from "), hqlQuery); + assertTrue(hqlQuery.contains("mq1 full outer join ") + && hqlQuery.endsWith("mq2 on mq1.booleancut <=> mq2.booleancut"), + hqlQuery); + } + @Test public void testFallbackPartCol() throws Exception { Configuration conf = getConfWithStorages("C1"); conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false); http://git-wip-us.apache.org/repos/asf/lens/blob/0b3203d0/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index e569fb1..45b4334 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -396,6 +396,7 @@ public class TestCubeRewriter extends TestQueryRewrite { conf.set(getValidUpdatePeriodsKey("testfact", "C1"), "DAILY,HOURLY"); conf.set(getValidUpdatePeriodsKey("testfact2", "C1"), "YEARLY"); conf.set(getValidUpdatePeriodsKey("testfact", "C2"), "MONTHLY,DAILY"); + conf.set(CubeQueryConfUtil.DISABLE_AUTO_JOINS, "false"); ArrayList<String> storages = Lists.newArrayList("c1_testfact", "c2_testfact"); try { getStorageToUpdatePeriodMap().put("c1_testfact", Lists.newArrayList(HOURLY, DAILY)); @@ -419,6 +420,26 @@ public class TestCubeRewriter extends TestQueryRewrite { } conf.setBoolean(CubeQueryConfUtil.ENABLE_STORAGES_UNION, true); + hqlQuery = rewrite("select ascii(cityname) as `City Name`, msr8, msr7 as `Third measure` " + + "from testCube where ascii(cityname) = 'c' and cityname = 'a' and zipcode = 'b' and " + + TWO_MONTHS_RANGE_UPTO_HOURS, conf); + expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider, + "SELECT testcube.alias0 as `City Name`, sum(testcube.alias1) + max(testcube.alias2), " + + "case when sum(testcube.alias1) = 0 then 0 else sum(testcube.alias3)/sum(testcube.alias1) end " + + "as `Third Measure`", + null, "group by testcube.alias0", + "select ascii(cubecity.name) as `alias0`, sum(testcube.msr2) as `alias1`, " + + "max(testcube.msr3) as `alias2`, " + + "sum(case when testcube.cityid = 'x' then testcube.msr21 else testcube.msr22 end) as `alias3`", " join " + + getDbName() + "c1_citytable cubecity on testcube.cityid = cubecity.id and (cubecity.dt = 'latest')", + "ascii(cubecity.name) = 'c' and cubecity.name = 'a' and testcube.zipcode = 'b'", + "group by ascii(cubecity.name))"); + compareQueries(hqlQuery, expected); + hqlQuery = rewrite("select asciicity as `City Name`, msr8, msr7 as `Third measure` " + + "from testCube where asciicity = 'c' and cityname = 'a' and zipcode = 'b' and " + + TWO_MONTHS_RANGE_UPTO_HOURS, conf); + compareQueries(hqlQuery, expected); + hqlQuery = rewrite("select ascii(cityid) as `City ID`, msr8, msr7 as `Third measure` " + "from testCube where ascii(cityid) = 'c' and cityid = 'a' and zipcode = 'b' and " + TWO_MONTHS_RANGE_UPTO_HOURS, conf); @@ -431,7 +452,7 @@ public class TestCubeRewriter extends TestQueryRewrite { "select ascii(testcube.cityid) as `alias0`, sum(testcube.msr2) as `alias1`, " + "max(testcube.msr3) as `alias2`, " + "sum(case when testcube.cityid = 'x' then testcube.msr21 else testcube.msr22 end) as `alias3`", - "testcube.alias0 = 'c' and testcube.cityid = 'a' and testcube.zipcode = 'b'", + "ascii(testcube.cityid) = 'c' and testcube.cityid = 'a' and testcube.zipcode = 'b'", "group by ascii(testcube.cityid)"); compareQueries(hqlQuery, expected); @@ -447,7 +468,7 @@ public class TestCubeRewriter extends TestQueryRewrite { "select testcube.cityid as `alias0`, sum(testcube.msr2) as `alias1`, " + "max(testcube.msr3) as `alias2`, " + "sum(case when testcube.cityid = 'x' then testcube.msr21 else testcube.msr22 end) as `alias3`", - "testcube.alias0 = 'a' and testcube.zipcode = 'b'", "group by testcube.cityid"); + "testcube.cityid = 'a' and testcube.zipcode = 'b'", "group by testcube.cityid"); compareQueries(hqlQuery, expected);