LENS-1432 : Map Join tuning broken for this query leading to Map timeouts
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/13ee2851 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/13ee2851 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/13ee2851 Branch: refs/heads/current-release-line Commit: 13ee28519fbb8aa92c30abdff4a286155fddeef5 Parents: 419e190 Author: Sushil Mohanty <sushil.k.moha...@gmail.com> Authored: Tue Jun 6 18:14:39 2017 +0530 Committer: rajub <raju.bairishe...@lazada.com> Committed: Sat Jun 10 13:30:08 2017 +0800 ---------------------------------------------------------------------- .../cube/parse/MultiCandidateQueryWriterContext.java | 11 +++++++++++ .../org/apache/lens/cube/parse/QueryWriterContext.java | 4 ++++ .../java/org/apache/lens/driver/cube/RewriterPlan.java | 6 +++--- .../org/apache/lens/cube/parse/TestRewriterPlan.java | 2 ++ .../lens/cube/parse/TestUnionAndJoinCandidates.java | 2 +- lens-cube/src/test/resources/schema/facts/b1fact1.xml | 2 +- 6 files changed, 22 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/13ee2851/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java index d57c027..50046a5 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java @@ -18,8 +18,11 @@ */ package org.apache.lens.cube.parse; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import org.apache.lens.cube.metadata.Dimension; import org.apache.lens.server.api.error.LensException; import com.google.common.collect.Lists; @@ -90,4 +93,12 @@ public class MultiCandidateQueryWriterContext implements QueryWriterContext { List<StorageCandidateHQLContext> leafWriterContexts = getLeafQueryWriterContexts(); return new UnionQueryWriter(leafWriterContexts, getCubeQueryContext()); } + + public Map<Dimension, CandidateDim> getDimsToQuery() { + Map<Dimension, CandidateDim> allDimsQueried = new HashMap<>(); + for (QueryWriterContext ctx : children) { + allDimsQueried.putAll(ctx.getDimsToQuery()); + } + return allDimsQueried; + } } http://git-wip-us.apache.org/repos/asf/lens/blob/13ee2851/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java index d55de1f..6531f22 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java @@ -18,6 +18,9 @@ */ package org.apache.lens.cube.parse; +import java.util.Map; + +import org.apache.lens.cube.metadata.Dimension; import org.apache.lens.server.api.error.LensException; /** @@ -31,4 +34,5 @@ public interface QueryWriterContext { QueryAST getQueryAst(); void updateFromString() throws LensException; QueryWriter toQueryWriter() throws LensException; + Map<Dimension, CandidateDim> getDimsToQuery(); } http://git-wip-us.apache.org/repos/asf/lens/blob/13ee2851/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java index caf8770..a27aec1 100644 --- a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java +++ b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java @@ -45,10 +45,10 @@ public final class RewriterPlan extends DriverQueryPlan { @SuppressWarnings("unchecked") // required for (Set<FactPartition>) casting void extractPlan(Collection<CubeQueryContext> cubeQueries) { - for (CubeQueryContext ctx : cubeQueries) { - if (ctx.getPickedDimTables() != null && !ctx.getPickedDimTables().isEmpty()) { - for (CandidateDim dim : ctx.getPickedDimTables()) { + if (ctx.getQueryWriterContext().getDimsToQuery() != null + && !ctx.getQueryWriterContext().getDimsToQuery().isEmpty()) { + for (CandidateDim dim : ctx.getQueryWriterContext().getDimsToQuery().values()) { addTablesQueried(dim.getStorageTable()); if (partitions.get(dim.getName()) == null || partitions.get(dim.getName()).isEmpty()) { // puts storage table to latest part http://git-wip-us.apache.org/repos/asf/lens/blob/13ee2851/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java index 76ea77d..6223df7 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java @@ -71,6 +71,8 @@ public class TestRewriterPlan extends TestQueryRewrite { + " cubecity.name != \"XYZ\" and " + TWO_DAYS_RANGE + " having sum(msr2) > 1000 order by cubecity.name limit 50", conf); ctx.toHQL(); + // One dimension table queried + Assert.assertEquals(ctx.getQueryWriterContext().getDimsToQuery().size(), 1); RewriterPlan plan = new RewriterPlan(Collections.singleton(ctx)); Assert.assertNotNull(plan); Assert.assertFalse(plan.getTablesQueried().isEmpty()); http://git-wip-us.apache.org/repos/asf/lens/blob/13ee2851/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java index 429e1c6..dc06ead 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java @@ -147,7 +147,7 @@ public class TestUnionAndJoinCandidates extends TestQueryRewrite { compareContains(outerGroupBy, rewrittenQuery); } - @Test(invocationCount = 100) + @Test public void testFinalCandidateRewrittenQuery() throws ParseException, LensException { try { // Query with non projected measure in having clause. http://git-wip-us.apache.org/repos/asf/lens/blob/13ee2851/lens-cube/src/test/resources/schema/facts/b1fact1.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/facts/b1fact1.xml b/lens-cube/src/test/resources/schema/facts/b1fact1.xml index b51a34a..1500037 100644 --- a/lens-cube/src/test/resources/schema/facts/b1fact1.xml +++ b/lens-cube/src/test/resources/schema/facts/b1fact1.xml @@ -19,7 +19,7 @@ under the License. --> -<x_fact_table name="b1fact1" cube_name="b1cube" weight="5.0" xmlns="uri:lens:cube:0.1"> +<x_fact_table name="b1fact1" cube_name="b1cube" weight="5.1" xmlns="uri:lens:cube:0.1"> <columns> <column name="zipcode" _type="int" comment="zip"/> <column name="cityid" _type="int" comment="city id"/>
