Repository: lens Updated Branches: refs/heads/master 9c03c76e6 -> c7451f8e8
LENS-917 : Fixs table pruning for multiple chains for same destination table Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c7451f8e Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c7451f8e Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c7451f8e Branch: refs/heads/master Commit: c7451f8e8e8f429fc55458b03dbb10a2b7428be9 Parents: 9c03c76 Author: Amareshwari Sriramadasu <amareshw...@apache.org> Authored: Tue Jan 12 11:09:24 2016 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Tue Jan 12 11:09:24 2016 +0530 ---------------------------------------------------------------------- .../org/apache/lens/cube/parse/Aliased.java | 4 + .../lens/cube/parse/CandidateTableResolver.java | 150 +++++++++---------- .../lens/cube/parse/CubeQueryContext.java | 43 +++--- .../lens/cube/parse/ExpressionResolver.java | 3 + .../lens/cube/parse/StorageTableResolver.java | 4 +- .../lens/cube/parse/join/AutoJoinContext.java | 12 +- .../apache/lens/cube/parse/CubeTestSetup.java | 28 ++++ .../cube/parse/TestDenormalizationResolver.java | 16 +- 8 files changed, 152 insertions(+), 108 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/c7451f8e/lens-cube/src/main/java/org/apache/lens/cube/parse/Aliased.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/Aliased.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/Aliased.java index 56fe9fc..160a9c6 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/Aliased.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/Aliased.java @@ -36,4 +36,8 @@ public class Aliased<T extends Named> { public static <K extends Named> Aliased<K> create(K obj, String alias) { return new Aliased<K>(obj, alias); } + + public String getName() { + return object.getName(); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/c7451f8e/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java index 38ff5a4..00ccf36 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java @@ -67,9 +67,9 @@ class CandidateTableResolver implements ContextRewriter { checkForQueriedColumns = false; } else { // populate optional tables - for (Dimension dim : cubeql.getOptionalDimensions()) { + for (Aliased<Dimension> dim : cubeql.getOptionalDimensions()) { log.info("Populating optional dim:{}", dim); - populateDimTables(dim, cubeql, true); + populateDimTables(dim.getObject(), cubeql, true); } if (cubeql.getAutoJoinCtx() != null) { // Before checking for candidate table columns, prune join paths containing non existing columns @@ -117,7 +117,7 @@ class CandidateTableResolver implements ContextRewriter { return; } try { - Set<CandidateDim> candidates = new HashSet<CandidateDim>(); + Set<CandidateDim> candidates = new HashSet<>(); cubeql.getCandidateDimTables().put(dim, candidates); List<CubeDimensionTable> dimtables = cubeql.getMetastoreClient().getAllDimensionTables(dim); if (dimtables.isEmpty()) { @@ -126,7 +126,7 @@ class CandidateTableResolver implements ContextRewriter { "Dimension tables do not exist"); } else { log.info("Not considering optional dimension {} as, No dimension tables exist", dim); - removeOptionalDim(cubeql, dim); + removeOptionalDimWithoutAlias(cubeql, dim); } } for (CubeDimensionTable dimtable : dimtables) { @@ -139,44 +139,30 @@ class CandidateTableResolver implements ContextRewriter { } } - private void pruneOptionalDims(CubeQueryContext cubeql) { - Set<Dimension> tobeRemoved = new HashSet<Dimension>(); - Set<CandidateTable> allCandidates = new HashSet<CandidateTable>(); - allCandidates.addAll(cubeql.getCandidateFacts()); - for (Set<CandidateDim> cdims : cubeql.getCandidateDimTables().values()) { - allCandidates.addAll(cdims); - } - Set<CandidateTable> removedCandidates = new HashSet<CandidateTable>(); - for (Map.Entry<Dimension, OptionalDimCtx> optdimEntry : cubeql.getOptionalDimensionMap().entrySet()) { - Dimension dim = optdimEntry.getKey(); - OptionalDimCtx optdim = optdimEntry.getValue(); - Iterator<CandidateTable> iter = optdim.requiredForCandidates.iterator(); - while (iter.hasNext()) { - CandidateTable candidate = iter.next(); - if (!allCandidates.contains(candidate)) { - log.info("Removing candidate {} from requiredForCandidates of {}, as it is no more candidate", candidate, - dim); - iter.remove(); - removedCandidates.add(candidate); - } + private void removeOptionalDimWithoutAlias(CubeQueryContext cubeql, Dimension dim) { + for (Aliased<Dimension> aDim : cubeql.getOptionalDimensions()) { + if (aDim.getName().equals(dim.getName())) { + removeOptionalDim(cubeql, aDim); } } - Set<CandidateTable> candidatesReachableThroughRefs = new HashSet<CandidateTable>(); - for (Map.Entry<Dimension, OptionalDimCtx> optdimEntry : cubeql.getOptionalDimensionMap().entrySet()) { - Dimension dim = optdimEntry.getKey(); + } + + private void pruneOptionalDims(CubeQueryContext cubeql) { + Set<Aliased<Dimension>> tobeRemoved = new HashSet<>(); + for (Map.Entry<Aliased<Dimension>, OptionalDimCtx> optdimEntry : cubeql.getOptionalDimensionMap().entrySet()) { + Aliased<Dimension> dim = optdimEntry.getKey(); OptionalDimCtx optdim = optdimEntry.getValue(); - candidatesReachableThroughRefs.addAll(optdim.requiredForCandidates); if ((!optdim.colQueried.isEmpty() && optdim.requiredForCandidates.isEmpty()) && !optdim.isRequiredInJoinChain) { log.info("Not considering optional dimension {} as all requiredForCandidates are removed", dim); tobeRemoved.add(dim); } } - for (Dimension dim : tobeRemoved) { + for (Aliased<Dimension> dim : tobeRemoved) { removeOptionalDim(cubeql, dim); } } - private void removeOptionalDim(CubeQueryContext cubeql, Dimension dim) { + private void removeOptionalDim(CubeQueryContext cubeql, Aliased<Dimension> dim) { OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().remove(dim); // remove all the depending candidate table as well for (CandidateTable candidate : optdim.requiredForCandidates) { @@ -240,7 +226,8 @@ class CandidateTableResolver implements ContextRewriter { // go over join chains and prune facts that dont have any of the columns in each chain for (JoinChain chain : cubeql.getJoinchains().values()) { - OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(cubeql.getCubeTbls().get(chain.getName())); + OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(Aliased.create((Dimension)cubeql.getCubeTbls() + .get(chain.getName()), chain.getName())); if (!checkForColumnExists(cfact, chain.getSourceColumns())) { // check if chain is optional or not if (optdim == null) { @@ -282,7 +269,7 @@ class CandidateTableResolver implements ContextRewriter { i.remove(); } } - Set<String> dimExprs = new HashSet<String>(cubeql.getQueriedExprs()); + Set<String> dimExprs = new HashSet<>(cubeql.getQueriedExprs()); dimExprs.removeAll(cubeql.getQueriedExprsWithMeasures()); if (cubeql.getCandidateFacts().size() == 0) { throw new LensException(LensCubeErrorCode.NO_FACT_HAS_COLUMN.getLensErrorInfo(), @@ -292,9 +279,9 @@ class CandidateTableResolver implements ContextRewriter { Set<Set<CandidateFact>> cfactset; if (queriedMsrs.isEmpty() && cubeql.getQueriedExprsWithMeasures().isEmpty()) { // if no measures are queried, add all facts individually as single covering sets - cfactset = new HashSet<Set<CandidateFact>>(); + cfactset = new HashSet<>(); for (CandidateFact cfact : cubeql.getCandidateFacts()) { - Set<CandidateFact> one = new LinkedHashSet<CandidateFact>(); + Set<CandidateFact> one = new LinkedHashSet<>(); one.add(cfact); cfactset.add(one); } @@ -302,7 +289,7 @@ class CandidateTableResolver implements ContextRewriter { } else { // Find out candidate fact table sets which contain all the measures // queried - List<CandidateFact> cfacts = new ArrayList<CandidateFact>(cubeql.getCandidateFacts()); + List<CandidateFact> cfacts = new ArrayList<>(cubeql.getCandidateFacts()); cfactset = findCoveringSets(cubeql, cfacts, queriedMsrs, cubeql.getQueriedExprsWithMeasures()); log.info("Measure covering fact sets :{}", cfactset); @@ -324,8 +311,8 @@ class CandidateTableResolver implements ContextRewriter { static Set<Set<CandidateFact>> findCoveringSets(CubeQueryContext cubeql, List<CandidateFact> cfactsPassed, Set<String> msrs, Set<String> exprsWithMeasures) { - Set<Set<CandidateFact>> cfactset = new HashSet<Set<CandidateFact>>(); - List<CandidateFact> cfacts = new ArrayList<CandidateFact>(cfactsPassed); + Set<Set<CandidateFact>> cfactset = new HashSet<>(); + List<CandidateFact> cfacts = new ArrayList<>(cfactsPassed); for (Iterator<CandidateFact> i = cfacts.iterator(); i.hasNext();) { CandidateFact cfact = i.next(); i.remove(); @@ -336,14 +323,14 @@ class CandidateTableResolver implements ContextRewriter { continue; } else if (cfact.getColumns().containsAll(msrs) && cubeql.getExprCtx().allEvaluable(cfact, exprsWithMeasures)) { // return single set - Set<CandidateFact> one = new LinkedHashSet<CandidateFact>(); + Set<CandidateFact> one = new LinkedHashSet<>(); one.add(cfact); cfactset.add(one); } else { // find the remaining measures in other facts if (i.hasNext()) { - Set<String> remainingMsrs = new HashSet<String>(msrs); - Set<String> remainingExprs = new HashSet<String>(exprsWithMeasures); + Set<String> remainingMsrs = new HashSet<>(msrs); + Set<String> remainingExprs = new HashSet<>(exprsWithMeasures); remainingMsrs.removeAll(cfact.getColumns()); remainingExprs.removeAll(cubeql.getExprCtx().coveringExpressions(exprsWithMeasures, cfact)); Set<Set<CandidateFact>> coveringSets = findCoveringSets(cubeql, cfacts, remainingMsrs, remainingExprs); @@ -366,9 +353,13 @@ class CandidateTableResolver implements ContextRewriter { if (cubeql.getAutoJoinCtx() == null) { return; } - Set<Dimension> allDims = new HashSet<Dimension>(cubeql.getDimensions()); + Set<Aliased<Dimension>> allDims = new HashSet<>(); + for (Dimension dim : cubeql.getDimensions()) { + allDims.add(Aliased.create(dim)); + } allDims.addAll(cubeql.getOptionalDimensions()); - for (Dimension dim : allDims) { + for (Aliased<Dimension> aliasedDim : allDims) { + Dimension dim = aliasedDim.getObject(); if (cubeql.getCandidateDimTables().get(dim) != null && !cubeql.getCandidateDimTables().get(dim).isEmpty()) { for (Iterator<CandidateDim> i = cubeql.getCandidateDimTables().get(dim).iterator(); i.hasNext();) { CandidateDim cdim = i.next(); @@ -377,11 +368,11 @@ class CandidateTableResolver implements ContextRewriter { // can participate in join // for each join path check for columns involved in path boolean removed = false; - for (Map.Entry<Dimension, Map<AbstractCubeTable, List<String>>> joincolumnsEntry : cubeql.getAutoJoinCtx() - .getJoinPathFromColumns().entrySet()) { - Dimension reachableDim = joincolumnsEntry.getKey(); + for (Map.Entry<Aliased<Dimension>, Map<AbstractCubeTable, List<String>>> joincolumnsEntry : cubeql + .getAutoJoinCtx().getJoinPathFromColumns().entrySet()) { + Aliased<Dimension> reachableDim = joincolumnsEntry.getKey(); OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(reachableDim); - Collection<String> colSet = joincolumnsEntry.getValue().get((AbstractCubeTable) dim); + Collection<String> colSet = joincolumnsEntry.getValue().get(dim); if (!checkForColumnExists(cdim, colSet)) { if (optdim == null || optdim.isRequiredInJoinChain @@ -397,11 +388,11 @@ class CandidateTableResolver implements ContextRewriter { } if (!removed) { // check for to columns - for (Map.Entry<Dimension, Map<AbstractCubeTable, List<String>>> joincolumnsEntry : cubeql.getAutoJoinCtx() - .getJoinPathToColumns().entrySet()) { - Dimension reachableDim = joincolumnsEntry.getKey(); + for (Map.Entry<Aliased<Dimension>, Map<AbstractCubeTable, List<String>>> joincolumnsEntry : cubeql + .getAutoJoinCtx().getJoinPathToColumns().entrySet()) { + Aliased<Dimension> reachableDim = joincolumnsEntry.getKey(); OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(reachableDim); - Collection<String> colSet = joincolumnsEntry.getValue().get((AbstractCubeTable) dim); + Collection<String> colSet = joincolumnsEntry.getValue().get(dim); if (!checkForColumnExists(cdim, colSet)) { if (optdim == null || optdim.isRequiredInJoinChain @@ -418,18 +409,18 @@ class CandidateTableResolver implements ContextRewriter { } if (!removed) { // go over the referenced columns accessed in the query and find out which tables can participate - if (cubeql.getOptionalDimensionMap().get(dim) != null - && !checkForColumnExists(cdim, cubeql.getOptionalDimensionMap().get(dim).colQueried)) { + if (cubeql.getOptionalDimensionMap().get(aliasedDim) != null + && !checkForColumnExists(cdim, cubeql.getOptionalDimensionMap().get(aliasedDim).colQueried)) { i.remove(); log.info("Not considering optional dimtable:{} as its denorm fields do not exist. Denorm fields:{}", - dimtable, cubeql.getOptionalDimensionMap().get(dim).colQueried); - cubeql.addDimPruningMsgs(dim, dimtable, - CandidateTablePruneCause.noColumnPartOfAJoinPath(cubeql.getOptionalDimensionMap().get(dim).colQueried)); + dimtable, cubeql.getOptionalDimensionMap().get(aliasedDim).colQueried); + cubeql.addDimPruningMsgs(dim, dimtable, CandidateTablePruneCause + .noColumnPartOfAJoinPath(cubeql.getOptionalDimensionMap().get(aliasedDim).colQueried)); } } } if (cubeql.getCandidateDimTables().get(dim).size() == 0) { - OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(dim); + OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(aliasedDim); if ((cubeql.getDimensions() != null && cubeql.getDimensions().contains(dim)) || (optdim != null && optdim.isRequiredInJoinChain)) { throw new LensException(LensCubeErrorCode.NO_DIM_HAS_COLUMN.getLensErrorInfo(), dim.getName(), @@ -438,7 +429,7 @@ class CandidateTableResolver implements ContextRewriter { // remove it from optional tables log.info("Not considering optional dimension {} as, No dimension table has the queried columns:{}" + " Clearing the required for candidates:{}", dim, optdim.colQueried, optdim.requiredForCandidates); - removeOptionalDim(cubeql, dim); + removeOptionalDim(cubeql, aliasedDim); } } } @@ -456,11 +447,12 @@ class CandidateTableResolver implements ContextRewriter { CubeFactTable fact = cfact.fact; // for each join path check for columns involved in path - for (Map.Entry<Dimension, Map<AbstractCubeTable, List<String>>> joincolumnsEntry : cubeql.getAutoJoinCtx() + for (Map.Entry<Aliased<Dimension>, Map<AbstractCubeTable, List<String>>> joincolumnsEntry : cubeql + .getAutoJoinCtx() .getJoinPathFromColumns().entrySet()) { - Dimension reachableDim = joincolumnsEntry.getKey(); + Aliased<Dimension> reachableDim = joincolumnsEntry.getKey(); OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(reachableDim); - colSet = joincolumnsEntry.getValue().get((AbstractCubeTable) cubeql.getCube()); + colSet = joincolumnsEntry.getValue().get(cubeql.getCube()); if (!checkForColumnExists(cfact, colSet)) { if (optdim == null || optdim.isRequiredInJoinChain @@ -486,21 +478,21 @@ class CandidateTableResolver implements ContextRewriter { * available in candidate tables that want to use references */ private void checkForSourceReachabilityForDenormCandidates(CubeQueryContext cubeql) { - if (cubeql.getOptionalDimensionMap().isEmpty()) { + if (cubeql.getOptionalDimensions().isEmpty()) { return; } if (cubeql.getAutoJoinCtx() == null) { - Set<Dimension> optionaldims = new HashSet<Dimension>(cubeql.getOptionalDimensions()); - for (Dimension dim : optionaldims) { + Set<Aliased<Dimension>> optionaldims = new HashSet<>(cubeql.getOptionalDimensions()); + for (Aliased<Dimension> dim : optionaldims) { log.info("Not considering optional dimension {} as, automatic join resolver is disbled ", dim); removeOptionalDim(cubeql, dim); } return; } // check for source columns for denorm columns - Map<Dimension, Set<CandidateTable>> removedCandidates = new HashMap<Dimension, Set<CandidateTable>>(); - for (Map.Entry<Dimension, OptionalDimCtx> optdimEntry : cubeql.getOptionalDimensionMap().entrySet()) { - Dimension dim = optdimEntry.getKey(); + Map<Aliased<Dimension>, Set<CandidateTable>> removedCandidates = new HashMap<>(); + for (Map.Entry<Aliased<Dimension>, OptionalDimCtx> optdimEntry : cubeql.getOptionalDimensionMap().entrySet()) { + Aliased<Dimension> dim = optdimEntry.getKey(); OptionalDimCtx optdim = optdimEntry.getValue(); Iterator<CandidateTable> iter = optdim.requiredForCandidates.iterator(); // remove candidates from each optional dim if the dimension is not reachable from candidate @@ -537,17 +529,17 @@ class CandidateTableResolver implements ContextRewriter { // F5 | Directly available | Directly available // F6 | Directly available | Not reachable // F3 and F4 will get pruned while iterating over col1 and F1, F6 will get pruned while iterating over col2. - for (Map.Entry<String, Set<Dimension>> dimColEntry : cubeql.getRefColToDim().entrySet()) { - Set<CandidateTable> candidatesReachableThroughRefs = new HashSet<CandidateTable>(); + for (Map.Entry<String, Set<Aliased<Dimension>>> dimColEntry : cubeql.getRefColToDim().entrySet()) { + Set<CandidateTable> candidatesReachableThroughRefs = new HashSet<>(); String col = dimColEntry.getKey(); - Set<Dimension> dimSet = dimColEntry.getValue(); - for (Dimension dim : dimSet) { + Set<Aliased<Dimension>> dimSet = dimColEntry.getValue(); + for (Aliased<Dimension> dim : dimSet) { OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(dim); if (optdim != null) { candidatesReachableThroughRefs.addAll(optdim.requiredForCandidates); } } - for (Dimension dim : dimSet) { + for (Aliased<Dimension> dim : dimSet) { if (removedCandidates.get(dim) != null) { for (CandidateTable candidate : removedCandidates.get(dim)) { if (!candidatesReachableThroughRefs.contains(candidate)) { @@ -582,11 +574,11 @@ class CandidateTableResolver implements ContextRewriter { // F4 | Not evaluable | evaluable through D6 // F5 | Directly available | Directly available // F6 | Directly available | Not evaluable - for (Map.Entry<QueriedExprColumn, Set<Dimension>> exprColEntry : cubeql.getExprColToDim().entrySet()) { + for (Map.Entry<QueriedExprColumn, Set<Aliased<Dimension>>> exprColEntry : cubeql.getExprColToDim().entrySet()) { QueriedExprColumn col = exprColEntry.getKey(); - Set<Dimension> dimSet = exprColEntry.getValue(); + Set<Aliased<Dimension>> dimSet = exprColEntry.getValue(); ExpressionContext ec = cubeql.getExprCtx().getExpressionContext(col.getExprCol(), col.getAlias()); - for (Dimension dim : dimSet) { + for (Aliased<Dimension> dim : dimSet) { if (removedCandidates.get(dim) != null) { for (CandidateTable candidate : removedCandidates.get(dim)) { // check if evaluable expressions of this candidate are no more evaluable because dimension is not reachable @@ -595,7 +587,7 @@ class CandidateTableResolver implements ContextRewriter { Iterator<ExprSpecContext> escIter = ec.getEvaluableExpressions().get(candidate).iterator(); while (escIter.hasNext()) { ExprSpecContext esc = escIter.next(); - if (esc.getExprDims().contains(dim)) { + if (esc.getExprDims().contains(dim.getObject())) { escIter.remove(); } } @@ -623,16 +615,16 @@ class CandidateTableResolver implements ContextRewriter { } // remove optional dims which are not required any more. - Set<Dimension> tobeRemoved = new HashSet<Dimension>(); - for (Map.Entry<Dimension, OptionalDimCtx> optdimEntry : cubeql.getOptionalDimensionMap().entrySet()) { - Dimension dim = optdimEntry.getKey(); + Set<Aliased<Dimension>> tobeRemoved = new HashSet<>(); + for (Map.Entry<Aliased<Dimension>, OptionalDimCtx> optdimEntry : cubeql.getOptionalDimensionMap().entrySet()) { + Aliased<Dimension> dim = optdimEntry.getKey(); OptionalDimCtx optdim = optdimEntry.getValue(); if ((!optdim.colQueried.isEmpty() && optdim.requiredForCandidates.isEmpty()) && !optdim.isRequiredInJoinChain) { log.info("Not considering optional dimension {} as all requiredForCandidates are removed", dim); tobeRemoved.add(dim); } } - for (Dimension dim : tobeRemoved) { + for (Aliased<Dimension> dim : tobeRemoved) { removeOptionalDim(cubeql, dim); } } http://git-wip-us.apache.org/repos/asf/lens/blob/c7451f8e/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java index 06c2a0b..3e930de 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java @@ -96,13 +96,14 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { @Getter // Mapping of a qualified column name to its table alias - private final Map<String, String> colToTableAlias = new HashMap<String, String>(); + private final Map<String, String> colToTableAlias = new HashMap<>(); - @Getter() - private final Set<Set<CandidateFact>> candidateFactSets = new HashSet<Set<CandidateFact>>(); + @Getter + private final Set<Set<CandidateFact>> candidateFactSets = new HashSet<>(); + @Getter // would be added through join chains and de-normalized resolver - protected Map<Dimension, OptionalDimCtx> optionalDimensions = new HashMap<Dimension, OptionalDimCtx>(); + protected Map<Aliased<Dimension>, OptionalDimCtx> optionalDimensionMap = new HashMap<>(); // Alias to table object mapping of tables accessed in this query @Getter @@ -351,10 +352,10 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { // map of ref column in query to set of Dimension that have the column - which are added as optional dims @Getter - private Map<String, Set<Dimension>> refColToDim = Maps.newHashMap(); + private Map<String, Set<Aliased<Dimension>>> refColToDim = Maps.newHashMap(); - public void updateRefColDim(String col, Dimension dim) { - Set<Dimension> refDims = refColToDim.get(col.toLowerCase()); + public void updateRefColDim(String col, Aliased<Dimension> dim) { + Set<Aliased<Dimension>> refDims = refColToDim.get(col.toLowerCase()); if (refDims == null) { refDims = Sets.newHashSet(); refColToDim.put(col.toLowerCase(), refDims); @@ -371,12 +372,12 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { // map of expression column in query to set of Dimension that are accessed in the expression column - which are added // as optional dims @Getter - private Map<QueriedExprColumn, Set<Dimension>> exprColToDim = Maps.newHashMap(); + private Map<QueriedExprColumn, Set<Aliased<Dimension>>> exprColToDim = Maps.newHashMap(); - public void updateExprColDim(String tblAlias, String col, Dimension dim) { + public void updateExprColDim(String tblAlias, String col, Aliased<Dimension> dim) { QueriedExprColumn qexpr = new QueriedExprColumn(col, tblAlias); - Set<Dimension> exprDims = exprColToDim.get(qexpr); + Set<Aliased<Dimension>> exprDims = exprColToDim.get(qexpr); if (exprDims == null) { exprDims = Sets.newHashSet(); exprColToDim.put(qexpr, exprDims); @@ -419,10 +420,11 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { throw new LensException(LensCubeErrorCode.QUERIED_TABLE_NOT_FOUND.getLensErrorInfo(), alias); } Dimension dim = (Dimension) cubeTbls.get(alias); - OptionalDimCtx optDim = optionalDimensions.get(dim); + Aliased<Dimension> aliasedDim = Aliased.create(dim, alias); + OptionalDimCtx optDim = optionalDimensionMap.get(aliasedDim); if (optDim == null) { optDim = new OptionalDimCtx(); - optionalDimensions.put(dim, optDim); + optionalDimensionMap.put(aliasedDim, optDim); } if (cols != null && candidate != null) { for (String col : cols) { @@ -432,16 +434,16 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { } if (cubeCol != null) { if (isRef) { - updateRefColDim(cubeCol, dim); + updateRefColDim(cubeCol, aliasedDim); } else { - updateExprColDim(tableAlias, cubeCol, dim); + updateExprColDim(tableAlias, cubeCol, aliasedDim); } } if (!optDim.isRequiredInJoinChain) { optDim.isRequiredInJoinChain = isRequiredInJoin; } if (log.isDebugEnabled()) { - log.debug("Adding optional dimension:{} optDim:{} {} isRef:{}", dim, optDim, + log.debug("Adding optional dimension:{} optDim:{} {} isRef:{}", aliasedDim, optDim, (cubeCol == null ? "" : " for column:" + cubeCol), isRef); } } @@ -872,6 +874,7 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { public String toHQL() throws LensException { Set<CandidateFact> cfacts = pickCandidateFactToQuery(); Map<Dimension, CandidateDim> dimsToQuery = pickCandidateDimsToQuery(dimensions); + log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery); if (autoJoinCtx != null) { // prune join paths for picked fact and dimensions autoJoinCtx.pruneAllPaths(cube, cfacts, dimsToQuery); @@ -908,6 +911,7 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { exprDimensions.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, this)); } dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimensions)); + log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery); // pick denorm tables for the picked fact and dimensions Set<Dimension> denormTables = new HashSet<Dimension>(); @@ -923,6 +927,7 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { denormTables.addAll(deNormCtx.rewriteDenormctx(null, dimsToQuery, false)); } dimsToQuery.putAll(pickCandidateDimsToQuery(denormTables)); + log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery); // Prune join paths once denorm tables are picked if (autoJoinCtx != null) { // prune join paths for picked fact and dimensions @@ -1129,12 +1134,8 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { } } - public Set<Dimension> getOptionalDimensions() { - return optionalDimensions.keySet(); - } - - public Map<Dimension, OptionalDimCtx> getOptionalDimensionMap() { - return optionalDimensions; + public Set<Aliased<Dimension>> getOptionalDimensions() { + return optionalDimensionMap.keySet(); } /** http://git-wip-us.apache.org/repos/asf/lens/blob/c7451f8e/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index 26514d8..5ff265d 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -151,6 +151,7 @@ class ExpressionResolver implements ContextRewriter { } void addDirectlyAvailable(CandidateTable cTable) { + log.debug("Directly available in {}", cTable); directlyAvailableIn.add(cTable); } @@ -447,6 +448,7 @@ class ExpressionResolver implements ContextRewriter { } // Replace picked expressions in all the base trees replacePickedExpressions(queryAST); + log.debug("Picked expressions: {}", pickedExpressions); for (Set<PickedExpression> peSet : pickedExpressions.values()) { for (PickedExpression pe : peSet) { exprDims.addAll(pe.pickedCtx.exprDims); @@ -518,6 +520,7 @@ class ExpressionResolver implements ContextRewriter { for (ExpressionContext ec : ecSet) { if (ec.getSrcTable().getName().equals(cTable.getBaseTable().getName())) { if (!ec.directlyAvailableIn.contains(cTable)) { + log.debug("{} is not directly evaluable in {}", ec, cTable); if (ec.evaluableExpressions.get(cTable) != null && !ec.evaluableExpressions.get(cTable).isEmpty()) { // pick first evaluable expression Set<PickedExpression> peSet = pickedExpressions.get(ecEntry.getKey()); http://git-wip-us.apache.org/repos/asf/lens/blob/c7451f8e/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java index de5f95e..46b6bb7 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java @@ -156,7 +156,9 @@ class StorageTableResolver implements ContextRewriter { private void resolveDimStorageTablesAndPartitions(CubeQueryContext cubeql) throws LensException { Set<Dimension> allDims = new HashSet<Dimension>(cubeql.getDimensions()); - allDims.addAll(cubeql.getOptionalDimensions()); + for (Aliased<Dimension> dim : cubeql.getOptionalDimensions()) { + allDims.add(dim.getObject()); + } for (Dimension dim : allDims) { Set<CandidateDim> dimTables = cubeql.getCandidateDimTables().get(dim); if (dimTables == null || dimTables.isEmpty()) { http://git-wip-us.apache.org/repos/asf/lens/blob/c7451f8e/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java index 993955a..4c30d3f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java @@ -57,10 +57,10 @@ public class AutoJoinContext { // Map of a joined table to its columns which are part of any of the join // paths. This is used in candidate table resolver @Getter - private Map<Dimension, Map<AbstractCubeTable, List<String>>> joinPathFromColumns = new HashMap<>(); + private Map<Aliased<Dimension>, Map<AbstractCubeTable, List<String>>> joinPathFromColumns = new HashMap<>(); @Getter - private Map<Dimension, Map<AbstractCubeTable, List<String>>> joinPathToColumns = new HashMap<>(); + private Map<Aliased<Dimension>, Map<AbstractCubeTable, List<String>>> joinPathToColumns = new HashMap<>(); // there can be separate join clause for each fact in-case of multi fact queries @Getter @@ -122,12 +122,12 @@ public class AutoJoinContext { Map<AbstractCubeTable, List<String>> toColPaths = joinPathToColumns.get(joinPathEntry.getKey().getObject()); if (fromColPaths == null) { fromColPaths = new HashMap<>(); - joinPathFromColumns.put(joinPathEntry.getKey().getObject(), fromColPaths); + joinPathFromColumns.put(joinPathEntry.getKey(), fromColPaths); } if (toColPaths == null) { toColPaths = new HashMap<>(); - joinPathToColumns.put(joinPathEntry.getKey().getObject(), toColPaths); + joinPathToColumns.put(joinPathEntry.getKey(), toColPaths); } populateJoinPathCols(joinPaths, fromColPaths, toColPaths); } @@ -159,8 +159,8 @@ public class AutoJoinContext { } } - public void removeJoinedTable(Dimension dim) { - allPaths.remove(Aliased.create(dim)); + public void removeJoinedTable(Aliased<Dimension> dim) { + allPaths.remove(dim); joinPathFromColumns.remove(dim); } http://git-wip-us.apache.org/repos/asf/lens/blob/c7451f8e/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 4366938..caea3af 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -563,6 +563,10 @@ public class CubeTestSetup { chainRefs.add(new ChainRefCol("timedatechain2", "full_date")); cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("test_time_dim2", "date", "chained dim"), "Timedim full date", chainRefs, null, null, null, null)); + cubeDimensions.add(new BaseDimAttribute(new FieldSchema("cityid1", "int", "id to city"), + "City1", null, null, null)); + cubeDimensions.add(new BaseDimAttribute(new FieldSchema("cityid2", "int", "id to city"), + "City2", null, null, null)); Map<String, JoinChain> joinChains = new HashMap<>(); addCubeChains(joinChains, TEST_CUBE_NAME); @@ -717,6 +721,26 @@ public class CubeTestSetup { }); } }); + joinChains.put("cubeCity1", new JoinChain("cubeCity1", "cube-city", "city thru cube") { + { + addPath(new ArrayList<TableReference>() { + { + add(new TableReference(cubeName, "cityid1")); + add(new TableReference("citydim", "id")); + } + }); + } + }); + joinChains.put("cubeCity2", new JoinChain("cubeCity2", "cube-city", "city thru cube") { + { + addPath(new ArrayList<TableReference>() { + { + add(new TableReference(cubeName, "cityid2")); + add(new TableReference("citydim", "id")); + } + }); + } + }); joinChains.put("cubeState", new JoinChain("cubeState", "cube-state", "state thru cube") { { addPath(new ArrayList<TableReference>() { @@ -1345,6 +1369,7 @@ public class CubeTestSetup { // add dimensions of the cube factColumns.add(new FieldSchema("zipcode", "int", "zip")); factColumns.add(new FieldSchema("cityid", "int", "city id")); + factColumns.add(new FieldSchema("cityid1", "int", "city id")); factColumns.add(new FieldSchema("stateid", "int", "city id")); factColumns.add(new FieldSchema("test_time_dim_day_id", "int", "time id")); factColumns.add(new FieldSchema("test_time_dim_day_id2", "int", "time id")); @@ -1580,6 +1605,7 @@ public class CubeTestSetup { // add dimensions of the cube factColumns.add(new FieldSchema("zipcode", "int", "zip")); factColumns.add(new FieldSchema("cityid", "int", "city id")); + factColumns.add(new FieldSchema("cityid2", "int", "city id")); factColumns.add(new FieldSchema("test_time_dim_hour_id", "int", "time id")); factColumns.add(new FieldSchema("test_time_dim_hour_id2", "int", "time id")); factColumns.add(new FieldSchema("cdim2", "int", "cycledim id")); @@ -1714,6 +1740,8 @@ public class CubeTestSetup { // add dimensions of the cube factColumns.add(new FieldSchema("zipcode", "int", "zip")); factColumns.add(new FieldSchema("cityid", "int", "city id")); + factColumns.add(new FieldSchema("cityid1", "int", "city id")); + factColumns.add(new FieldSchema("cityid2", "int", "city id")); factColumns.add(new FieldSchema("stateid", "int", "state id")); factColumns.add(new FieldSchema("countryid", "int", "country id")); factColumns.add(new FieldSchema("dim1", "string", "dim1")); http://git-wip-us.apache.org/repos/asf/lens/blob/c7451f8e/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java index a8390ef..d7707a9 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java @@ -267,7 +267,7 @@ public class TestDenormalizationResolver extends TestQueryRewrite { // candidate Assert.assertEquals(getLensExceptionErrorMessageInRewrite( "select citydim.name, citydim.statename, citydim.nocandidatecol " + "from citydim", conf), - "No dimension table has the queried columns " + "for citydim, columns: [name, statename, nocandidatecol]"); + "No dimension table has the queried columns " + "for citydim, columns: [name, statename, nocandidatecol]"); } @Test @@ -330,6 +330,20 @@ public class TestDenormalizationResolver extends TestQueryRewrite { } @Test + public void testTwoFieldsFromDifferentChainButSameTable() throws Exception { + String hqlQuery = rewrite("select cubecity1.name, cubecity2.name, msr2 from testcube where " + TWO_DAYS_RANGE, + conf); + String joinExpr = " join " + getDbName() + + "c1_citytable cubecity1 on testcube.cityid1 = cubecity1.id and (cubecity1.dt = 'latest') " + + " join " + getDbName() + + "c1_citytable cubecity2 on testcube.cityid2 = cubecity2.id and (cubecity2.dt = 'latest')"; + String expected = + getExpectedQuery("testcube", "select cubecity1.name, cubecity2.name, sum(testcube.msr2) FROM ", + joinExpr, null, " group by cubecity1.name, cubecity2.name", null, + getWhereForHourly2days("testcube", "c1_testfact2_raw")); + TestCubeRewriter.compareQueries(hqlQuery, expected); + } + @Test public void testDimensionQueryWithTwoRefCols() throws Exception { Configuration tConf = new Configuration(conf); tConf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "");
