http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java index 2ab7f4b..97a73a8 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java @@ -51,9 +51,6 @@ class CandidateTableResolver implements ContextRewriter { private boolean checkForQueriedColumns = true; - public CandidateTableResolver(Configuration ignored) { - } - @Override public void rewriteContext(CubeQueryContext cubeql) throws LensException { if (checkForQueriedColumns) { @@ -87,27 +84,20 @@ class CandidateTableResolver implements ContextRewriter { } private void populateCandidateTables(CubeQueryContext cubeql) throws LensException { - int aliasCounter = 0; if (cubeql.getCube() != null) { List<CubeFactTable> factTables = cubeql.getMetastoreClient().getAllFacts(cubeql.getCube()); - // TODO union : Check for cube table partially valid, else remove it. if (factTables.isEmpty()) { throw new LensException(LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo(), cubeql.getCube().getName() + " does not have any facts"); } for (CubeFactTable fact : factTables) { - Iterator<String> it = fact.getStorages().iterator(); - //TODO union : Add MISSING_STORAGES pruning message - /* Moved this from StorageTableResolver if (fact.getUpdatePeriods().isEmpty()) { - cubeql.addFactPruningMsgs(fact, new CandidateTablePruneCause(CandidateTablePruneCode.MISSING_STORAGES)); - i.remove(); - continue; - } - */ - while(it.hasNext()) { - StorageCandidate sc = new StorageCandidate(cubeql.getCube(), fact, it.next(), cubeql); - cubeql.getCandidates().add(sc); + log.info("Not considering fact: {} as it has no update periods", fact.getName()); + } else { + for (String s : fact.getStorages()) { + StorageCandidate sc = new StorageCandidate(cubeql.getCube(), fact, s, cubeql); + cubeql.getCandidates().add(sc); + } } } log.info("Populated storage candidates: {}", cubeql.getCandidates()); @@ -284,7 +274,7 @@ class CandidateTableResolver implements ContextRewriter { if (!qur.isEvaluable(cubeql, sc)) { log.info("Not considering storage candidate:{} as columns {} are not available", sc, qur.getColumns()); cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.columnNotFound( - CandidateTablePruneCode.COLUMN_NOT_FOUND, qur.getColumns())); + qur.getColumns())); toRemove = true; break; } @@ -297,7 +287,7 @@ class CandidateTableResolver implements ContextRewriter { Set<String> columns = getColumns(queriedMsrs); log.info("Not considering storage candidate:{} as columns {} is not available", sc, columns); cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.columnNotFound( - CandidateTablePruneCode.COLUMN_NOT_FOUND, columns)); + columns)); toRemove = true; } @@ -311,7 +301,7 @@ class CandidateTableResolver implements ContextRewriter { log.info("Not considering storage candidate:{} as columns {} are not available", sc, chain.getSourceColumns()); cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.columnNotFound( - CandidateTablePruneCode.COLUMN_NOT_FOUND, chain.getSourceColumns())); + chain.getSourceColumns())); toRemove = true; break; } @@ -539,7 +529,7 @@ class CandidateTableResolver implements ContextRewriter { log.info("Not considering Storage:{} as its required optional dims are not reachable", candidate); cubeql.getCandidates().remove(candidate); cubeql.addStoragePruningMsg((StorageCandidate) candidate, - CandidateTablePruneCause.columnNotFound(CandidateTablePruneCode.COLUMN_NOT_FOUND, col)); + CandidateTablePruneCause.columnNotFound(col)); Collection<Candidate> prunedCandidates = CandidateUtil. filterCandidates(cubeql.getCandidates(), (StorageCandidate) candidate); cubeql.addCandidatePruningMsg(prunedCandidates, @@ -550,7 +540,7 @@ class CandidateTableResolver implements ContextRewriter { cubeql.getCandidateDimTables().get(((CandidateDim) candidate).getBaseTable()).remove(candidate); cubeql.addDimPruningMsgs((Dimension) candidate.getBaseTable(), (CubeDimensionTable) candidate.getTable(), - CandidateTablePruneCause.columnNotFound(CandidateTablePruneCode.COLUMN_NOT_FOUND, col)); + CandidateTablePruneCause.columnNotFound(col)); } } } @@ -645,12 +635,12 @@ class CandidateTableResolver implements ContextRewriter { i.remove(); break; } - } else if (!cubeql.getDeNormCtx().addRefUsage(cdim, col, dim.getName())) { + } else if (!cubeql.getDeNormCtx().addRefUsage(cubeql, cdim, col, dim.getName())) { // check if it available as reference, if not remove the // candidate log.info("Not considering dimtable: {} as column {} is not available", cdim, col); cubeql.addDimPruningMsgs(dim, cdim.getTable(), CandidateTablePruneCause.columnNotFound( - CandidateTablePruneCode.COLUMN_NOT_FOUND, col)); + col)); i.remove(); break; }
http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java index bdde27c..68449f6 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java @@ -43,20 +43,6 @@ import com.google.common.collect.TreeRangeSet; public class CandidateUtil { /** - * Is calculated measure expression answerable by the Candidate - * @param exprNode - * @param candidate - * @param context - * @return - * @throws LensException - */ - public static boolean isMeasureExpressionAnswerable(ASTNode exprNode, Candidate candidate, CubeQueryContext context) - throws LensException { - return candidate.getColumns().containsAll(HQLParser.getColsInExpr( - context.getAliasForTableName(context.getCube()), exprNode)); - } - - /** * Returns true if the Candidate is valid for all the timeranges based on its start and end times. * @param candidate * @param timeRanges @@ -72,39 +58,10 @@ public class CandidateUtil { return true; } - public static boolean isPartiallyValidForTimeRanges(Candidate cand, List<TimeRange> timeRanges) { - for (TimeRange timeRange : timeRanges) { - if ((cand.getStartTime().before(timeRange.getFromDate()) && cand.getEndTime().after(timeRange.getFromDate())) - || (cand.getStartTime().before(timeRange.getToDate()) && cand.getEndTime().after(timeRange.getToDate()))) { - return true; - } - } - return false; - } - - /** - * Gets the time partition columns for a storage candidate - * TODO decide is this needs to be supported for all Candidate types. - * - * @param candidate : Stoarge Candidate - * @param metastoreClient : Cube metastore client - * @return - * @throws LensException - */ - public Set<String> getTimePartitionCols(StorageCandidate candidate, CubeMetastoreClient metastoreClient) - throws LensException { - Set<String> cubeTimeDimensions = candidate.getCube().getTimedDimensions(); - Set<String> timePartDimensions = new HashSet<String>(); - String singleStorageTable = candidate.getStorageName(); - List<FieldSchema> partitionKeys = null; - partitionKeys = metastoreClient.getTable(singleStorageTable).getPartitionKeys(); - for (FieldSchema fs : partitionKeys) { - if (cubeTimeDimensions.contains(CubeQueryContext.getTimeDimOfPartitionColumn(candidate.getCube(), - fs.getName()))) { - timePartDimensions.add(fs.getName()); - } - } - return timePartDimensions; + static boolean isPartiallyValidForTimeRanges(Candidate cand, List<TimeRange> timeRanges) { + return timeRanges.stream().anyMatch(timeRange -> + (cand.getStartTime().before(timeRange.getFromDate()) && cand.getEndTime().after(timeRange.getFromDate())) + || (cand.getStartTime().before(timeRange.getToDate()) && cand.getEndTime().after(timeRange.getToDate()))); } /** @@ -114,7 +71,7 @@ public class CandidateUtil { * @param targetAst * @throws LensException */ - public static void copyASTs(QueryAST sourceAst, QueryAST targetAst) throws LensException { + static void copyASTs(QueryAST sourceAst, QueryAST targetAst) throws LensException { targetAst.setSelectAST(MetastoreUtil.copyAST(sourceAst.getSelectAST())); targetAst.setWhereAST(MetastoreUtil.copyAST(sourceAst.getWhereAST())); if (sourceAst.getJoinAST() != null) { @@ -132,9 +89,10 @@ public class CandidateUtil { return getStorageCandidates(new HashSet<Candidate>(1) {{ add(candidate); }}); } - - public static Set<QueriedPhraseContext> coveredMeasures(Candidate candSet, Collection<QueriedPhraseContext> msrs, - CubeQueryContext cubeql) throws LensException { + // this function should only be used for union candidates and never for join candidates. + // future scope of improvement: move the data model to use polymorphism + static Set<QueriedPhraseContext> coveredMeasures(Candidate candSet, Collection<QueriedPhraseContext> msrs, + CubeQueryContext cubeql) throws LensException { Set<QueriedPhraseContext> coveringSet = new HashSet<>(); for (QueriedPhraseContext msr : msrs) { if (candSet.getChildren() == null) { @@ -142,12 +100,16 @@ public class CandidateUtil { coveringSet.add(msr); } } else { - // TODO union : all candidates should answer + boolean allCanAnswer = true; for (Candidate cand : candSet.getChildren()) { - if (msr.isEvaluable(cubeql, (StorageCandidate) cand)) { - coveringSet.add(msr); + if (!msr.isEvaluable(cubeql, (StorageCandidate) cand)) { + allCanAnswer = false; + break; } } + if (allCanAnswer) { + coveringSet.add(msr); + } } } return coveringSet; @@ -235,7 +197,7 @@ public class CandidateUtil { public static class ChildrenSizeBasedCandidateComparator<T> implements Comparator<Candidate> { @Override public int compare(Candidate o1, Candidate o2) { - return Integer.valueOf(o1.getChildren().size() - o2.getChildren().size()); + return o1.getChildren().size() - o2.getChildren().size(); } } @@ -279,7 +241,7 @@ public class CandidateUtil { if (limit != null) { queryFormat.append(" LIMIT %s"); } - return String.format(queryFormat.toString(), qstrs.toArray(new String[0])); + return String.format(queryFormat.toString(), qstrs.toArray(new String[qstrs.size()])); } /** @@ -312,15 +274,4 @@ public class CandidateUtil { } } - public static boolean containsAny(Set<String> srcSet, Set<String> colSet) { - if (colSet == null || colSet.isEmpty()) { - return true; - } - for (String column : colSet) { - if (srcSet.contains(column)) { - return true; - } - } - return false; - } } http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/CheckTableNames.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CheckTableNames.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CheckTableNames.java index 8586262..df35a42 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CheckTableNames.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CheckTableNames.java @@ -30,7 +30,6 @@ public class CheckTableNames extends ValidationRule { @Override public boolean validate(CubeQueryContext ctx) throws LensException { - // TODO return true; } http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnLifetimeChecker.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnLifetimeChecker.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnLifetimeChecker.java new file mode 100644 index 0000000..24eb8f0 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnLifetimeChecker.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + +import static org.apache.hadoop.hive.ql.parse.HiveParser.*; + +import java.util.*; + +import org.apache.lens.cube.error.ColUnAvailableInTimeRange; +import org.apache.lens.cube.error.ColUnAvailableInTimeRangeException; +import org.apache.lens.cube.error.LensCubeErrorCode; +import org.apache.lens.cube.metadata.*; +import org.apache.lens.cube.metadata.join.JoinPath; +import org.apache.lens.cube.parse.join.AutoJoinContext; +import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.error.LensException; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.plan.PlanUtils; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ColumnLifetimeChecker implements ContextRewriter { + @Override + public void rewriteContext(CubeQueryContext cubeql) throws LensException { + if (cubeql.getCube() == null) { + return; + } + doColLifeValidation(cubeql); + } + + private void doColLifeValidation(CubeQueryContext cubeql) throws LensException, + ColUnAvailableInTimeRangeException { + Set<String> cubeColumns = cubeql.getColumnsQueriedForTable(cubeql.getCube().getName()); + if (cubeColumns == null || cubeColumns.isEmpty()) { + // Query doesn't have any columns from cube + return; + } + + for (String col : cubeql.getColumnsQueriedForTable(cubeql.getCube().getName())) { + CubeColumn column = cubeql.getCube().getColumnByName(col); + for (TimeRange range : cubeql.getTimeRanges()) { + if (column == null) { + if (!cubeql.getCube().getTimedDimensions().contains(col)) { + throw new LensException(LensCubeErrorCode.NOT_A_CUBE_COLUMN.getLensErrorInfo(), col); + } + continue; + } + if (!column.isColumnAvailableInTimeRange(range)) { + throwException(column); + } + } + } + + // Remove join paths that have columns with invalid life span + AutoJoinContext joinContext = cubeql.getAutoJoinCtx(); + if (joinContext == null) { + return; + } + // Get cube columns which are part of join chain + Set<String> joinColumns = joinContext.getAllJoinPathColumnsOfTable((AbstractCubeTable) cubeql.getCube()); + if (joinColumns == null || joinColumns.isEmpty()) { + return; + } + + // Loop over all cube columns part of join paths + for (String col : joinColumns) { + CubeColumn column = cubeql.getCube().getColumnByName(col); + for (TimeRange range : cubeql.getTimeRanges()) { + if (!column.isColumnAvailableInTimeRange(range)) { + log.info("Timerange queried is not in column life for {}, Removing join paths containing the column", column); + // Remove join paths containing this column + Map<Aliased<Dimension>, List<JoinPath>> allPaths = joinContext.getAllPaths(); + + for (Aliased<Dimension> dimension : allPaths.keySet()) { + List<JoinPath> joinPaths = allPaths.get(dimension); + Iterator<JoinPath> joinPathIterator = joinPaths.iterator(); + + while (joinPathIterator.hasNext()) { + JoinPath path = joinPathIterator.next(); + if (path.containsColumnOfTable(col, (AbstractCubeTable) cubeql.getCube())) { + log.info("Removing join path: {} as columns :{} is not available in the range", path, col); + joinPathIterator.remove(); + if (joinPaths.isEmpty()) { + // This dimension doesn't have any paths left + throw new LensException(LensCubeErrorCode.NO_JOIN_PATH.getLensErrorInfo(), + "No valid join path available for dimension " + dimension + " which would satisfy time range " + + range.getFromDate() + "-" + range.getToDate()); + } + } + } // End loop to remove path + + } // End loop for all paths + } + } // End time range loop + } // End column loop + } + + private void throwException(CubeColumn column) throws ColUnAvailableInTimeRangeException { + + final Long availabilityStartTime = (column.getStartTimeMillisSinceEpoch().isPresent()) + ? column.getStartTimeMillisSinceEpoch().get() : null; + + final Long availabilityEndTime = column.getEndTimeMillisSinceEpoch().isPresent() + ? column.getEndTimeMillisSinceEpoch().get() : null; + + ColUnAvailableInTimeRange col = new ColUnAvailableInTimeRange(column.getName(), availabilityStartTime, + availabilityEndTime); + + throw new ColUnAvailableInTimeRangeException(col); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java index e56193c..76031ec 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,22 +19,44 @@ package org.apache.lens.cube.parse; -import static org.apache.lens.cube.parse.CubeQueryConfUtil.*; - -import static org.apache.hadoop.hive.ql.parse.HiveParser.*; - import static com.google.common.base.Preconditions.checkArgument; - - +import static java.util.stream.Collectors.toSet; +import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier; +import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TABLE_OR_COL; +import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TMP_FILE; +import static org.apache.lens.cube.parse.CubeQueryConfUtil.DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL; +import static org.apache.lens.cube.parse.CubeQueryConfUtil.DEFAULT_REWRITE_DIM_FILTER_TO_FACT_FILTER; +import static org.apache.lens.cube.parse.CubeQueryConfUtil.NON_EXISTING_PARTITIONS; +import static org.apache.lens.cube.parse.CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL; +import static org.apache.lens.cube.parse.CubeQueryConfUtil.REWRITE_DIM_FILTER_TO_FACT_FILTER; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.Predicate; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.error.NoCandidateDimAvailableException; import org.apache.lens.cube.error.NoCandidateFactAvailableException; -import org.apache.lens.cube.metadata.*; +import org.apache.lens.cube.metadata.AbstractCubeTable; +import org.apache.lens.cube.metadata.Cube; +import org.apache.lens.cube.metadata.CubeDimensionTable; +import org.apache.lens.cube.metadata.CubeInterface; +import org.apache.lens.cube.metadata.CubeMetastoreClient; +import org.apache.lens.cube.metadata.DerivedCube; +import org.apache.lens.cube.metadata.Dimension; +import org.apache.lens.cube.metadata.JoinChain; +import org.apache.lens.cube.metadata.Named; +import org.apache.lens.cube.metadata.TimeRange; import org.apache.lens.cube.metadata.join.TableRelationship; import org.apache.lens.cube.parse.join.AutoJoinContext; import org.apache.lens.cube.parse.join.JoinClause; @@ -47,21 +69,33 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.parse.*; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.HiveParser; +import org.apache.hadoop.hive.ql.parse.JoinCond; +import org.apache.hadoop.hive.ql.parse.ParseDriver; +import org.apache.hadoop.hive.ql.parse.ParseException; +import org.apache.hadoop.hive.ql.parse.ParseUtils; +import org.apache.hadoop.hive.ql.parse.QB; +import org.apache.hadoop.hive.ql.parse.QBJoinTree; +import org.apache.hadoop.hive.ql.parse.QBParseInfo; import org.codehaus.jackson.map.ObjectMapper; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import lombok.*; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; @Slf4j -public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { - public static final String TIME_RANGE_FUNC = "time_range_in"; +public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, TrackDenormContext { + static final String TIME_RANGE_FUNC = "time_range_in"; public static final String NOW = "now"; - public static final String DEFAULT_TABLE = "_default_"; + static final String DEFAULT_TABLE = "_default_"; private final ASTNode ast; @Getter private final QB qb; @@ -124,12 +158,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { } boolean isColumnAnAlias(String col) { - for (SelectPhraseContext sel : selectPhrases) { - if (col.equals(sel.getActualAlias())) { - return true; - } - } - return false; + return selectPhrases.stream().map(SelectPhraseContext::getActualAlias).anyMatch(Predicate.isEqual(col)); } void addQueriedPhrase(QueriedPhraseContext qur) { @@ -145,9 +174,9 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { // Join conditions used in all join expressions @Getter - private final Map<QBJoinTree, String> joinConds = new HashMap<QBJoinTree, String>(); + private final Map<QBJoinTree, String> joinConds = new HashMap<>(); @Getter - protected final Map<Dimension, Set<CandidateDim>> candidateDims = new HashMap<Dimension, Set<CandidateDim>>(); + protected final Map<Dimension, Set<CandidateDim>> candidateDims = new HashMap<>(); // query trees @Getter @Setter @@ -192,7 +221,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { this.qb = qb; this.conf = queryConf; this.clauseName = getClause(); - this.timeRanges = new ArrayList<TimeRange>(); + this.timeRanges = new ArrayList<>(); try { metastoreClient = CubeMetastoreClient.getInstance(metastoreConf); } catch (HiveException e) { @@ -217,14 +246,10 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { extractMetaTables(); } - public boolean hasCubeInQuery() { + boolean hasCubeInQuery() { return cube != null; } - public boolean hasDimensionInQuery() { - return dimensions != null && !dimensions.isEmpty(); - } - private void extractMetaTables() throws LensException { List<String> tabAliases = new ArrayList<String>(qb.getTabAliases()); Set<String> missing = new HashSet<String>(); @@ -280,10 +305,10 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { return true; } - return retVal; + return false; } - public boolean addQueriedTable(String alias) throws LensException { + boolean addQueriedTable(String alias) throws LensException { return addQueriedTable(alias, false); } @@ -346,36 +371,24 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { return true; } - public boolean isAutoJoinResolved() { + boolean isAutoJoinResolved() { return autoJoinCtx != null && autoJoinCtx.isJoinsResolved(); } - public Cube getBaseCube() { - if (cube instanceof Cube) { - return (Cube) cube; - } - return ((DerivedCube) cube).getParent(); + Cube getBaseCube() { + return cube instanceof Cube ? (Cube) cube : ((DerivedCube) cube).getParent(); } - public Set<String> getPartitionColumnsQueried() { - Set<String> partsQueried = Sets.newHashSet(); - for (TimeRange range : getTimeRanges()) { - partsQueried.add(range.getPartitionColumn()); - } - return partsQueried; + Set<String> getPartitionColumnsQueried() { + return getTimeRanges().stream().map(TimeRange::getPartitionColumn).collect(toSet()); } // map of ref column in query to set of Dimension that have the column - which are added as optional dims @Getter private Map<String, Set<Aliased<Dimension>>> refColToDim = Maps.newHashMap(); - public void updateRefColDim(String col, Aliased<Dimension> dim) { - Set<Aliased<Dimension>> refDims = refColToDim.get(col.toLowerCase()); - if (refDims == null) { - refDims = Sets.newHashSet(); - refColToDim.put(col.toLowerCase(), refDims); - } - refDims.add(dim); + private void updateRefColDim(String col, Aliased<Dimension> dim) { + refColToDim.computeIfAbsent(col.toLowerCase(), k -> Sets.newHashSet()).add(dim); } @Data @@ -390,15 +403,8 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { @Getter private Map<QueriedExprColumn, Set<Aliased<Dimension>>> exprColToDim = Maps.newHashMap(); - public void updateExprColDim(String tblAlias, String col, Aliased<Dimension> dim) { - - QueriedExprColumn qexpr = new QueriedExprColumn(col, tblAlias); - Set<Aliased<Dimension>> exprDims = exprColToDim.get(qexpr); - if (exprDims == null) { - exprDims = Sets.newHashSet(); - exprColToDim.put(qexpr, exprDims); - } - exprDims.add(dim); + private void updateExprColDim(String tblAlias, String col, Aliased<Dimension> dim) { + exprColToDim.computeIfAbsent(new QueriedExprColumn(col, tblAlias), k -> Sets.newHashSet()).add(dim); } // Holds the context of optional dimension @@ -406,7 +412,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { // required by a candidate table to get a denormalized field from reference // or required in a join chain @ToString - public static class OptionalDimCtx { + static class OptionalDimCtx { OptionalDimCtx() { } @@ -415,16 +421,16 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { boolean isRequiredInJoinChain = false; } - public void addOptionalJoinDimTable(String alias, boolean isRequired) throws LensException { + void addOptionalJoinDimTable(String alias, boolean isRequired) throws LensException { addOptionalDimTable(alias, null, isRequired, null, false, (String[]) null); } - public void addOptionalExprDimTable(String dimAlias, String queriedExpr, String srcTableAlias, + void addOptionalExprDimTable(String dimAlias, String queriedExpr, String srcTableAlias, CandidateTable candidate, String... cols) throws LensException { addOptionalDimTable(dimAlias, candidate, false, queriedExpr, false, srcTableAlias, cols); } - public void addOptionalDimTable(String alias, CandidateTable candidate, boolean isRequiredInJoin, String cubeCol, + void addOptionalDimTable(String alias, CandidateTable candidate, boolean isRequiredInJoin, String cubeCol, boolean isRef, String... cols) throws LensException { addOptionalDimTable(alias, candidate, isRequiredInJoin, cubeCol, isRef, null, cols); } @@ -437,15 +443,9 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { } Dimension dim = (Dimension) cubeTbls.get(alias); Aliased<Dimension> aliasedDim = Aliased.create(dim, alias); - OptionalDimCtx optDim = optionalDimensionMap.get(aliasedDim); - if (optDim == null) { - optDim = new OptionalDimCtx(); - optionalDimensionMap.put(aliasedDim, optDim); - } + OptionalDimCtx optDim = optionalDimensionMap.computeIfAbsent(aliasedDim, k -> new OptionalDimCtx()); if (cols != null && candidate != null) { - for (String col : cols) { - optDim.colQueried.add(col); - } + optDim.colQueried.addAll(Arrays.asList(cols)); optDim.requiredForCandidates.add(candidate); } if (cubeCol != null) { @@ -480,32 +480,28 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { return candidateDims; } - public void addCandidatePruningMsg(Collection<Candidate> candidateCollection, CandidateTablePruneCause pruneCause) { + void addCandidatePruningMsg(Collection<Candidate> candidateCollection, CandidateTablePruneCause pruneCause) { for (Candidate c : candidateCollection){ addCandidatePruningMsg(c, pruneCause); } - } - public void addCandidatePruningMsg(Candidate cand, CandidateTablePruneCause pruneCause) { + void addCandidatePruningMsg(Candidate cand, CandidateTablePruneCause pruneCause) { Set<StorageCandidate> scs = CandidateUtil.getStorageCandidates(cand); for (StorageCandidate sc : scs) { addStoragePruningMsg(sc, pruneCause); } } - public void addStoragePruningMsg(StorageCandidate sc, CandidateTablePruneCause factPruningMsg) { - log.info("Pruning Storage {} with cause: {}", sc, factPruningMsg); - storagePruningMsgs.addPruningMsg(sc, factPruningMsg); + void addStoragePruningMsg(StorageCandidate sc, CandidateTablePruneCause... factPruningMsgs) { + for (CandidateTablePruneCause factPruningMsg: factPruningMsgs) { + log.info("Pruning Storage {} with cause: {}", sc, factPruningMsg); + storagePruningMsgs.addPruningMsg(sc, factPruningMsg); + } } public void addDimPruningMsgs(Dimension dim, CubeDimensionTable dimtable, CandidateTablePruneCause msg) { - PruneCauses<CubeDimensionTable> dimMsgs = dimPruningMsgs.get(dim); - if (dimMsgs == null) { - dimMsgs = new PruneCauses<CubeDimensionTable>(); - dimPruningMsgs.put(dim, dimMsgs); - } - dimMsgs.addPruningMsg(dimtable, msg); + dimPruningMsgs.computeIfAbsent(dim, k -> new PruneCauses<>()).addPruningMsg(dimtable, msg); } public String getAliasForTableName(Named named) { @@ -532,73 +528,75 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { if (!log.isDebugEnabled()) { return; } - StringBuilder builder = new StringBuilder(); - builder.append("ASTNode:" + ast.dump() + "\n"); - builder.append("QB:"); - builder.append("\n numJoins:" + qb.getNumJoins()); - builder.append("\n numGbys:" + qb.getNumGbys()); - builder.append("\n numSels:" + qb.getNumSels()); - builder.append("\n numSelDis:" + qb.getNumSelDi()); - builder.append("\n aliasToTabs:"); + StringBuilder builder = new StringBuilder() + .append("ASTNode:").append(ast.dump()).append("\n") + .append("QB:") + .append("\n numJoins:").append(qb.getNumJoins()) + .append("\n numGbys:").append(qb.getNumGbys()) + .append("\n numSels:").append(qb.getNumSels()) + .append("\n numSelDis:").append(qb.getNumSelDi()) + .append("\n aliasToTabs:"); Set<String> tabAliases = qb.getTabAliases(); for (String alias : tabAliases) { - builder.append("\n\t" + alias + ":" + qb.getTabNameForAlias(alias)); + builder.append("\n\t").append(alias).append(":").append(qb.getTabNameForAlias(alias)); } builder.append("\n aliases:"); for (String alias : qb.getAliases()) { builder.append(alias); builder.append(", "); } - builder.append("id:" + qb.getId()); - builder.append("isQuery:" + qb.getIsQuery()); - builder.append("\n QBParseInfo"); + builder + .append("id:").append(qb.getId()) + .append("isQuery:").append(qb.getIsQuery()) + .append("\n QBParseInfo"); QBParseInfo parseInfo = qb.getParseInfo(); - builder.append("\n isSubQ: " + parseInfo.getIsSubQ()); - builder.append("\n alias: " + parseInfo.getAlias()); + builder + .append("\n isSubQ: ").append(parseInfo.getIsSubQ()) + .append("\n alias: ").append(parseInfo.getAlias()); if (parseInfo.getJoinExpr() != null) { - builder.append("\n joinExpr: " + parseInfo.getJoinExpr().dump()); + builder.append("\n joinExpr: ").append(parseInfo.getJoinExpr().dump()); } - builder.append("\n hints: " + parseInfo.getHints()); + builder.append("\n hints: ").append(parseInfo.getHints()); builder.append("\n aliasToSrc: "); for (String alias : tabAliases) { - builder.append("\n\t" + alias + ": " + parseInfo.getSrcForAlias(alias).dump()); + builder.append("\n\t").append(alias).append(": ").append(parseInfo.getSrcForAlias(alias).dump()); } TreeSet<String> clauses = new TreeSet<String>(parseInfo.getClauseNames()); for (String clause : clauses) { - builder.append("\n\t" + clause + ": " + parseInfo.getClauseNamesForDest()); + builder.append("\n\t").append(clause).append(": ").append(parseInfo.getClauseNamesForDest()); } String clause = clauses.first(); if (parseInfo.getWhrForClause(clause) != null) { - builder.append("\n whereexpr: " + parseInfo.getWhrForClause(clause).dump()); + builder.append("\n whereexpr: ").append(parseInfo.getWhrForClause(clause).dump()); } if (parseInfo.getGroupByForClause(clause) != null) { - builder.append("\n groupby expr: " + parseInfo.getGroupByForClause(clause).dump()); + builder.append("\n groupby expr: ").append(parseInfo.getGroupByForClause(clause).dump()); } if (parseInfo.getSelForClause(clause) != null) { - builder.append("\n sel expr: " + parseInfo.getSelForClause(clause).dump()); + builder.append("\n sel expr: ").append(parseInfo.getSelForClause(clause).dump()); } if (parseInfo.getHavingForClause(clause) != null) { - builder.append("\n having expr: " + parseInfo.getHavingForClause(clause).dump()); + builder.append("\n having expr: ").append(parseInfo.getHavingForClause(clause).dump()); } if (parseInfo.getDestLimit(clause) != null) { - builder.append("\n limit: " + parseInfo.getDestLimit(clause)); + builder.append("\n limit: ").append(parseInfo.getDestLimit(clause)); } if (parseInfo.getAllExprToColumnAlias() != null && !parseInfo.getAllExprToColumnAlias().isEmpty()) { builder.append("\n exprToColumnAlias:"); for (Map.Entry<ASTNode, String> entry : parseInfo.getAllExprToColumnAlias().entrySet()) { - builder.append("\n\t expr: " + entry.getKey().dump() + " ColumnAlias: " + entry.getValue()); + builder.append("\n\t expr: ").append(entry.getKey().dump()).append(" ColumnAlias: ").append(entry.getValue()); } } if (parseInfo.getAggregationExprsForClause(clause) != null) { builder.append("\n aggregateexprs:"); for (Map.Entry<String, ASTNode> entry : parseInfo.getAggregationExprsForClause(clause).entrySet()) { - builder.append("\n\t key: " + entry.getKey() + " expr: " + entry.getValue().dump()); + builder.append("\n\t key: ").append(entry.getKey()).append(" expr: ").append(entry.getValue().dump()); } } if (parseInfo.getDistinctFuncExprsForClause(clause) != null) { builder.append("\n distinctFuncExprs:"); for (ASTNode entry : parseInfo.getDistinctFuncExprsForClause(clause)) { - builder.append("\n\t expr: " + entry.dump()); + builder.append("\n\t expr: ").append(entry.dump()); } } @@ -609,24 +607,24 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { } if (qb.getParseInfo().getDestForClause(clause) != null) { - builder.append("\n Destination:"); - builder.append("\n\t dest expr:" + qb.getParseInfo().getDestForClause(clause).dump()); + builder.append("\n Destination:") + .append("\n\t dest expr:").append(qb.getParseInfo().getDestForClause(clause).dump()); } log.debug(builder.toString()); } - void printJoinTree(QBJoinTree joinTree, StringBuilder builder) { - builder.append("leftAlias:" + joinTree.getLeftAlias()); + private void printJoinTree(QBJoinTree joinTree, StringBuilder builder) { + builder.append("leftAlias:").append(joinTree.getLeftAlias()); if (joinTree.getLeftAliases() != null) { builder.append("\n leftAliases:"); for (String alias : joinTree.getLeftAliases()) { - builder.append("\n\t " + alias); + builder.append("\n\t ").append(alias); } } if (joinTree.getRightAliases() != null) { builder.append("\n rightAliases:"); for (String alias : joinTree.getRightAliases()) { - builder.append("\n\t " + alias); + builder.append("\n\t ").append(alias); } } if (joinTree.getJoinSrc() != null) { @@ -637,22 +635,24 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { if (joinTree.getBaseSrc() != null) { builder.append("\n baseSrcs:"); for (String src : joinTree.getBaseSrc()) { - builder.append("\n\t " + src); + builder.append("\n\t ").append(src); } } - builder.append("\n noOuterJoin: " + joinTree.getNoOuterJoin()); - builder.append("\n noSemiJoin: " + joinTree.getNoSemiJoin()); - builder.append("\n mapSideJoin: " + joinTree.isMapSideJoin()); + builder.append("\n noOuterJoin: ").append(joinTree.getNoOuterJoin()); + builder.append("\n noSemiJoin: ").append(joinTree.getNoSemiJoin()); + builder.append("\n mapSideJoin: ").append(joinTree.isMapSideJoin()); if (joinTree.getJoinCond() != null) { builder.append("\n joinConds:"); for (JoinCond cond : joinTree.getJoinCond()) { - builder.append("\n\t left: " + cond.getLeft() + " right: " + cond.getRight() + " type:" + cond.getJoinType() - + " preserved:" + cond.getPreserved()); + builder.append("\n\t left: ").append(cond.getLeft()) + .append(" right: ").append(cond.getRight()) + .append(" type:").append(cond.getJoinType()) + .append(" preserved:").append(cond.getPreserved()); } } } - void updateFromString(StorageCandidate sc, Map<Dimension, CandidateDim> dimsToQuery) throws LensException { + private void updateFromString(StorageCandidate sc, Map<Dimension, CandidateDim> dimsToQuery) throws LensException { fromString = "%s"; // storage string is updated later if (isAutoJoinResolved()) { fromString = @@ -816,7 +816,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { } } - public String getNonExistingParts() { + String getNonExistingParts() { return conf.get(NON_EXISTING_PARTITIONS); } @@ -830,24 +830,13 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { cdim.dimtable); dimsToQuery.put(dim, cdim); } else { - String reason = ""; if (dimPruningMsgs.get(dim) != null && !dimPruningMsgs.get(dim).isEmpty()) { - ByteArrayOutputStream out = null; - try { + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { ObjectMapper mapper = new ObjectMapper(); - out = new ByteArrayOutputStream(); mapper.writeValue(out, dimPruningMsgs.get(dim).getJsonObject()); - reason = out.toString("UTF-8"); + log.info("No candidate dim found because: {}", out.toString("UTF-8")); } catch (Exception e) { throw new LensException("Error writing dim pruning messages", e); - } finally { - if (out != null) { - try { - out.close(); - } catch (IOException e) { - throw new LensException(e); - } - } } } log.error("Query rewrite failed due to NO_CANDIDATE_DIM_AVAILABLE, Cause {}", @@ -866,24 +855,13 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { cand = candidates.iterator().next(); log.info("Available Candidates:{}, picking up Candaidate: {} for querying", candidates, cand); } else { - String reason = ""; if (!storagePruningMsgs.isEmpty()) { - ByteArrayOutputStream out = null; - try { + try(ByteArrayOutputStream out = new ByteArrayOutputStream()) { ObjectMapper mapper = new ObjectMapper(); - out = new ByteArrayOutputStream(); mapper.writeValue(out, storagePruningMsgs.getJsonObject()); - reason = out.toString("UTF-8"); + log.info("No candidate found because: {}", out.toString("UTF-8")); } catch (Exception e) { throw new LensException("Error writing fact pruning messages", e); - } finally { - if (out != null) { - try { - out.close(); - } catch (IOException e) { - throw new LensException(e); - } - } } } log.error("Query rewrite failed due to NO_CANDIDATE_FACT_AVAILABLE, Cause {}", @@ -946,13 +924,13 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { Set<Dimension> exprDimensions = new HashSet<>(); if (!scSet.isEmpty()) { for (StorageCandidate sc : scSet) { - Set<Dimension> factExprDimTables = exprCtx.rewriteExprCtx(sc, dimsToQuery, sc.getQueryAst()); + Set<Dimension> factExprDimTables = exprCtx.rewriteExprCtx(this, sc, dimsToQuery, sc.getQueryAst()); exprDimensions.addAll(factExprDimTables); factDimMap.get(sc).addAll(factExprDimTables); } } else { // dim only query - exprDimensions.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, this)); + exprDimensions.addAll(exprCtx.rewriteExprCtx(this, null, dimsToQuery, this)); } dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimensions)); log.info("StorageCandidates: {}, DimsToQuery: {}", scSet, dimsToQuery); @@ -961,12 +939,12 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { Set<Dimension> denormTables = new HashSet<>(); if (!scSet.isEmpty()) { for (StorageCandidate sc : scSet) { - Set<Dimension> factDenormTables = deNormCtx.rewriteDenormctx(sc, dimsToQuery, !scSet.isEmpty()); + Set<Dimension> factDenormTables = deNormCtx.rewriteDenormctx(this, sc, dimsToQuery, !scSet.isEmpty()); denormTables.addAll(factDenormTables); factDimMap.get(sc).addAll(factDenormTables); } } else { - denormTables.addAll(deNormCtx.rewriteDenormctx(null, dimsToQuery, false)); + denormTables.addAll(deNormCtx.rewriteDenormctx(this, null, dimsToQuery, false)); } dimsToQuery.putAll(pickCandidateDimsToQuery(denormTables)); log.info("StorageCandidates: {}, DimsToQuery: {}", scSet, dimsToQuery); @@ -1000,7 +978,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { updateFromString(null, dimsToQuery); } //update dim filter with fact filter - if (scSet != null && scSet.size() > 0) { + if (scSet.size() > 0) { for (StorageCandidate sc : scSet) { if (!sc.getStorageName().isEmpty()) { String qualifiedStorageTable = sc.getStorageName(); @@ -1039,18 +1017,18 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { return ParseUtils.findRootNonNullToken(tree); } - public Set<String> getColumnsQueriedForTable(String tblName) { + Set<String> getColumnsQueriedForTable(String tblName) { return getColumnsQueried(getAliasForTableName(tblName)); } - public void addColumnsQueriedWithTimeDimCheck(QueriedPhraseContext qur, String alias, String timeDimColumn) { + void addColumnsQueriedWithTimeDimCheck(QueriedPhraseContext qur, String alias, String timeDimColumn) { if (!shouldReplaceTimeDimWithPart()) { qur.addColumnsQueried(alias, timeDimColumn); } } - public boolean isCubeMeasure(String col) { + boolean isCubeMeasure(String col) { if (col == null) { return false; } @@ -1100,6 +1078,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { ASTNode colIdent = (ASTNode) node.getChild(1); colname = colIdent.getText(); + assert tabident != null; tabname = tabident.getText(); } @@ -1108,7 +1087,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { return isCubeMeasure(msrname); } - public boolean hasAggregates() { + boolean hasAggregates() { if (getExprCtx().hasAggregates()) { return true; } @@ -1120,7 +1099,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { return false; } - public void setJoinCond(QBJoinTree qb, String cond) { + void setJoinCond(QBJoinTree qb, String cond) { joinConds.put(qb, cond); } @@ -1136,7 +1115,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { return null; } - public String getInsertClause() { + String getInsertClause() { ASTNode destTree = qb.getParseInfo().getDestForClause(clauseName); if (destTree != null && ((ASTNode) (destTree.getChild(0))).getToken().getType() != TOK_TMP_FILE) { return "INSERT OVERWRITE " + HQLParser.getString(destTree) + " "; @@ -1144,14 +1123,14 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { return ""; } - public Set<Aliased<Dimension>> getOptionalDimensions() { + Set<Aliased<Dimension>> getOptionalDimensions() { return optionalDimensionMap.keySet(); } /** * @return the hqlContext */ - public HQLContextInterface getHqlContext() { + HQLContextInterface getHqlContext() { return hqlContext; } @@ -1159,15 +1138,15 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { return getConf().getBoolean(REPLACE_TIMEDIM_WITH_PART_COL, DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL); } - public boolean shouldReplaceDimFilterWithFactFilter() { + private boolean shouldReplaceDimFilterWithFactFilter() { return getConf().getBoolean(REWRITE_DIM_FILTER_TO_FACT_FILTER, DEFAULT_REWRITE_DIM_FILTER_TO_FACT_FILTER); } - public String getPartitionColumnOfTimeDim(String timeDimName) { + String getPartitionColumnOfTimeDim(String timeDimName) { return getPartitionColumnOfTimeDim(cube, timeDimName); } - public static String getPartitionColumnOfTimeDim(CubeInterface cube, String timeDimName) { + private static String getPartitionColumnOfTimeDim(CubeInterface cube, String timeDimName) { if (cube == null) { return timeDimName; } @@ -1178,11 +1157,11 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { } } - public String getTimeDimOfPartitionColumn(String partCol) { + String getTimeDimOfPartitionColumn(String partCol) { return getTimeDimOfPartitionColumn(cube, partCol); } - public static String getTimeDimOfPartitionColumn(CubeInterface cube, String partCol) { + private static String getTimeDimOfPartitionColumn(CubeInterface cube, String partCol) { if (cube == null) { return partCol; } @@ -1193,25 +1172,25 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { } } - public void addQueriedMsrs(Set<String> msrs) { + void addQueriedMsrs(Set<String> msrs) { queriedMsrs.addAll(msrs); } - public void addQueriedExprs(Set<String> exprs) { + void addQueriedExprs(Set<String> exprs) { queriedExprs.addAll(exprs); } - public void addQueriedExprsWithMeasures(Set<String> exprs) { + void addQueriedExprsWithMeasures(Set<String> exprs) { queriedExprsWithMeasures.addAll(exprs); } - public void addQueriedTimeDimensionCols(final String timeDimColName) { + void addQueriedTimeDimensionCols(final String timeDimColName) { checkArgument(StringUtils.isNotBlank(timeDimColName)); this.queriedTimeDimCols.add(timeDimColName); } - public ImmutableSet<String> getQueriedTimeDimCols() { + ImmutableSet<String> getQueriedTimeDimCols() { return ImmutableSet.copyOf(this.queriedTimeDimCols); } @@ -1230,7 +1209,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { return whereString; } - private List<String> getAllFilters(ASTNode node, String cubeAlias, List<String> allFilters, + private void getAllFilters(ASTNode node, String cubeAlias, List<String> allFilters, JoinClause joinClause, Map<Dimension, CandidateDim> dimToQuery) throws LensException { @@ -1255,7 +1234,6 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { ASTNode child = (ASTNode) node.getChild(i); getAllFilters(child, cubeAlias, allFilters, joinClause, dimToQuery); } - return allFilters; } private String getFilter(String table, String cubeAlias, ASTNode node, JoinClause joinClause, @@ -1273,7 +1251,6 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { } private TableRelationship getStarJoin(JoinClause joinClause, String table) { - TableRelationship rel; for (Map.Entry<TableRelationship, JoinTree> entry : joinClause.getJoinTree().getSubtrees().entrySet()) { if (entry.getValue().getDepthFromRoot() == 1 && table.equals(entry.getValue().getAlias())) { return entry.getKey(); @@ -1285,8 +1262,9 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { private String getTableFromFilterAST(ASTNode node) { if (node.getToken().getType() == HiveParser.DOT) { - return HQLParser.findNodeByPath((ASTNode) node, - TOK_TABLE_OR_COL, Identifier).getText(); + ASTNode n = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier); + assert n != null; + return n.getText(); } else { // recurse down for (int i = 0; i < node.getChildCount(); i++) { @@ -1305,7 +1283,8 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { String cubeAlias) throws LensException { StringBuilder builder = new StringBuilder(); - String storageClause = dimToQuery.get(tabRelation.getToTable()).getWhereClause(); + CandidateDim dim = dimToQuery.get(tabRelation.getToTable()); + String storageClause = dim.getWhereClause(); builder.append(cubeAlias) .append(".") @@ -1314,9 +1293,9 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { .append("select ") .append(tabRelation.getToColumn()) .append(" from ") - .append(dimToQuery.get(tabRelation.getToTable()).getStorageString(dimAlias)) + .append(dim.getStorageString(dimAlias)) .append(" where ") - .append(HQLParser.getString((ASTNode) dimFilter)); + .append(HQLParser.getString(dimFilter)); if (storageClause != null) { builder.append(" and ") .append(String.format(storageClause, dimAlias)) http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java index f052a2f..5713069 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java @@ -139,9 +139,9 @@ public class CubeQueryRewriter { // Rewrite base trees (groupby, having, orderby, limit) using aliases rewriters.add(new AliasReplacer(conf)); - ExpressionResolver exprResolver = new ExpressionResolver(conf); - DenormalizationResolver denormResolver = new DenormalizationResolver(conf); - CandidateTableResolver candidateTblResolver = new CandidateTableResolver(conf); + ExpressionResolver exprResolver = new ExpressionResolver(); + DenormalizationResolver denormResolver = new DenormalizationResolver(); + CandidateTableResolver candidateTblResolver = new CandidateTableResolver(); StorageTableResolver storageTableResolver = new StorageTableResolver(conf); // Phase 1 of exprResolver: Resolve expressions @@ -160,7 +160,7 @@ public class CubeQueryRewriter { // Resolve joins and generate base join tree rewriters.add(new JoinResolver(conf)); // Do col life validation for the time range(s) queried - rewriters.add(new TimeRangeChecker(conf)); + rewriters.add(new ColumnLifetimeChecker()); // Phase 1 of storageTableResolver: Validate and prune candidate storages rewriters.add(storageTableResolver); // Phase 2 of candidateTblResolver: Resolve candidate storages and dimension tables for columns included @@ -168,7 +168,7 @@ public class CubeQueryRewriter { rewriters.add(candidateTblResolver); // Find Union and Join combinations over Storage Candidates that can answer the queried time range(s) and all // queried measures - rewriters.add(new CandidateCoveringSetsResolver(conf)); + rewriters.add(new CandidateCoveringSetsResolver()); // If lightest fact first option is enabled for this driver (via lens.cube.query.pick.lightest.fact.first = true), // run LightestFactResolver and keep only the lighted combination(s) generated by CandidateCoveringSetsResolver @@ -209,7 +209,7 @@ public class CubeQueryRewriter { public CubeQueryContext rewrite(ASTNode astnode) throws LensException { CubeSemanticAnalyzer analyzer; try { - analyzer = new CubeSemanticAnalyzer(conf, hconf); + analyzer = new CubeSemanticAnalyzer(hconf); analyzer.analyze(astnode, qlCtx); } catch (SemanticException e) { throw new LensException(SYNTAX_ERROR.getLensErrorInfo(), e, e.getMessage()); http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java index fc96055..0e2ca82 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java @@ -33,16 +33,12 @@ import lombok.Getter; * Accepts cube query AST and rewrites into storage table query */ public class CubeSemanticAnalyzer extends SemanticAnalyzer { - private final Configuration queryConf; - private final HiveConf hiveConf; - private final List<ValidationRule> validationRules = new ArrayList<ValidationRule>(); + private final List<ValidationRule> validationRules = new ArrayList<>(); @Getter private QB cubeQB; - public CubeSemanticAnalyzer(Configuration queryConf, HiveConf hiveConf) throws SemanticException { + public CubeSemanticAnalyzer(HiveConf hiveConf) throws SemanticException { super(new QueryState(hiveConf)); - this.queryConf = queryConf; - this.hiveConf = hiveConf; setupRules(); } @@ -65,10 +61,6 @@ public class CubeSemanticAnalyzer extends SemanticAnalyzer { ast.deleteChild(ast.getChildCount() - 1); } } - // analyzing from the ASTNode. - if (!doPhase1(ast, cubeQB, initPhase1Ctx(), null)) { - // if phase1Result false return - return; - } + doPhase1(ast, cubeQB, initPhase1Ctx(), null); } } http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java index bb29034..bcea7ed 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,18 +20,17 @@ package org.apache.lens.cube.parse; import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TABLE_OR_COL; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.denormColumnNotFound; import java.util.*; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.metadata.ReferencedDimAttribute.ChainRefCol; -import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; import org.apache.lens.cube.parse.ExpressionResolver.ExprSpecContext; import org.apache.lens.cube.parse.ExpressionResolver.ExpressionContext; import org.apache.lens.server.api.error.LensException; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; @@ -50,11 +49,8 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class DenormalizationResolver implements ContextRewriter { - public DenormalizationResolver(Configuration conf) { - } - @ToString - public static class ReferencedQueriedColumn { + static class ReferencedQueriedColumn { ReferencedDimAttribute col; AbstractCubeTable srcTable; transient List<ChainRefCol> chainRefCols = new ArrayList<>(); @@ -67,17 +63,12 @@ public class DenormalizationResolver implements ContextRewriter { } @ToString - public static class PickedReference { + static class PickedReference { @Getter ChainRefCol chainRef; String srcAlias; String pickedFor; - PickedReference(String srcAlias, String pickedFor) { - this.srcAlias = srcAlias; - this.pickedFor = pickedFor; - } - PickedReference(ChainRefCol chainRef, String srcAlias, String pickedFor) { this.srcAlias = srcAlias; this.chainRef = chainRef; @@ -85,38 +76,29 @@ public class DenormalizationResolver implements ContextRewriter { } } - public static class DenormalizationContext { + static class DenormalizationContext { // map of column name to all references + @Getter private Map<String, Set<ReferencedQueriedColumn>> referencedCols = new HashMap<>(); // candidate table name to all the references columns it needs + @Getter private Map<String, Set<ReferencedQueriedColumn>> tableToRefCols = new HashMap<>(); - private CubeQueryContext cubeql; - // set of all picked references once all candidate tables are picked private Set<PickedReference> pickedRefs = new HashSet<>(); // index on column name for picked references with map from column name to // pickedrefs private Map<String, Set<PickedReference>> pickedReferences = new HashMap<>(); - DenormalizationContext(CubeQueryContext cubeql) { - this.cubeql = cubeql; - } - void addReferencedCol(String col, ReferencedQueriedColumn refer) { - Set<ReferencedQueriedColumn> refCols = referencedCols.get(col); - if (refCols == null) { - refCols = new HashSet<>(); - referencedCols.put(col, refCols); - } - refCols.add(refer); + referencedCols.computeIfAbsent(col, k -> new HashSet<>()).add(refer); } // When candidate table does not have the field, this method checks // if the field can be reached through reference, // if yes adds the ref usage and returns to true, if not returns false. - boolean addRefUsage(CandidateTable table, String col, String srcTbl) throws LensException { + boolean addRefUsage(CubeQueryContext cubeql, CandidateTable table, String col, String srcTbl) throws LensException { // available as referenced col if (referencedCols.containsKey(col)) { for (ReferencedQueriedColumn refer : referencedCols.get(col)) { @@ -126,12 +108,7 @@ public class DenormalizationResolver implements ContextRewriter { // there is no path // to the source table log.info("Adding denormalized column for column:{} for table:{}", col, table); - Set<ReferencedQueriedColumn> refCols = tableToRefCols.get(table.getName()); - if (refCols == null) { - refCols = new HashSet<>(); - tableToRefCols.put(table.getName(), refCols); - } - refCols.add(refer); + tableToRefCols.computeIfAbsent(table.getName(), k -> new HashSet<>()).add(refer); // Add to optional tables for (ChainRefCol refCol : refer.col.getChainRefColumns()) { cubeql.addOptionalDimTable(refCol.getChainName(), table, false, refer.col.getName(), true, @@ -144,17 +121,8 @@ public class DenormalizationResolver implements ContextRewriter { return false; } - Map<String, Set<ReferencedQueriedColumn>> getReferencedCols() { - return referencedCols; - } - private void addPickedReference(String col, PickedReference refer) { - Set<PickedReference> refCols = pickedReferences.get(col); - if (refCols == null) { - refCols = new HashSet<>(); - pickedReferences.put(col, refCols); - } - refCols.add(refer); + pickedReferences.computeIfAbsent(col, k -> new HashSet<>()).add(refer); } private PickedReference getPickedReference(String col, String srcAlias) { @@ -169,24 +137,23 @@ public class DenormalizationResolver implements ContextRewriter { return null; } - public Set<Dimension> rewriteDenormctx(StorageCandidate sc, Map<Dimension, CandidateDim> dimsToQuery, - boolean replaceFact) throws LensException { + Set<Dimension> rewriteDenormctx(CubeQueryContext cubeql, + StorageCandidate sc, Map<Dimension, CandidateDim> dimsToQuery, boolean replaceFact) throws LensException { Set<Dimension> refTbls = new HashSet<>(); if (!tableToRefCols.isEmpty()) { // pick referenced columns for fact if (sc != null) { - pickColumnsForTable(sc.getName()); + pickColumnsForTable(cubeql, sc.getName()); } // pick referenced columns for dimensions - if (dimsToQuery != null && !dimsToQuery.isEmpty()) { + if (dimsToQuery != null) { for (CandidateDim cdim : dimsToQuery.values()) { - pickColumnsForTable(cdim.getName()); + pickColumnsForTable(cubeql, cdim.getName()); } } // Replace picked reference in all the base trees - replaceReferencedColumns(sc, replaceFact); - + replaceReferencedColumns(cubeql, sc, replaceFact); // Add the picked references to dimsToQuery for (PickedReference picked : pickedRefs) { if (isPickedFor(picked, sc, dimsToQuery)) { @@ -195,9 +162,43 @@ public class DenormalizationResolver implements ContextRewriter { } } } + pickedReferences.clear(); + pickedRefs.clear(); return refTbls; } + boolean hasReferences() { + return !tableToRefCols.isEmpty(); + } + Set<Dimension> rewriteDenormctxInExpression(CubeQueryContext cubeql, StorageCandidate sc, Map<Dimension, + CandidateDim> dimsToQuery, ASTNode exprAST) throws LensException { + Set<Dimension> refTbls = new HashSet<>(); + if (!tableToRefCols.isEmpty()) { + // pick referenced columns for fact + if (sc != null) { + pickColumnsForTable(cubeql, sc.getName()); + } + // pick referenced columns for dimensions + if (dimsToQuery != null) { + for (CandidateDim cdim : dimsToQuery.values()) { + pickColumnsForTable(cubeql, cdim.getName()); + } + } + // Replace picked reference in expression ast + resolveClause(exprAST); + + // Add the picked references to dimsToQuery + for (PickedReference picked : pickedRefs) { + if (isPickedFor(picked, sc, dimsToQuery)) { + refTbls.add((Dimension) cubeql.getCubeTableForAlias(picked.getChainRef().getChainName())); + cubeql.addColumnsQueried(picked.getChainRef().getChainName(), picked.getChainRef().getRefColumn()); + } + } + } + pickedReferences.clear(); + pickedRefs.clear(); + return refTbls; + } // checks if the reference if picked for facts and dimsToQuery passed private boolean isPickedFor(PickedReference picked, StorageCandidate sc, Map<Dimension, CandidateDim> dimsToQuery) { if (sc != null && picked.pickedFor.equalsIgnoreCase(sc.getName())) { @@ -213,18 +214,12 @@ public class DenormalizationResolver implements ContextRewriter { return false; } - private void pickColumnsForTable(String tbl) throws LensException { + private void pickColumnsForTable(CubeQueryContext cubeql, String tbl) throws LensException { if (tableToRefCols.containsKey(tbl)) { for (ReferencedQueriedColumn refered : tableToRefCols.get(tbl)) { - Iterator<ChainRefCol> iter = refered.chainRefCols.iterator(); - while (iter.hasNext()) { - // remove unreachable references - ChainRefCol reference = iter.next(); - if (!cubeql.getAutoJoinCtx().isReachableDim( - (Dimension) cubeql.getCubeTableForAlias(reference.getChainName()), reference.getChainName())) { - iter.remove(); - } - } + // remove unreachable references + refered.chainRefCols.removeIf(reference -> !cubeql.getAutoJoinCtx().isReachableDim( + (Dimension) cubeql.getCubeTableForAlias(reference.getChainName()), reference.getChainName())); if (refered.chainRefCols.isEmpty()) { throw new LensException(LensCubeErrorCode.NO_REF_COL_AVAILABLE.getLensErrorInfo(), refered.col.getName()); } @@ -236,26 +231,55 @@ public class DenormalizationResolver implements ContextRewriter { } } } + void pruneReferences(CubeQueryContext cubeql) { + for (Set<ReferencedQueriedColumn> referencedQueriedColumns : referencedCols.values()) { + for(Iterator<ReferencedQueriedColumn> iterator = referencedQueriedColumns.iterator(); iterator.hasNext();) { + ReferencedQueriedColumn rqc = iterator.next(); + for (Iterator<ChainRefCol> iter = rqc.chainRefCols.iterator(); iter.hasNext();) { + // remove unreachable references + ChainRefCol reference = iter.next(); + if (cubeql.getAutoJoinCtx() == null || !cubeql.getAutoJoinCtx().isReachableDim( + (Dimension) cubeql.getCubeTableForAlias(reference.getChainName()), reference.getChainName())) { + log.info("{} is not reachable", reference.getChainName()); + iter.remove(); + } + } + if (rqc.chainRefCols.isEmpty()) { + log.info("The referenced column: {} is not reachable", rqc.col.getName()); + iterator.remove(); + continue; + } + // do column life validation + for (TimeRange range : cubeql.getTimeRanges()) { + if (!rqc.col.isColumnAvailableInTimeRange(range)) { + log.info("The referenced column: {} is not in the range queried", rqc.col.getName()); + iterator.remove(); + break; + } + } + } + } + } - private void replaceReferencedColumns(StorageCandidate sc, boolean replaceFact) throws LensException { + private void replaceReferencedColumns(CubeQueryContext cubeql, StorageCandidate sc, boolean replaceFact) throws LensException { QueryAST ast = cubeql; boolean factRefExists = sc != null && tableToRefCols.get(sc.getName()) != null && !tableToRefCols.get(sc .getName()).isEmpty(); if (replaceFact && factRefExists) { ast = sc.getQueryAst(); } - resolveClause(cubeql, ast.getSelectAST()); + resolveClause(ast.getSelectAST()); if (factRefExists) { - resolveClause(cubeql, sc.getQueryAst().getWhereAST()); + resolveClause(sc.getQueryAst().getWhereAST()); } else { - resolveClause(cubeql, ast.getWhereAST()); + resolveClause(ast.getWhereAST()); } - resolveClause(cubeql, ast.getGroupByAST()); - resolveClause(cubeql, ast.getHavingAST()); - resolveClause(cubeql, cubeql.getOrderByAST()); + resolveClause(ast.getGroupByAST()); + resolveClause(ast.getHavingAST()); + resolveClause(cubeql.getOrderByAST()); } - private void resolveClause(CubeQueryContext query, ASTNode node) throws LensException { + private void resolveClause(ASTNode node) throws LensException { if (node == null) { return; } @@ -271,6 +295,7 @@ public class DenormalizationResolver implements ContextRewriter { ASTNode tableNode = (ASTNode) node.getChild(0); ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier); + assert tabident != null; PickedReference refered = getPickedReference(colName, tabident.getText().toLowerCase()); if (refered == null) { return; @@ -286,16 +311,29 @@ public class DenormalizationResolver implements ContextRewriter { // recurse down for (int i = 0; i < node.getChildCount(); i++) { ASTNode child = (ASTNode) node.getChild(i); - resolveClause(query, child); + resolveClause(child); + } + } + } + + Set<String> getNonReachableReferenceFields(String table) { + Set<String> nonReachableFields = new HashSet<>(); + if (tableToRefCols.containsKey(table)) { + for (ReferencedQueriedColumn refcol : tableToRefCols.get(table)) { + if (getReferencedCols().get(refcol.col.getName()).isEmpty()) { + log.info("For table:{}, the column {} is not available", table, refcol.col); + nonReachableFields.add(refcol.col.getName()); + } } } + return nonReachableFields; } } private void addRefColsQueried(CubeQueryContext cubeql, TrackQueriedColumns tqc, DenormalizationContext denormCtx) { for (Map.Entry<String, Set<String>> entry : tqc.getTblAliasToColumns().entrySet()) { // skip default alias - if (entry.getKey() == CubeQueryContext.DEFAULT_TABLE) { + if (Objects.equals(entry.getKey(), CubeQueryContext.DEFAULT_TABLE)) { continue; } // skip join chain aliases @@ -318,6 +356,14 @@ public class DenormalizationResolver implements ContextRewriter { } } } + private static DenormalizationContext getOrCreateDeNormCtx(TrackDenormContext tdc) { + DenormalizationContext denormCtx = tdc.getDeNormCtx(); + if (denormCtx == null) { + denormCtx = new DenormalizationContext(); + tdc.setDeNormCtx(denormCtx); + } + return denormCtx; + } /** * Find all de-normalized columns, if these columns are not directly available in candidate tables, query will be * replaced with the corresponding table reference @@ -326,38 +372,32 @@ public class DenormalizationResolver implements ContextRewriter { public void rewriteContext(CubeQueryContext cubeql) throws LensException { DenormalizationContext denormCtx = cubeql.getDeNormCtx(); if (denormCtx == null) { + DenormalizationContext ctx = getOrCreateDeNormCtx(cubeql); // Adds all the reference dimensions as eligible for denorm fields - denormCtx = new DenormalizationContext(cubeql); - cubeql.setDeNormCtx(denormCtx); // add ref columns in cube - addRefColsQueried(cubeql, cubeql, denormCtx); + addRefColsQueried(cubeql, cubeql, ctx); // add ref columns from expressions for (Set<ExpressionContext> ecSet : cubeql.getExprCtx().getAllExprsQueried().values()) { for (ExpressionContext ec : ecSet) { for (ExprSpecContext esc : ec.getAllExprs()) { - addRefColsQueried(cubeql, esc, denormCtx); + addRefColsQueried(cubeql, esc, getOrCreateDeNormCtx(esc)); } } } } else if (!denormCtx.tableToRefCols.isEmpty()) { + denormCtx.pruneReferences(cubeql); // In the second iteration of denorm resolver // candidate tables which require denorm fields and the refernces are no // more valid will be pruned if (cubeql.getCube() != null && !cubeql.getCandidates().isEmpty()) { for (Iterator<StorageCandidate> i = CandidateUtil.getStorageCandidates(cubeql.getCandidates()).iterator(); i.hasNext();) { - StorageCandidate sc = i.next(); - if (denormCtx.tableToRefCols.containsKey(sc.getFact().getName())) { - for (ReferencedQueriedColumn refcol : denormCtx.tableToRefCols.get(sc.getFact().getName())) { - if (denormCtx.getReferencedCols().get(refcol.col.getName()).isEmpty()) { - log.info("Not considering storage candidate :{} as column {} is not available", sc, refcol.col); - cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.columnNotFound( - CandidateTablePruneCode.DENORM_COLUMN_NOT_FOUND, refcol.col.getName())); - Collection<Candidate> prunedCandidates = CandidateUtil.filterCandidates(cubeql.getCandidates(), sc); - cubeql.addCandidatePruningMsg(prunedCandidates, - new CandidateTablePruneCause(CandidateTablePruneCode.ELEMENT_IN_SET_PRUNED)); - } - } + StorageCandidate candidate = i.next(); + Set<String> nonReachableFields = denormCtx.getNonReachableReferenceFields(candidate.getName()); + if (!nonReachableFields.isEmpty()) { + log.info("Not considering fact table:{} as columns {} are not available", candidate, nonReachableFields); + cubeql.addCandidatePruningMsg(candidate, denormColumnNotFound(nonReachableFields)); + i.remove(); } } if (cubeql.getCandidates().size() == 0) { @@ -370,16 +410,11 @@ public class DenormalizationResolver implements ContextRewriter { for (Dimension dim : cubeql.getDimensions()) { for (Iterator<CandidateDim> i = cubeql.getCandidateDimTables().get(dim).iterator(); i.hasNext();) { CandidateDim cdim = i.next(); - if (denormCtx.tableToRefCols.containsKey(cdim.getName())) { - for (ReferencedQueriedColumn refcol : denormCtx.tableToRefCols.get(cdim.getName())) { - if (denormCtx.getReferencedCols().get(refcol.col.getName()).isEmpty()) { - log.info("Not considering dim table:{} as column {} is not available", cdim, refcol.col); - cubeql.addDimPruningMsgs(dim, cdim.dimtable, - CandidateTablePruneCause.columnNotFound(CandidateTablePruneCode.DENORM_COLUMN_NOT_FOUND, - refcol.col.getName())); - i.remove(); - } - } + Set<String> nonReachableFields = denormCtx.getNonReachableReferenceFields(cdim.getName()); + if (!nonReachableFields.isEmpty()) { + log.info("Not considering dim table:{} as column {} is not available", cdim, nonReachableFields); + cubeql.addDimPruningMsgs(dim, cdim.dimtable, denormColumnNotFound(nonReachableFields)); + i.remove(); } }
