http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java index 125b432..c63c4c0 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,24 +19,25 @@ package org.apache.lens.cube.parse; + +import static java.util.stream.Collectors.toSet; + import static org.apache.lens.cube.parse.CubeQueryConfUtil.*; import static org.apache.hadoop.hive.ql.parse.HiveParser.*; import static com.google.common.base.Preconditions.checkArgument; - - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.*; +import java.util.function.Predicate; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.error.NoCandidateDimAvailableException; import org.apache.lens.cube.error.NoCandidateFactAvailableException; import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.metadata.join.TableRelationship; -import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; import org.apache.lens.cube.parse.join.AutoJoinContext; import org.apache.lens.cube.parse.join.JoinClause; import org.apache.lens.cube.parse.join.JoinTree; @@ -47,8 +48,10 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.*; +import org.apache.hadoop.util.ReflectionUtils; import org.codehaus.jackson.map.ObjectMapper; @@ -60,9 +63,9 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, TrackDenormContext { - public static final String TIME_RANGE_FUNC = "time_range_in"; + static final String TIME_RANGE_FUNC = "time_range_in"; public static final String NOW = "now"; - public static final String DEFAULT_TABLE = "_default_"; + static final String DEFAULT_TABLE = "_default_"; private final ASTNode ast; @Getter private final QB qb; @@ -102,8 +105,14 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, // Mapping of a qualified column name to its table alias private final Map<String, String> colToTableAlias = new HashMap<>(); + /** + * This is the set of working Candidates that gets updated during different phases of + * query resolution. Each {@link ContextRewriter} may add/remove/update Candiadtes in + * this working set and from the final set of Candidates single {@link #pickedCandidate} + * is chosen. + */ @Getter - private final Set<Set<CandidateFact>> candidateFactSets = new HashSet<>(); + private final Set<Candidate> candidates = new HashSet<>(); @Getter // would be added through join chains and de-normalized resolver @@ -119,17 +128,14 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, } boolean isColumnAnAlias(String col) { - for (SelectPhraseContext sel : selectPhrases) { - if (col.equals(sel.getActualAlias())) { - return true; - } - } - return false; + return selectPhrases.stream().map(SelectPhraseContext::getActualAlias).anyMatch(Predicate.isEqual(col)); } void addQueriedPhrase(QueriedPhraseContext qur) { queriedPhrases.add(qur); + qur.setPosition(queriedPhrases.size() -1); } + @Getter private final List<SelectPhraseContext> selectPhrases = new ArrayList<>(); @@ -138,14 +144,9 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, // Join conditions used in all join expressions @Getter - private final Map<QBJoinTree, String> joinConds = new HashMap<QBJoinTree, String>(); - - // storage specific - @Getter - protected final Set<CandidateFact> candidateFacts = new HashSet<CandidateFact>(); + private final Map<QBJoinTree, String> joinConds = new HashMap<>(); @Getter - protected final Map<Dimension, Set<CandidateDim>> candidateDims = new HashMap<Dimension, Set<CandidateDim>>(); - + protected final Map<Dimension, Set<CandidateDim>> candidateDims = new HashMap<>(); // query trees @Getter @Setter @@ -178,20 +179,22 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, @Setter private DenormalizationResolver.DenormalizationContext deNormCtx; @Getter - private PruneCauses<CubeFactTable> factPruningMsgs = - new PruneCauses<CubeFactTable>(); + private PruneCauses<StorageCandidate> storagePruningMsgs = new PruneCauses<>(); @Getter private Map<Dimension, PruneCauses<CubeDimensionTable>> dimPruningMsgs = new HashMap<Dimension, PruneCauses<CubeDimensionTable>>(); + @Setter @Getter private String fromString; + @Getter + private TimeRangeWriter rangeWriter = null; public CubeQueryContext(ASTNode ast, QB qb, Configuration queryConf, HiveConf metastoreConf) throws LensException { this.ast = ast; this.qb = qb; this.conf = queryConf; this.clauseName = getClause(); - this.timeRanges = new ArrayList<TimeRange>(); + this.timeRanges = new ArrayList<>(); try { metastoreClient = CubeMetastoreClient.getInstance(metastoreConf); } catch (HiveException e) { @@ -212,16 +215,14 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, if (qb.getParseInfo().getSelForClause(clauseName) != null) { this.selectAST = qb.getParseInfo().getSelForClause(clauseName); } - extractMetaTables(); - } - public boolean hasCubeInQuery() { - return cube != null; + this.rangeWriter = ReflectionUtils.newInstance(conf.getClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, + CubeQueryConfUtil.DEFAULT_TIME_RANGE_WRITER, TimeRangeWriter.class), conf); } - public boolean hasDimensionInQuery() { - return dimensions != null && !dimensions.isEmpty(); + boolean hasCubeInQuery() { + return cube != null; } private void extractMetaTables() throws LensException { @@ -279,10 +280,10 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, return true; } - return retVal; + return false; } - public boolean addQueriedTable(String alias) throws LensException { + boolean addQueriedTable(String alias) throws LensException { return addQueriedTable(alias, false); } @@ -340,42 +341,29 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, return false; } } catch (LensException e) { - //TODO: check if catch can be removed return false; } return true; } - public boolean isAutoJoinResolved() { + boolean isAutoJoinResolved() { return autoJoinCtx != null && autoJoinCtx.isJoinsResolved(); } - public Cube getBaseCube() { - if (cube instanceof Cube) { - return (Cube) cube; - } - return ((DerivedCube) cube).getParent(); + Cube getBaseCube() { + return cube instanceof Cube ? (Cube) cube : ((DerivedCube) cube).getParent(); } - public Set<String> getPartitionColumnsQueried() { - Set<String> partsQueried = Sets.newHashSet(); - for (TimeRange range : getTimeRanges()) { - partsQueried.add(range.getPartitionColumn()); - } - return partsQueried; + Set<String> getPartitionColumnsQueried() { + return getTimeRanges().stream().map(TimeRange::getPartitionColumn).collect(toSet()); } // map of ref column in query to set of Dimension that have the column - which are added as optional dims @Getter private Map<String, Set<Aliased<Dimension>>> refColToDim = Maps.newHashMap(); - public void updateRefColDim(String col, Aliased<Dimension> dim) { - Set<Aliased<Dimension>> refDims = refColToDim.get(col.toLowerCase()); - if (refDims == null) { - refDims = Sets.newHashSet(); - refColToDim.put(col.toLowerCase(), refDims); - } - refDims.add(dim); + private void updateRefColDim(String col, Aliased<Dimension> dim) { + refColToDim.computeIfAbsent(col.toLowerCase(), k -> Sets.newHashSet()).add(dim); } @Data @@ -390,15 +378,8 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, @Getter private Map<QueriedExprColumn, Set<Aliased<Dimension>>> exprColToDim = Maps.newHashMap(); - public void updateExprColDim(String tblAlias, String col, Aliased<Dimension> dim) { - - QueriedExprColumn qexpr = new QueriedExprColumn(col, tblAlias); - Set<Aliased<Dimension>> exprDims = exprColToDim.get(qexpr); - if (exprDims == null) { - exprDims = Sets.newHashSet(); - exprColToDim.put(qexpr, exprDims); - } - exprDims.add(dim); + private void updateExprColDim(String tblAlias, String col, Aliased<Dimension> dim) { + exprColToDim.computeIfAbsent(new QueriedExprColumn(col, tblAlias), k -> Sets.newHashSet()).add(dim); } // Holds the context of optional dimension @@ -406,7 +387,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, // required by a candidate table to get a denormalized field from reference // or required in a join chain @ToString - public static class OptionalDimCtx { + static class OptionalDimCtx { OptionalDimCtx() { } @@ -415,16 +396,16 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, boolean isRequiredInJoinChain = false; } - public void addOptionalJoinDimTable(String alias, boolean isRequired) throws LensException { + void addOptionalJoinDimTable(String alias, boolean isRequired) throws LensException { addOptionalDimTable(alias, null, isRequired, null, false, (String[]) null); } - public void addOptionalExprDimTable(String dimAlias, String queriedExpr, String srcTableAlias, + void addOptionalExprDimTable(String dimAlias, String queriedExpr, String srcTableAlias, CandidateTable candidate, String... cols) throws LensException { addOptionalDimTable(dimAlias, candidate, false, queriedExpr, false, srcTableAlias, cols); } - public void addOptionalDimTable(String alias, CandidateTable candidate, boolean isRequiredInJoin, String cubeCol, + void addOptionalDimTable(String alias, CandidateTable candidate, boolean isRequiredInJoin, String cubeCol, boolean isRef, String... cols) throws LensException { addOptionalDimTable(alias, candidate, isRequiredInJoin, cubeCol, isRef, null, cols); } @@ -437,15 +418,9 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, } Dimension dim = (Dimension) cubeTbls.get(alias); Aliased<Dimension> aliasedDim = Aliased.create(dim, alias); - OptionalDimCtx optDim = optionalDimensionMap.get(aliasedDim); - if (optDim == null) { - optDim = new OptionalDimCtx(); - optionalDimensionMap.put(aliasedDim, optDim); - } + OptionalDimCtx optDim = optionalDimensionMap.computeIfAbsent(aliasedDim, k -> new OptionalDimCtx()); if (cols != null && candidate != null) { - for (String col : cols) { - optDim.colQueried.add(col); - } + optDim.colQueried.addAll(Arrays.asList(cols)); optDim.requiredForCandidates.add(candidate); } if (cubeCol != null) { @@ -480,18 +455,28 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, return candidateDims; } - public void addFactPruningMsgs(CubeFactTable fact, CandidateTablePruneCause factPruningMsg) { - log.info("Pruning fact {} with cause: {}", fact, factPruningMsg); - factPruningMsgs.addPruningMsg(fact, factPruningMsg); + void addCandidatePruningMsg(Collection<Candidate> candidateCollection, CandidateTablePruneCause pruneCause) { + for (Candidate c : candidateCollection){ + addCandidatePruningMsg(c, pruneCause); + } } - public void addDimPruningMsgs(Dimension dim, CubeDimensionTable dimtable, CandidateTablePruneCause msg) { - PruneCauses<CubeDimensionTable> dimMsgs = dimPruningMsgs.get(dim); - if (dimMsgs == null) { - dimMsgs = new PruneCauses<CubeDimensionTable>(); - dimPruningMsgs.put(dim, dimMsgs); + void addCandidatePruningMsg(Candidate cand, CandidateTablePruneCause pruneCause) { + Set<StorageCandidate> scs = CandidateUtil.getStorageCandidates(cand); + for (StorageCandidate sc : scs) { + addStoragePruningMsg(sc, pruneCause); + } + } + + void addStoragePruningMsg(StorageCandidate sc, CandidateTablePruneCause... factPruningMsgs) { + for (CandidateTablePruneCause factPruningMsg: factPruningMsgs) { + log.info("Pruning Storage {} with cause: {}", sc, factPruningMsg); + storagePruningMsgs.addPruningMsg(sc, factPruningMsg); } - dimMsgs.addPruningMsg(dimtable, msg); + } + + public void addDimPruningMsgs(Dimension dim, CubeDimensionTable dimtable, CandidateTablePruneCause msg) { + dimPruningMsgs.computeIfAbsent(dim, k -> new PruneCauses<>()).addPruningMsg(dimtable, msg); } public String getAliasForTableName(Named named) { @@ -518,73 +503,75 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, if (!log.isDebugEnabled()) { return; } - StringBuilder builder = new StringBuilder(); - builder.append("ASTNode:" + ast.dump() + "\n"); - builder.append("QB:"); - builder.append("\n numJoins:" + qb.getNumJoins()); - builder.append("\n numGbys:" + qb.getNumGbys()); - builder.append("\n numSels:" + qb.getNumSels()); - builder.append("\n numSelDis:" + qb.getNumSelDi()); - builder.append("\n aliasToTabs:"); + StringBuilder builder = new StringBuilder() + .append("ASTNode:").append(ast.dump()).append("\n") + .append("QB:") + .append("\n numJoins:").append(qb.getNumJoins()) + .append("\n numGbys:").append(qb.getNumGbys()) + .append("\n numSels:").append(qb.getNumSels()) + .append("\n numSelDis:").append(qb.getNumSelDi()) + .append("\n aliasToTabs:"); Set<String> tabAliases = qb.getTabAliases(); for (String alias : tabAliases) { - builder.append("\n\t" + alias + ":" + qb.getTabNameForAlias(alias)); + builder.append("\n\t").append(alias).append(":").append(qb.getTabNameForAlias(alias)); } builder.append("\n aliases:"); for (String alias : qb.getAliases()) { builder.append(alias); builder.append(", "); } - builder.append("id:" + qb.getId()); - builder.append("isQuery:" + qb.getIsQuery()); - builder.append("\n QBParseInfo"); + builder + .append("id:").append(qb.getId()) + .append("isQuery:").append(qb.getIsQuery()) + .append("\n QBParseInfo"); QBParseInfo parseInfo = qb.getParseInfo(); - builder.append("\n isSubQ: " + parseInfo.getIsSubQ()); - builder.append("\n alias: " + parseInfo.getAlias()); + builder + .append("\n isSubQ: ").append(parseInfo.getIsSubQ()) + .append("\n alias: ").append(parseInfo.getAlias()); if (parseInfo.getJoinExpr() != null) { - builder.append("\n joinExpr: " + parseInfo.getJoinExpr().dump()); + builder.append("\n joinExpr: ").append(parseInfo.getJoinExpr().dump()); } - builder.append("\n hints: " + parseInfo.getHints()); + builder.append("\n hints: ").append(parseInfo.getHints()); builder.append("\n aliasToSrc: "); for (String alias : tabAliases) { - builder.append("\n\t" + alias + ": " + parseInfo.getSrcForAlias(alias).dump()); + builder.append("\n\t").append(alias).append(": ").append(parseInfo.getSrcForAlias(alias).dump()); } TreeSet<String> clauses = new TreeSet<String>(parseInfo.getClauseNames()); for (String clause : clauses) { - builder.append("\n\t" + clause + ": " + parseInfo.getClauseNamesForDest()); + builder.append("\n\t").append(clause).append(": ").append(parseInfo.getClauseNamesForDest()); } String clause = clauses.first(); if (parseInfo.getWhrForClause(clause) != null) { - builder.append("\n whereexpr: " + parseInfo.getWhrForClause(clause).dump()); + builder.append("\n whereexpr: ").append(parseInfo.getWhrForClause(clause).dump()); } if (parseInfo.getGroupByForClause(clause) != null) { - builder.append("\n groupby expr: " + parseInfo.getGroupByForClause(clause).dump()); + builder.append("\n groupby expr: ").append(parseInfo.getGroupByForClause(clause).dump()); } if (parseInfo.getSelForClause(clause) != null) { - builder.append("\n sel expr: " + parseInfo.getSelForClause(clause).dump()); + builder.append("\n sel expr: ").append(parseInfo.getSelForClause(clause).dump()); } if (parseInfo.getHavingForClause(clause) != null) { - builder.append("\n having expr: " + parseInfo.getHavingForClause(clause).dump()); + builder.append("\n having expr: ").append(parseInfo.getHavingForClause(clause).dump()); } if (parseInfo.getDestLimit(clause) != null) { - builder.append("\n limit: " + parseInfo.getDestLimit(clause)); + builder.append("\n limit: ").append(parseInfo.getDestLimit(clause)); } if (parseInfo.getAllExprToColumnAlias() != null && !parseInfo.getAllExprToColumnAlias().isEmpty()) { builder.append("\n exprToColumnAlias:"); for (Map.Entry<ASTNode, String> entry : parseInfo.getAllExprToColumnAlias().entrySet()) { - builder.append("\n\t expr: " + entry.getKey().dump() + " ColumnAlias: " + entry.getValue()); + builder.append("\n\t expr: ").append(entry.getKey().dump()).append(" ColumnAlias: ").append(entry.getValue()); } } if (parseInfo.getAggregationExprsForClause(clause) != null) { builder.append("\n aggregateexprs:"); for (Map.Entry<String, ASTNode> entry : parseInfo.getAggregationExprsForClause(clause).entrySet()) { - builder.append("\n\t key: " + entry.getKey() + " expr: " + entry.getValue().dump()); + builder.append("\n\t key: ").append(entry.getKey()).append(" expr: ").append(entry.getValue().dump()); } } if (parseInfo.getDistinctFuncExprsForClause(clause) != null) { builder.append("\n distinctFuncExprs:"); for (ASTNode entry : parseInfo.getDistinctFuncExprsForClause(clause)) { - builder.append("\n\t expr: " + entry.dump()); + builder.append("\n\t expr: ").append(entry.dump()); } } @@ -595,24 +582,24 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, } if (qb.getParseInfo().getDestForClause(clause) != null) { - builder.append("\n Destination:"); - builder.append("\n\t dest expr:" + qb.getParseInfo().getDestForClause(clause).dump()); + builder.append("\n Destination:") + .append("\n\t dest expr:").append(qb.getParseInfo().getDestForClause(clause).dump()); } log.debug(builder.toString()); } - void printJoinTree(QBJoinTree joinTree, StringBuilder builder) { - builder.append("leftAlias:" + joinTree.getLeftAlias()); + private void printJoinTree(QBJoinTree joinTree, StringBuilder builder) { + builder.append("leftAlias:").append(joinTree.getLeftAlias()); if (joinTree.getLeftAliases() != null) { builder.append("\n leftAliases:"); for (String alias : joinTree.getLeftAliases()) { - builder.append("\n\t " + alias); + builder.append("\n\t ").append(alias); } } if (joinTree.getRightAliases() != null) { builder.append("\n rightAliases:"); for (String alias : joinTree.getRightAliases()) { - builder.append("\n\t " + alias); + builder.append("\n\t ").append(alias); } } if (joinTree.getJoinSrc() != null) { @@ -623,26 +610,28 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, if (joinTree.getBaseSrc() != null) { builder.append("\n baseSrcs:"); for (String src : joinTree.getBaseSrc()) { - builder.append("\n\t " + src); + builder.append("\n\t ").append(src); } } - builder.append("\n noOuterJoin: " + joinTree.getNoOuterJoin()); - builder.append("\n noSemiJoin: " + joinTree.getNoSemiJoin()); - builder.append("\n mapSideJoin: " + joinTree.isMapSideJoin()); + builder.append("\n noOuterJoin: ").append(joinTree.getNoOuterJoin()); + builder.append("\n noSemiJoin: ").append(joinTree.getNoSemiJoin()); + builder.append("\n mapSideJoin: ").append(joinTree.isMapSideJoin()); if (joinTree.getJoinCond() != null) { builder.append("\n joinConds:"); for (JoinCond cond : joinTree.getJoinCond()) { - builder.append("\n\t left: " + cond.getLeft() + " right: " + cond.getRight() + " type:" + cond.getJoinType() - + " preserved:" + cond.getPreserved()); + builder.append("\n\t left: ").append(cond.getLeft()) + .append(" right: ").append(cond.getRight()) + .append(" type:").append(cond.getJoinType()) + .append(" preserved:").append(cond.getPreserved()); } } } - void updateFromString(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery) throws LensException { + private void updateFromString(StorageCandidate sc, Map<Dimension, CandidateDim> dimsToQuery) throws LensException { fromString = "%s"; // storage string is updated later if (isAutoJoinResolved()) { fromString = - getAutoJoinCtx().getFromString(fromString, fact, dimsToQuery.keySet(), dimsToQuery, this, this); + getAutoJoinCtx().getFromString(fromString, sc, dimsToQuery.keySet(), dimsToQuery, this, this); } } @@ -650,6 +639,11 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, return HQLParser.getString(selectAST); } + + public void setWhereString(String whereString) { + //NO OP + } + public String getWhereString() { if (whereAST != null) { return HQLParser.getString(whereAST); @@ -675,6 +669,11 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, return qb.getParseInfo().getJoinExpr(); } + @Override + public void setJoinAST(ASTNode node) { + //NO-OP + } + public String getOrderByString() { if (orderByAST != null) { return HQLParser.getString(orderByAST); @@ -690,9 +689,10 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, qb.getParseInfo().setDestLimit(getClause(), 0, value); } - private String getStorageStringWithAlias(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, String alias) { + private String getStorageStringWithAlias(StorageCandidate candidate, Map<Dimension, + CandidateDim> dimsToQuery, String alias) { if (cubeTbls.get(alias) instanceof CubeInterface) { - return fact.getStorageString(alias); + return candidate.getAliasForTable(alias); } else { return dimsToQuery.get(cubeTbls.get(alias)).getStorageString(alias); } @@ -702,14 +702,14 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, return StorageUtil.getWhereClause(dimsToQuery.get(cubeTbls.get(alias)), alias); } - String getQBFromString(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery) throws LensException { + String getQBFromString(StorageCandidate candidate, Map<Dimension, CandidateDim> dimsToQuery) throws LensException { String fromString; if (getJoinAST() == null) { if (cube != null) { if (dimensions.size() > 0) { throw new LensException(LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE.getLensErrorInfo()); } - fromString = fact.getStorageString(getAliasForTableName(cube.getName())); + fromString = candidate.getAliasForTable(getAliasForTableName(cube.getName())); } else { if (dimensions.size() != 1) { throw new LensException(LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE.getLensErrorInfo()); @@ -719,22 +719,23 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, } } else { StringBuilder builder = new StringBuilder(); - getQLString(qb.getQbJoinTree(), builder, fact, dimsToQuery); + getQLString(qb.getQbJoinTree(), builder, candidate, dimsToQuery); fromString = builder.toString(); } return fromString; } - private void getQLString(QBJoinTree joinTree, StringBuilder builder, CandidateFact fact, + + private void getQLString(QBJoinTree joinTree, StringBuilder builder, StorageCandidate candidate, Map<Dimension, CandidateDim> dimsToQuery) throws LensException { List<String> joiningTables = new ArrayList<>(); if (joinTree.getBaseSrc()[0] == null) { if (joinTree.getJoinSrc() != null) { - getQLString(joinTree.getJoinSrc(), builder, fact, dimsToQuery); + getQLString(joinTree.getJoinSrc(), builder, candidate, dimsToQuery); } } else { // (joinTree.getBaseSrc()[0] != null){ String alias = joinTree.getBaseSrc()[0].toLowerCase(); - builder.append(getStorageStringWithAlias(fact, dimsToQuery, alias)); + builder.append(getStorageStringWithAlias(candidate, dimsToQuery, alias)); joiningTables.add(alias); } if (joinTree.getJoinCond() != null) { @@ -743,11 +744,11 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, } if (joinTree.getBaseSrc()[1] == null) { if (joinTree.getJoinSrc() != null) { - getQLString(joinTree.getJoinSrc(), builder, fact, dimsToQuery); + getQLString(joinTree.getJoinSrc(), builder, candidate, dimsToQuery); } } else { // (joinTree.getBaseSrc()[1] != null){ String alias = joinTree.getBaseSrc()[1].toLowerCase(); - builder.append(getStorageStringWithAlias(fact, dimsToQuery, alias)); + builder.append(getStorageStringWithAlias(candidate, dimsToQuery, alias)); joiningTables.add(alias); } @@ -795,7 +796,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, } } - public String getNonExistingParts() { + String getNonExistingParts() { return conf.get(NON_EXISTING_PARTITIONS); } @@ -809,24 +810,13 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, cdim.dimtable); dimsToQuery.put(dim, cdim); } else { - String reason = ""; if (dimPruningMsgs.get(dim) != null && !dimPruningMsgs.get(dim).isEmpty()) { - ByteArrayOutputStream out = null; - try { + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { ObjectMapper mapper = new ObjectMapper(); - out = new ByteArrayOutputStream(); mapper.writeValue(out, dimPruningMsgs.get(dim).getJsonObject()); - reason = out.toString("UTF-8"); + log.info("No candidate dim found because: {}", out.toString("UTF-8")); } catch (Exception e) { throw new LensException("Error writing dim pruning messages", e); - } finally { - if (out != null) { - try { - out.close(); - } catch (IOException e) { - throw new LensException(e); - } - } } } log.error("Query rewrite failed due to NO_CANDIDATE_DIM_AVAILABLE, Cause {}", @@ -838,192 +828,226 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, return dimsToQuery; } - private Set<CandidateFact> pickCandidateFactToQuery() throws LensException { - Set<CandidateFact> facts = null; + private Candidate pickCandidateToQuery() throws LensException { + Candidate cand = null; if (hasCubeInQuery()) { - if (candidateFactSets.size() > 0) { - facts = candidateFactSets.iterator().next(); - log.info("Available candidate facts:{}, picking up {} for querying", candidateFactSets, facts); + if (candidates.size() > 0) { + cand = candidates.iterator().next(); + log.info("Available Candidates:{}, picking up Candidate: {} for querying", candidates, cand); } else { - String reason = ""; - if (!factPruningMsgs.isEmpty()) { - ByteArrayOutputStream out = null; + throwNoCandidateFactException(); + } + } + return cand; + } + + void throwNoCandidateFactException() throws LensException { + String reason = ""; + if (!storagePruningMsgs.isEmpty()) { + ByteArrayOutputStream out = null; + try { + ObjectMapper mapper = new ObjectMapper(); + out = new ByteArrayOutputStream(); + mapper.writeValue(out, storagePruningMsgs.getJsonObject()); + reason = out.toString("UTF-8"); + } catch (Exception e) { + throw new LensException("Error writing fact pruning messages", e); + } finally { + if (out != null) { try { - ObjectMapper mapper = new ObjectMapper(); - out = new ByteArrayOutputStream(); - mapper.writeValue(out, factPruningMsgs.getJsonObject()); - reason = out.toString("UTF-8"); - } catch (Exception e) { - throw new LensException("Error writing fact pruning messages", e); - } finally { - if (out != null) { - try { - out.close(); - } catch (IOException e) { - throw new LensException(e); - } - } + out.close(); + } catch (IOException e) { + throw new LensException(e); } } - log.error("Query rewrite failed due to NO_CANDIDATE_FACT_AVAILABLE, Cause {}", factPruningMsgs.toJsonObject()); - throw new NoCandidateFactAvailableException(factPruningMsgs); } } - return facts; + log.error("Query rewrite failed due to NO_CANDIDATE_FACT_AVAILABLE, Cause {}", storagePruningMsgs.toJsonObject()); + throw new NoCandidateFactAvailableException(this); } private HQLContextInterface hqlContext; + @Getter - private Collection<CandidateFact> pickedFacts; + private Candidate pickedCandidate; @Getter private Collection<CandidateDim> pickedDimTables; - private void addRangeClauses(CandidateFact fact) throws LensException { - if (fact != null) { + private void addRangeClauses(StorageCandidate sc) throws LensException { + if (sc != null) { // resolve timerange positions and replace it by corresponding where clause for (TimeRange range : getTimeRanges()) { - for (Map.Entry<String, String> entry : fact.getRangeToStorageWhereMap().get(range).entrySet()) { - String table = entry.getKey(); - String rangeWhere = entry.getValue(); - if (!StringUtils.isBlank(rangeWhere)) { - ASTNode rangeAST = HQLParser.parseExpr(rangeWhere, conf); - range.getParent().setChild(range.getChildIndex(), rangeAST); - } - fact.getStorgeWhereClauseMap().put(table, HQLParser.parseExpr(getWhereString(), conf)); + String rangeWhere = CandidateUtil.getTimeRangeWhereClasue(rangeWriter, sc, range); + if (!StringUtils.isBlank(rangeWhere)) { + ASTNode updatedRangeAST = HQLParser.parseExpr(rangeWhere, conf); + updateTimeRangeNode(sc.getQueryAst().getWhereAST(), range.getAstNode(), updatedRangeAST); + } + } + } + } + + + /** + * Find the appropriate time range node in the AST and update it with "updatedTimeRange". + * Time Range node looks like this + * time_range_in(dt, '2017', '2018') -> + * TOK_FUNCTION [TOK_FUNCTION] (l5c2p37) { + * time_range_in [Identifier] (l6c1p37)$ + * TOK_TABLE_OR_COL [TOK_TABLE_OR_COL] (l6c2p51) { + * dt [Identifier] (l7c1p51)$ + * } + * '2017' [StringLiteral] (l6c3p55)$ + * '2018' [StringLiteral] (l6c4p63)$ + } + * @param root + * @param timeRangeFuncNode + * @param updatedTimeRange + */ + private void updateTimeRangeNode(ASTNode root, ASTNode timeRangeFuncNode, ASTNode updatedTimeRange) { + ASTNode childNode; + if (root.getChildCount() == 0) { + return; + } + for (Node child : root.getChildren()) { + childNode = (ASTNode) child; + if (childNode.getType() == timeRangeFuncNode.getType() + && childNode.getChildCount() == timeRangeFuncNode.getChildCount() + && childNode.getChild(0).getText().equalsIgnoreCase(timeRangeFuncNode.getChild(0).getText())) { + //Found the "time_range_in" function node. Check the details further as there can be more than one time ranges + if (HQLParser.getString(timeRangeFuncNode).equalsIgnoreCase(HQLParser.getString(childNode))) { + //This is the correct time range node . Replace it with "updatedTimeRange" + childNode.getParent().setChild(childNode.getChildIndex(), updatedTimeRange); + return; } } + updateTimeRangeNode(childNode, timeRangeFuncNode, updatedTimeRange); } } + public String toHQL() throws LensException { - Set<CandidateFact> cfacts = pickCandidateFactToQuery(); + Candidate cand = pickCandidateToQuery(); Map<Dimension, CandidateDim> dimsToQuery = pickCandidateDimsToQuery(dimensions); - log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery); + Collection<StorageCandidate> scSet = new HashSet<>(); + if (cand != null) { + scSet.addAll(CandidateUtil.getStorageCandidates(cand)); + } + + //Expand and get update period specific storage candidates if required. + scSet = expandStorageCandidates(scSet); + + log.info("Candidate: {}, DimsToQuery: {}", cand, dimsToQuery); if (autoJoinCtx != null) { // prune join paths for picked fact and dimensions - autoJoinCtx.pruneAllPaths(cube, cfacts, dimsToQuery); + autoJoinCtx.pruneAllPaths(cube, scSet, dimsToQuery); } - Map<CandidateFact, Set<Dimension>> factDimMap = new HashMap<>(); - if (cfacts != null) { - if (cfacts.size() > 1) { - // copy ASTs for each fact - for (CandidateFact cfact : cfacts) { - cfact.copyASTs(this); - factDimMap.put(cfact, new HashSet<>(dimsToQuery.keySet())); - } + Map<StorageCandidate, Set<Dimension>> factDimMap = new HashMap<>(); + if (cand != null) { + // Set the default queryAST for StorageCandidate and copy child ASTs from cubeql. + // Later in the rewrite flow each Storage candidate will modify them accordingly. + for (StorageCandidate sc : scSet) { + sc.setQueryAst(DefaultQueryAST.fromStorageCandidate(sc, this)); + CandidateUtil.copyASTs(this, sc.getQueryAst()); + factDimMap.put(sc, new HashSet<>(dimsToQuery.keySet())); } - for (CandidateFact fact : cfacts) { - addRangeClauses(fact); + for (StorageCandidate sc : scSet) { + addRangeClauses(sc); } } // pick dimension tables required during expression expansion for the picked fact and dimensions Set<Dimension> exprDimensions = new HashSet<>(); - if (cfacts != null) { - for (CandidateFact cfact : cfacts) { - Set<Dimension> factExprDimTables = exprCtx.rewriteExprCtx(this, cfact, dimsToQuery, - cfacts.size() > 1 ? cfact : this); - exprDimensions.addAll(factExprDimTables); - if (cfacts.size() > 1) { - factDimMap.get(cfact).addAll(factExprDimTables); - } - } - if (cfacts.size() > 1) { - havingAST = MultiFactHQLContext.pushDownHaving(havingAST, this, cfacts); + if (!scSet.isEmpty()) { + for (StorageCandidate sc : scSet) { + Set<Dimension> scExprDimTables = exprCtx.rewriteExprCtx(this, sc, dimsToQuery, sc.getQueryAst()); + exprDimensions.addAll(scExprDimTables); + factDimMap.get(sc).addAll(scExprDimTables); } } else { // dim only query exprDimensions.addAll(exprCtx.rewriteExprCtx(this, null, dimsToQuery, this)); } dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimensions)); - log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery); + log.info("StorageCandidates: {}, DimsToQuery: {}", scSet, dimsToQuery); // pick denorm tables for the picked fact and dimensions Set<Dimension> denormTables = new HashSet<>(); - if (cfacts != null) { - for (CandidateFact cfact : cfacts) { - Set<Dimension> factDenormTables = deNormCtx.rewriteDenormctx(this, cfact, dimsToQuery, cfacts.size() > 1); - denormTables.addAll(factDenormTables); - if (cfacts.size() > 1) { - factDimMap.get(cfact).addAll(factDenormTables); - } + if (!scSet.isEmpty()) { + for (StorageCandidate sc : scSet) { + Set<Dimension> scDenormTables = deNormCtx.rewriteDenormctx(this, sc, dimsToQuery, !scSet.isEmpty()); + denormTables.addAll(scDenormTables); + factDimMap.get(sc).addAll(scDenormTables); } } else { denormTables.addAll(deNormCtx.rewriteDenormctx(this, null, dimsToQuery, false)); } dimsToQuery.putAll(pickCandidateDimsToQuery(denormTables)); - log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery); + log.info("StorageCandidates: {}, DimsToQuery: {}", scSet, dimsToQuery); // Prune join paths once denorm tables are picked if (autoJoinCtx != null) { // prune join paths for picked fact and dimensions - autoJoinCtx.pruneAllPaths(cube, cfacts, dimsToQuery); + autoJoinCtx.pruneAllPaths(cube, scSet, dimsToQuery); } if (autoJoinCtx != null) { // add optional dims from Join resolver Set<Dimension> joiningTables = new HashSet<>(); - if (cfacts != null && cfacts.size() > 1) { - for (CandidateFact cfact : cfacts) { - Set<Dimension> factJoiningTables = autoJoinCtx.pickOptionalTables(cfact, factDimMap.get(cfact), this); - factDimMap.get(cfact).addAll(factJoiningTables); - joiningTables.addAll(factJoiningTables); + if (scSet != null && scSet.size() > 1) { + for (StorageCandidate sc : scSet) { + Set<Dimension> scJoiningTables = autoJoinCtx.pickOptionalTables(sc, factDimMap.get(sc), this); + factDimMap.get(sc).addAll(scJoiningTables); + joiningTables.addAll(scJoiningTables); } } else { joiningTables.addAll(autoJoinCtx.pickOptionalTables(null, dimsToQuery.keySet(), this)); } dimsToQuery.putAll(pickCandidateDimsToQuery(joiningTables)); } - log.info("Picked Fact:{} dimsToQuery: {}", cfacts, dimsToQuery); + log.info("Picked StorageCandidates: {} DimsToQuery: {}", scSet, dimsToQuery); pickedDimTables = dimsToQuery.values(); - pickedFacts = cfacts; - if (cfacts != null) { - if (cfacts.size() > 1) { - // Update ASTs for each fact - for (CandidateFact cfact : cfacts) { - cfact.updateASTs(this); - } - whereAST = MultiFactHQLContext.convertHavingToWhere(havingAST, this, cfacts, new DefaultAliasDecider()); - for (CandidateFact cFact : cfacts) { - cFact.updateFromString(this, factDimMap.get(cFact), dimsToQuery); - } + pickedCandidate = cand; + + //Set From string and time range clause + if (!scSet.isEmpty()) { + for (StorageCandidate sc : scSet) { + sc.updateFromString(this, factDimMap.get(sc), dimsToQuery); } - } - if (cfacts == null || cfacts.size() == 1) { - updateFromString(cfacts == null ? null : cfacts.iterator().next(), dimsToQuery); - } - //update dim filter with fact filter - if (cfacts != null && cfacts.size() > 0) { - for (CandidateFact cfact : cfacts) { - if (!cfact.getStorageTables().isEmpty()) { - for (String qualifiedStorageTable : cfact.getStorageTables()) { - String storageTable = qualifiedStorageTable.substring(qualifiedStorageTable.indexOf(".") + 1); - String where = getWhere(cfact, autoJoinCtx, - cfact.getStorageWhereClause(storageTable), getAliasForTableName(cfact.getBaseTable().getName()), - shouldReplaceDimFilterWithFactFilter(), storageTable, dimsToQuery); - cfact.getStorgeWhereStringMap().put(storageTable, where); - } - } + } else { + updateFromString(null, dimsToQuery); + } + + //update dim filter with fact filter, set where string in sc + if (scSet.size() > 0) { + for (StorageCandidate sc : scSet) { + String qualifiedStorageTable = sc.getStorageTable(); + String storageTable = qualifiedStorageTable.substring(qualifiedStorageTable.indexOf(".") + 1); + String where = getWhere(sc, autoJoinCtx, + sc.getQueryAst().getWhereAST(), getAliasForTableName(sc.getBaseTable().getName()), + shouldReplaceDimFilterWithFactFilter(), storageTable, dimsToQuery); + sc.setWhereString(where); } } - hqlContext = createHQLContext(cfacts, dimsToQuery, factDimMap); - return hqlContext.toHQL(); - } - - private HQLContextInterface createHQLContext(Set<CandidateFact> facts, Map<Dimension, CandidateDim> dimsToQuery, - Map<CandidateFact, Set<Dimension>> factDimMap) throws LensException { - if (facts == null || facts.size() == 0) { - return new DimOnlyHQLContext(dimsToQuery, this, this); - } else if (facts.size() == 1 && facts.iterator().next().getStorageTables().size() > 1) { - //create single fact with multiple storage context - return new SingleFactMultiStorageHQLContext(facts.iterator().next(), dimsToQuery, this, this); - } else if (facts.size() == 1 && facts.iterator().next().getStorageTables().size() == 1) { - CandidateFact fact = facts.iterator().next(); - // create single fact context - return new SingleFactSingleStorageHQLContext(fact, null, - dimsToQuery, this, DefaultQueryAST.fromCandidateFact(fact, fact.getStorageTables().iterator().next(), this)); + + if (cand == null) { + hqlContext = new DimOnlyHQLContext(dimsToQuery, this, this); + return hqlContext.toHQL(); + } else if (scSet.size() == 1) { + StorageCandidate sc = (StorageCandidate) scSet.iterator().next(); + sc.updateAnswerableSelectColumns(this); + return getInsertClause() + sc.toHQL(factDimMap.get(sc)); } else { - return new MultiFactHQLContext(facts, dimsToQuery, factDimMap, this); + UnionQueryWriter uqc = new UnionQueryWriter(scSet, this); + return getInsertClause() + uqc.toHQL(factDimMap); + } + } + + private Collection<StorageCandidate> expandStorageCandidates(Collection<StorageCandidate> scSet) + throws LensException { + Collection<StorageCandidate> expandedList = new ArrayList<StorageCandidate>(); + for (StorageCandidate sc : scSet) { + expandedList.addAll(sc.splitAtUpdatePeriodLevelIfReq()); } + return expandedList; } public ASTNode toAST(Context ctx) throws LensException { @@ -1039,18 +1063,18 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, return ParseUtils.findRootNonNullToken(tree); } - public Set<String> getColumnsQueriedForTable(String tblName) { + Set<String> getColumnsQueriedForTable(String tblName) { return getColumnsQueried(getAliasForTableName(tblName)); } - public void addColumnsQueriedWithTimeDimCheck(QueriedPhraseContext qur, String alias, String timeDimColumn) { + void addColumnsQueriedWithTimeDimCheck(QueriedPhraseContext qur, String alias, String timeDimColumn) { if (!shouldReplaceTimeDimWithPart()) { qur.addColumnsQueried(alias, timeDimColumn); } } - public boolean isCubeMeasure(String col) { + boolean isCubeMeasure(String col) { if (col == null) { return false; } @@ -1100,6 +1124,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, ASTNode colIdent = (ASTNode) node.getChild(1); colname = colIdent.getText(); + assert tabident != null; tabname = tabident.getText(); } @@ -1108,7 +1133,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, return isCubeMeasure(msrname); } - public boolean hasAggregates() { + boolean hasAggregates() { if (getExprCtx().hasAggregates()) { return true; } @@ -1120,7 +1145,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, return false; } - public void setJoinCond(QBJoinTree qb, String cond) { + void setJoinCond(QBJoinTree qb, String cond) { joinConds.put(qb, cond); } @@ -1136,22 +1161,22 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, return null; } - public String getInsertClause() { + String getInsertClause() { ASTNode destTree = qb.getParseInfo().getDestForClause(clauseName); if (destTree != null && ((ASTNode) (destTree.getChild(0))).getToken().getType() != TOK_TMP_FILE) { - return "INSERT OVERWRITE" + HQLParser.getString(destTree); + return "INSERT OVERWRITE " + HQLParser.getString(destTree) + " "; } return ""; } - public Set<Aliased<Dimension>> getOptionalDimensions() { + Set<Aliased<Dimension>> getOptionalDimensions() { return optionalDimensionMap.keySet(); } /** * @return the hqlContext */ - public HQLContextInterface getHqlContext() { + HQLContextInterface getHqlContext() { return hqlContext; } @@ -1159,15 +1184,15 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, return getConf().getBoolean(REPLACE_TIMEDIM_WITH_PART_COL, DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL); } - public boolean shouldReplaceDimFilterWithFactFilter() { + private boolean shouldReplaceDimFilterWithFactFilter() { return getConf().getBoolean(REWRITE_DIM_FILTER_TO_FACT_FILTER, DEFAULT_REWRITE_DIM_FILTER_TO_FACT_FILTER); } - public String getPartitionColumnOfTimeDim(String timeDimName) { + String getPartitionColumnOfTimeDim(String timeDimName) { return getPartitionColumnOfTimeDim(cube, timeDimName); } - public static String getPartitionColumnOfTimeDim(CubeInterface cube, String timeDimName) { + private static String getPartitionColumnOfTimeDim(CubeInterface cube, String timeDimName) { if (cube == null) { return timeDimName; } @@ -1178,11 +1203,11 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, } } - public String getTimeDimOfPartitionColumn(String partCol) { + String getTimeDimOfPartitionColumn(String partCol) { return getTimeDimOfPartitionColumn(cube, partCol); } - public static String getTimeDimOfPartitionColumn(CubeInterface cube, String partCol) { + private static String getTimeDimOfPartitionColumn(CubeInterface cube, String partCol) { if (cube == null) { return partCol; } @@ -1193,93 +1218,44 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, } } - public void addQueriedMsrs(Set<String> msrs) { + void addQueriedMsrs(Set<String> msrs) { queriedMsrs.addAll(msrs); } - public void addQueriedExprs(Set<String> exprs) { + void addQueriedExprs(Set<String> exprs) { queriedExprs.addAll(exprs); } - public void addQueriedExprsWithMeasures(Set<String> exprs) { + void addQueriedExprsWithMeasures(Set<String> exprs) { queriedExprsWithMeasures.addAll(exprs); } - /** - * Prune candidate fact sets with respect to available candidate facts. - * <p></p> - * Prune a candidate set, if any of the fact is missing. - * - * @param pruneCause - */ - public void pruneCandidateFactSet(CandidateTablePruneCode pruneCause) { - // remove candidate fact sets that have missing facts - for (Iterator<Set<CandidateFact>> i = candidateFactSets.iterator(); i.hasNext();) { - Set<CandidateFact> cfacts = i.next(); - if (!candidateFacts.containsAll(cfacts)) { - log.info("Not considering fact table set:{} as they have non candidate tables and facts missing because of {}", - cfacts, pruneCause); - i.remove(); - } - } - // prune candidate facts - pruneCandidateFactWithCandidateSet(CandidateTablePruneCode.ELEMENT_IN_SET_PRUNED); - } - - /** - * Prune candidate fact with respect to available candidate fact sets. - * <p></p> - * If candidate fact is not present in any of the candidate fact sets, remove it. - * - * @param pruneCause - */ - public void pruneCandidateFactWithCandidateSet(CandidateTablePruneCode pruneCause) { - // remove candidate facts that are not part of any covering set - pruneCandidateFactWithCandidateSet(new CandidateTablePruneCause(pruneCause)); - } - - public void pruneCandidateFactWithCandidateSet(CandidateTablePruneCause pruneCause) { - // remove candidate facts that are not part of any covering set - Set<CandidateFact> allCoveringFacts = new HashSet<CandidateFact>(); - for (Set<CandidateFact> set : candidateFactSets) { - allCoveringFacts.addAll(set); - } - for (Iterator<CandidateFact> i = candidateFacts.iterator(); i.hasNext();) { - CandidateFact cfact = i.next(); - if (!allCoveringFacts.contains(cfact)) { - log.info("Not considering fact table:{} as {}", cfact, pruneCause); - addFactPruningMsgs(cfact.fact, pruneCause); - i.remove(); - } - } - } - - public void addQueriedTimeDimensionCols(final String timeDimColName) { + void addQueriedTimeDimensionCols(final String timeDimColName) { checkArgument(StringUtils.isNotBlank(timeDimColName)); this.queriedTimeDimCols.add(timeDimColName); } - public ImmutableSet<String> getQueriedTimeDimCols() { + ImmutableSet<String> getQueriedTimeDimCols() { return ImmutableSet.copyOf(this.queriedTimeDimCols); } - private String getWhere(CandidateFact cfact, AutoJoinContext autoJoinCtx, + private String getWhere(StorageCandidate sc, AutoJoinContext autoJoinCtx, ASTNode node, String cubeAlias, boolean shouldReplaceDimFilter, String storageTable, Map<Dimension, CandidateDim> dimToQuery) throws LensException { String whereString; if (autoJoinCtx != null && shouldReplaceDimFilter) { List<String> allfilters = new ArrayList<>(); - getAllFilters(node, cubeAlias, allfilters, autoJoinCtx.getJoinClause(cfact), dimToQuery); + getAllFilters(node, cubeAlias, allfilters, autoJoinCtx.getJoinClause(sc), dimToQuery); whereString = StringUtils.join(allfilters, " and "); } else { - whereString = HQLParser.getString(cfact.getStorageWhereClause(storageTable)); + whereString = HQLParser.getString(sc.getQueryAst().getWhereAST()); } return whereString; } - private List<String> getAllFilters(ASTNode node, String cubeAlias, List<String> allFilters, + private void getAllFilters(ASTNode node, String cubeAlias, List<String> allFilters, JoinClause joinClause, Map<Dimension, CandidateDim> dimToQuery) throws LensException { @@ -1304,7 +1280,6 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, ASTNode child = (ASTNode) node.getChild(i); getAllFilters(child, cubeAlias, allFilters, joinClause, dimToQuery); } - return allFilters; } private String getFilter(String table, String cubeAlias, ASTNode node, JoinClause joinClause, @@ -1322,7 +1297,6 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, } private TableRelationship getStarJoin(JoinClause joinClause, String table) { - TableRelationship rel; for (Map.Entry<TableRelationship, JoinTree> entry : joinClause.getJoinTree().getSubtrees().entrySet()) { if (entry.getValue().getDepthFromRoot() == 1 && table.equals(entry.getValue().getAlias())) { return entry.getKey(); @@ -1334,8 +1308,9 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, private String getTableFromFilterAST(ASTNode node) { if (node.getToken().getType() == HiveParser.DOT) { - return HQLParser.findNodeByPath((ASTNode) node, - TOK_TABLE_OR_COL, Identifier).getText(); + ASTNode n = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier); + assert n != null; + return n.getText(); } else { // recurse down for (int i = 0; i < node.getChildCount(); i++) { @@ -1354,7 +1329,8 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, String cubeAlias) throws LensException { StringBuilder builder = new StringBuilder(); - String storageClause = dimToQuery.get(tabRelation.getToTable()).getWhereClause(); + CandidateDim dim = dimToQuery.get(tabRelation.getToTable()); + String storageClause = dim.getWhereClause(); builder.append(cubeAlias) .append(".") @@ -1363,9 +1339,9 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, .append("select ") .append(tabRelation.getToColumn()) .append(" from ") - .append(dimToQuery.get(tabRelation.getToTable()).getStorageString(dimAlias)) + .append(dim.getStorageString(dimAlias)) .append(" where ") - .append(HQLParser.getString((ASTNode) dimFilter)); + .append(HQLParser.getString(dimFilter)); if (storageClause != null) { builder.append(" and ") .append(String.format(storageClause, dimAlias))
http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java index ea2eb7e..6bee386 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java @@ -139,52 +139,68 @@ public class CubeQueryRewriter { // Rewrite base trees (groupby, having, orderby, limit) using aliases rewriters.add(new AliasReplacer()); ExpressionResolver exprResolver = new ExpressionResolver(); - DenormalizationResolver denormResolver = new DenormalizationResolver(conf); + DenormalizationResolver denormResolver = new DenormalizationResolver(); CandidateTableResolver candidateTblResolver = new CandidateTableResolver(); StorageTableResolver storageTableResolver = new StorageTableResolver(conf); + + // Phase 1 of exprResolver: Resolve expressions rewriters.add(exprResolver); - // De-normalized columns resolved + // Phase 1 of denormResolver: De-normalized columns resolved rewriters.add(denormResolver); // Resolve time ranges rewriters.add(new TimerangeResolver()); - // Resolve candidate fact tables and dimension tables for columns queried + // Phase 1 of candidateTblResolver: Resolve candidate storages and dimension tables for columns queried rewriters.add(candidateTblResolver); // Resolve aggregations and generate base select tree rewriters.add(new AggregateResolver()); rewriters.add(new GroupbyResolver(conf)); + //validate fields queryability (in case of derived cubes setup) rewriters.add(new FieldValidator()); // Resolve joins and generate base join tree rewriters.add(new JoinResolver()); - // Do col life validation - rewriters.add(new TimeRangeChecker(conf)); - // Resolve candidate fact tables and dimension tables for columns included + // Do col life validation for the time range(s) queried + rewriters.add(new ColumnLifetimeChecker()); + // Phase 1 of storageTableResolver: Validate and prune candidate storages + rewriters.add(storageTableResolver); + // Phase 2 of candidateTblResolver: Resolve candidate storages and dimension tables for columns included // in join and denorm resolvers rewriters.add(candidateTblResolver); + // Find Union and Join combinations over Storage Candidates that can answer the queried time range(s) and all + // queried measures + rewriters.add(new CandidateCoveringSetsResolver()); - // Phase 1: resolve fact tables. - rewriters.add(storageTableResolver); + // If lightest fact first option is enabled for this driver (via lens.cube.query.pick.lightest.fact.first = true), + // run LightestFactResolver and keep only the lighted combination(s) generated by CandidateCoveringSetsResolver if (lightFactFirst) { // Prune candidate tables for which denorm column references do not exist rewriters.add(denormResolver); - // Prune candidate facts without any valid expressions + // Phase 2 of exprResolver:Prune candidate facts without any valid expressions rewriters.add(exprResolver); + // Pick the least cost combination(s) (and prune others) out of a set of combinations produced + // by CandidateCoveringSetsResolver rewriters.add(new LightestFactResolver()); } - // Phase 2: resolve fact table partitions. + + // Phase 2 of storageTableResolver: resolve storage table partitions. rewriters.add(storageTableResolver); + // In case partial data is allowed (via lens.cube.query.fail.if.data.partial = false) and there are many + // combinations with partial data, pick the one that covers the maximum part of time ranges(s) queried rewriters.add(new MaxCoveringFactResolver(conf)); - // Phase 3: resolve dimension tables and partitions. + // Phase 3 of storageTableResolver: resolve dimension tables and partitions. rewriters.add(storageTableResolver); // Prune candidate tables for which denorm column references do not exist + //TODO union: phase 2 of denormResolver needs to be moved before CoveringSetResolver.. check if this makes sense rewriters.add(denormResolver); - // Prune candidate facts without any valid expressions + // Phase 2 of exprResolver : Prune candidate facts without any valid expressions rewriters.add(exprResolver); - // We can have LightestFactResolver before LeastPartitionResolver - that says - // "if two facts have the same least weight, then the fact with least number of time partitions queried will be - // picked". This will be useful, if users did not set fact weights. + if (!lightFactFirst) { + // Pick the least cost combination(s) (and prune others) out of a set of combinations produced + // by CandidateCoveringSetsResolver rewriters.add(new LightestFactResolver()); } + // if two combinations have the same least weight/cost, then the combination with least number of time partitions + // queried will be picked. Rest of the combinations will be pruned rewriters.add(new LeastPartitionResolver()); rewriters.add(new LightestDimensionResolver()); } @@ -228,6 +244,7 @@ public class CubeQueryRewriter { */ MethodMetricsContext mgauge = MethodMetricsFactory.createMethodGauge(ctx.getConf(), true, rewriter.getClass().getCanonicalName() + ITER_STR + i); + rewriter.rewriteContext(ctx); mgauge.markSuccess(); i++; http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java index fc96055..e2015c2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java @@ -33,16 +33,12 @@ import lombok.Getter; * Accepts cube query AST and rewrites into storage table query */ public class CubeSemanticAnalyzer extends SemanticAnalyzer { - private final Configuration queryConf; - private final HiveConf hiveConf; - private final List<ValidationRule> validationRules = new ArrayList<ValidationRule>(); + private final List<ValidationRule> validationRules = new ArrayList<>(); @Getter private QB cubeQB; public CubeSemanticAnalyzer(Configuration queryConf, HiveConf hiveConf) throws SemanticException { super(new QueryState(hiveConf)); - this.queryConf = queryConf; - this.hiveConf = hiveConf; setupRules(); } @@ -65,10 +61,6 @@ public class CubeSemanticAnalyzer extends SemanticAnalyzer { ast.deleteChild(ast.getChildCount() - 1); } } - // analyzing from the ASTNode. - if (!doPhase1(ast, cubeQB, initPhase1Ctx(), null)) { - // if phase1Result false return - return; - } + doPhase1(ast, cubeQB, initPhase1Ctx(), null); } } http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultAliasDecider.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultAliasDecider.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultAliasDecider.java index 80ceae4..c8bf787 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultAliasDecider.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultAliasDecider.java @@ -20,8 +20,12 @@ package org.apache.lens.cube.parse; import org.apache.hadoop.hive.ql.parse.ASTNode; +import lombok.Getter; +import lombok.Setter; public class DefaultAliasDecider implements AliasDecider { + @Getter + @Setter int counter = 0; private static final String ALIAS_PREFIX = "alias"; http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java index c9993f3..29da0a2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java @@ -24,16 +24,17 @@ import org.apache.hadoop.hive.ql.parse.ASTNode; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; @Data @AllArgsConstructor +@NoArgsConstructor public class DefaultQueryAST implements QueryAST { private ASTNode selectAST, whereAST, groupByAST, havingAST, joinAST, orderByAST; private Integer limitValue; private String fromString; private String whereString; - public String getSelectString() { return HQLParser.getString(selectAST); } @@ -60,12 +61,12 @@ public class DefaultQueryAST implements QueryAST { return null; } - public static DefaultQueryAST fromCandidateFact(CandidateFact fact, String storageTable, QueryAST ast) throws - LensException { + public static DefaultQueryAST fromStorageCandidate(StorageCandidate sc, QueryAST ast) throws + LensException { return new DefaultQueryAST(ast.getSelectAST(), - null, - ast.getGroupByAST(), ast.getHavingAST(), ast.getJoinAST(), ast.getOrderByAST(), ast.getLimitValue(), - ast.getFromString(), - fact.getStorageWhereString(storageTable.substring(storageTable.indexOf(".") + 1))); + null, + ast.getGroupByAST(), ast.getHavingAST(), ast.getJoinAST(), ast.getOrderByAST(), ast.getLimitValue(), + ast.getFromString(), + sc.getWhereString()); } } http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java index cb26878..30fa873 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,6 +18,8 @@ */ package org.apache.lens.cube.parse; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.denormColumnNotFound; + import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TABLE_OR_COL; @@ -26,12 +28,10 @@ import java.util.*; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.metadata.ReferencedDimAttribute.ChainRefCol; -import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; import org.apache.lens.cube.parse.ExpressionResolver.ExprSpecContext; import org.apache.lens.cube.parse.ExpressionResolver.ExpressionContext; import org.apache.lens.server.api.error.LensException; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; @@ -50,11 +50,8 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class DenormalizationResolver implements ContextRewriter { - public DenormalizationResolver(Configuration conf) { - } - @ToString - public static class ReferencedQueriedColumn { + static class ReferencedQueriedColumn { ReferencedDimAttribute col; AbstractCubeTable srcTable; transient List<ChainRefCol> chainRefCols = new ArrayList<>(); @@ -67,17 +64,12 @@ public class DenormalizationResolver implements ContextRewriter { } @ToString - public static class PickedReference { + static class PickedReference { @Getter ChainRefCol chainRef; String srcAlias; String pickedFor; - PickedReference(String srcAlias, String pickedFor) { - this.srcAlias = srcAlias; - this.pickedFor = pickedFor; - } - PickedReference(ChainRefCol chainRef, String srcAlias, String pickedFor) { this.srcAlias = srcAlias; this.chainRef = chainRef; @@ -85,7 +77,7 @@ public class DenormalizationResolver implements ContextRewriter { } } - public static class DenormalizationContext { + static class DenormalizationContext { // map of column name to all references @Getter private Map<String, Set<ReferencedQueriedColumn>> referencedCols = new HashMap<>(); @@ -101,12 +93,7 @@ public class DenormalizationResolver implements ContextRewriter { private Map<String, Set<PickedReference>> pickedReferences = new HashMap<>(); void addReferencedCol(String col, ReferencedQueriedColumn refer) { - Set<ReferencedQueriedColumn> refCols = referencedCols.get(col); - if (refCols == null) { - refCols = new HashSet<>(); - referencedCols.put(col, refCols); - } - refCols.add(refer); + referencedCols.computeIfAbsent(col, k -> new HashSet<>()).add(refer); } // When candidate table does not have the field, this method checks @@ -122,12 +109,8 @@ public class DenormalizationResolver implements ContextRewriter { // there is no path // to the source table log.info("Adding denormalized column for column:{} for table:{}", col, table); - Set<ReferencedQueriedColumn> refCols = tableToRefCols.get(table.getName()); - if (refCols == null) { - refCols = new HashSet<>(); - tableToRefCols.put(table.getName(), refCols); - } - refCols.add(refer); + String name = (table instanceof CandidateDim) ? table.getName() : table.getStorageTable(); + tableToRefCols.computeIfAbsent(name, k -> new HashSet<>()).add(refer); // Add to optional tables for (ChainRefCol refCol : refer.col.getChainRefColumns()) { cubeql.addOptionalDimTable(refCol.getChainName(), table, false, refer.col.getName(), true, @@ -141,12 +124,7 @@ public class DenormalizationResolver implements ContextRewriter { } private void addPickedReference(String col, PickedReference refer) { - Set<PickedReference> refCols = pickedReferences.get(col); - if (refCols == null) { - refCols = new HashSet<>(); - pickedReferences.put(col, refCols); - } - refCols.add(refer); + pickedReferences.computeIfAbsent(col, k -> new HashSet<>()).add(refer); } private PickedReference getPickedReference(String col, String srcAlias) { @@ -161,28 +139,27 @@ public class DenormalizationResolver implements ContextRewriter { return null; } - public Set<Dimension> rewriteDenormctx(CubeQueryContext cubeql, CandidateFact cfact, Map<Dimension, - CandidateDim> dimsToQuery, boolean replaceFact) throws LensException { + Set<Dimension> rewriteDenormctx(CubeQueryContext cubeql, + StorageCandidate sc, Map<Dimension, CandidateDim> dimsToQuery, boolean replaceFact) throws LensException { Set<Dimension> refTbls = new HashSet<>(); - log.info("Doing denorm changes for fact :{}", cfact); + log.info("Doing denorm changes for fact :{}", sc); if (!tableToRefCols.isEmpty()) { // pick referenced columns for fact - if (cfact != null) { - pickColumnsForTable(cubeql, cfact.getName()); + if (sc != null) { + pickColumnsForTable(cubeql, sc.getStorageTable()); } // pick referenced columns for dimensions - if (dimsToQuery != null && !dimsToQuery.isEmpty()) { + if (dimsToQuery != null) { for (CandidateDim cdim : dimsToQuery.values()) { pickColumnsForTable(cubeql, cdim.getName()); } } // Replace picked reference in all the base trees - replaceReferencedColumns(cubeql, cfact, replaceFact); - + replaceReferencedColumns(cubeql, sc, replaceFact); // Add the picked references to dimsToQuery for (PickedReference picked : pickedRefs) { - if (isPickedFor(picked, cfact, dimsToQuery)) { + if (isPickedFor(picked, sc, dimsToQuery)) { refTbls.add((Dimension) cubeql.getCubeTableForAlias(picked.getChainRef().getChainName())); cubeql.addColumnsQueried(picked.getChainRef().getChainName(), picked.getChainRef().getRefColumn()); } @@ -193,21 +170,19 @@ public class DenormalizationResolver implements ContextRewriter { return refTbls; } - public boolean hasReferences() { + boolean hasReferences() { return !tableToRefCols.isEmpty(); } - public Set<Dimension> rewriteDenormctxInExpression(CubeQueryContext cubeql, CandidateFact cfact, Map<Dimension, + Set<Dimension> rewriteDenormctxInExpression(CubeQueryContext cubeql, StorageCandidate sc, Map<Dimension, CandidateDim> dimsToQuery, ASTNode exprAST) throws LensException { Set<Dimension> refTbls = new HashSet<>(); - - log.info("Doing denorm changes for expressions in fact :{}", cfact); if (!tableToRefCols.isEmpty()) { // pick referenced columns for fact - if (cfact != null) { - pickColumnsForTable(cubeql, cfact.getName()); + if (sc != null) { + pickColumnsForTable(cubeql, sc.getStorageTable()); } // pick referenced columns for dimensions - if (dimsToQuery != null && !dimsToQuery.isEmpty()) { + if (dimsToQuery != null) { for (CandidateDim cdim : dimsToQuery.values()) { pickColumnsForTable(cubeql, cdim.getName()); } @@ -217,7 +192,7 @@ public class DenormalizationResolver implements ContextRewriter { // Add the picked references to dimsToQuery for (PickedReference picked : pickedRefs) { - if (isPickedFor(picked, cfact, dimsToQuery)) { + if (isPickedFor(picked, sc, dimsToQuery)) { refTbls.add((Dimension) cubeql.getCubeTableForAlias(picked.getChainRef().getChainName())); cubeql.addColumnsQueried(picked.getChainRef().getChainName(), picked.getChainRef().getRefColumn()); } @@ -228,8 +203,8 @@ public class DenormalizationResolver implements ContextRewriter { return refTbls; } // checks if the reference if picked for facts and dimsToQuery passed - private boolean isPickedFor(PickedReference picked, CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery) { - if (cfact != null && picked.pickedFor.equalsIgnoreCase(cfact.getName())) { + private boolean isPickedFor(PickedReference picked, StorageCandidate sc, Map<Dimension, CandidateDim> dimsToQuery) { + if (sc != null && picked.pickedFor.equalsIgnoreCase(sc.getStorageTable())) { return true; } if (dimsToQuery != null) { @@ -245,15 +220,9 @@ public class DenormalizationResolver implements ContextRewriter { private void pickColumnsForTable(CubeQueryContext cubeql, String tbl) throws LensException { if (tableToRefCols.containsKey(tbl)) { for (ReferencedQueriedColumn refered : tableToRefCols.get(tbl)) { - Iterator<ChainRefCol> iter = refered.chainRefCols.iterator(); - while (iter.hasNext()) { - // remove unreachable references - ChainRefCol reference = iter.next(); - if (!cubeql.getAutoJoinCtx().isReachableDim( - (Dimension) cubeql.getCubeTableForAlias(reference.getChainName()), reference.getChainName())) { - iter.remove(); - } - } + // remove unreachable references + refered.chainRefCols.removeIf(reference -> !cubeql.getAutoJoinCtx().isReachableDim( + (Dimension) cubeql.getCubeTableForAlias(reference.getChainName()), reference.getChainName())); if (refered.chainRefCols.isEmpty()) { throw new LensException(LensCubeErrorCode.NO_REF_COL_AVAILABLE.getLensErrorInfo(), refered.col.getName()); } @@ -265,8 +234,7 @@ public class DenormalizationResolver implements ContextRewriter { } } } - - public void pruneReferences(CubeQueryContext cubeql) { + void pruneReferences(CubeQueryContext cubeql) { for (Set<ReferencedQueriedColumn> referencedQueriedColumns : referencedCols.values()) { for(Iterator<ReferencedQueriedColumn> iterator = referencedQueriedColumns.iterator(); iterator.hasNext();) { ReferencedQueriedColumn rqc = iterator.next(); @@ -295,25 +263,24 @@ public class DenormalizationResolver implements ContextRewriter { } } } - private void replaceReferencedColumns(CubeQueryContext cubeql, CandidateFact cfact, boolean replaceFact) - throws LensException { + + private void replaceReferencedColumns(CubeQueryContext cubeql, StorageCandidate sc, boolean replaceFact) + throws LensException { QueryAST ast = cubeql; - boolean factRefExists = cfact != null && tableToRefCols.get(cfact.getName()) != null && !tableToRefCols.get(cfact - .getName()).isEmpty(); + boolean factRefExists = sc != null && tableToRefCols.get(sc.getStorageTable()) != null + && !tableToRefCols.get(sc.getStorageTable()).isEmpty(); if (replaceFact && factRefExists) { - ast = cfact; + ast = sc.getQueryAst(); } resolveClause(ast.getSelectAST()); if (factRefExists) { - for (ASTNode storageWhereClauseAST : cfact.getStorgeWhereClauseMap().values()) { - resolveClause(storageWhereClauseAST); - } + resolveClause(sc.getQueryAst().getWhereAST()); } else { resolveClause(ast.getWhereAST()); } resolveClause(ast.getGroupByAST()); resolveClause(ast.getHavingAST()); - resolveClause(cubeql.getOrderByAST()); + resolveClause(ast.getOrderByAST()); } private void resolveClause(ASTNode node) throws LensException { @@ -332,6 +299,7 @@ public class DenormalizationResolver implements ContextRewriter { ASTNode tableNode = (ASTNode) node.getChild(0); ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier); + assert tabident != null; PickedReference refered = getPickedReference(colName, tabident.getText().toLowerCase()); if (refered == null) { return; @@ -352,7 +320,7 @@ public class DenormalizationResolver implements ContextRewriter { } } - public Set<String> getNonReachableReferenceFields(String table) { + Set<String> getNonReachableReferenceFields(String table) { Set<String> nonReachableFields = new HashSet<>(); if (tableToRefCols.containsKey(table)) { for (ReferencedQueriedColumn refcol : tableToRefCols.get(table)) { @@ -369,7 +337,7 @@ public class DenormalizationResolver implements ContextRewriter { private void addRefColsQueried(CubeQueryContext cubeql, TrackQueriedColumns tqc, DenormalizationContext denormCtx) { for (Map.Entry<String, Set<String>> entry : tqc.getTblAliasToColumns().entrySet()) { // skip default alias - if (entry.getKey() == CubeQueryContext.DEFAULT_TABLE) { + if (Objects.equals(entry.getKey(), CubeQueryContext.DEFAULT_TABLE)) { continue; } // skip join chain aliases @@ -408,9 +376,10 @@ public class DenormalizationResolver implements ContextRewriter { public void rewriteContext(CubeQueryContext cubeql) throws LensException { DenormalizationContext denormCtx = cubeql.getDeNormCtx(); if (denormCtx == null) { + DenormalizationContext ctx = getOrCreateDeNormCtx(cubeql); // Adds all the reference dimensions as eligible for denorm fields // add ref columns in cube - addRefColsQueried(cubeql, cubeql, getOrCreateDeNormCtx(cubeql)); + addRefColsQueried(cubeql, cubeql, ctx); // add ref columns from expressions for (Set<ExpressionContext> ecSet : cubeql.getExprCtx().getAllExprsQueried().values()) { for (ExpressionContext ec : ecSet) { @@ -424,21 +393,22 @@ public class DenormalizationResolver implements ContextRewriter { // In the second iteration of denorm resolver // candidate tables which require denorm fields and the refernces are no // more valid will be pruned - if (cubeql.getCube() != null && !cubeql.getCandidateFacts().isEmpty()) { - for (Iterator<CandidateFact> i = cubeql.getCandidateFacts().iterator(); i.hasNext();) { - CandidateFact cfact = i.next(); - Set<String> nonReachableFields = denormCtx.getNonReachableReferenceFields(cfact.getName()); + if (cubeql.getCube() != null && !cubeql.getCandidates().isEmpty()) { + for (Iterator<StorageCandidate> i = + CandidateUtil.getStorageCandidates(cubeql.getCandidates()).iterator(); i.hasNext();) { + StorageCandidate candidate = i.next(); + Set<String> nonReachableFields = denormCtx.getNonReachableReferenceFields(candidate.getStorageTable()); if (!nonReachableFields.isEmpty()) { - log.info("Not considering fact table:{} as columns {} are not available", cfact, nonReachableFields); - cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(nonReachableFields)); + log.info("Not considering fact table:{} as columns {} are not available", candidate, nonReachableFields); + cubeql.addCandidatePruningMsg(candidate, denormColumnNotFound(nonReachableFields)); i.remove(); } } - if (cubeql.getCandidateFacts().size() == 0) { + if (cubeql.getCandidates().size() == 0) { throw new LensException(LensCubeErrorCode.NO_FACT_HAS_COLUMN.getLensErrorInfo(), cubeql.getColumnsQueriedForTable(cubeql.getCube().getName()).toString()); } - cubeql.pruneCandidateFactSet(CandidateTablePruneCode.COLUMN_NOT_FOUND); + } if (cubeql.getDimensions() != null && !cubeql.getDimensions().isEmpty()) { for (Dimension dim : cubeql.getDimensions()) { @@ -447,8 +417,7 @@ public class DenormalizationResolver implements ContextRewriter { Set<String> nonReachableFields = denormCtx.getNonReachableReferenceFields(cdim.getName()); if (!nonReachableFields.isEmpty()) { log.info("Not considering dim table:{} as column {} is not available", cdim, nonReachableFields); - cubeql.addDimPruningMsgs(dim, cdim.dimtable, - CandidateTablePruneCause.columnNotFound(nonReachableFields)); + cubeql.addDimPruningMsgs(dim, cdim.dimtable, denormColumnNotFound(nonReachableFields)); i.remove(); } }