Author: jpullokk
Date: Fri Jan 16 23:13:58 2015
New Revision: 1652542
URL: http://svn.apache.org/r1652542
Log:
HIVE-9224: Inline Table, Properties (Laljo John Pullokkaran via Ashutosh
Chauhan)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AvgPartitionSizeBasedBigTableSelectorForAutoSMJ.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Fri Jan 16
23:13:58 2015
@@ -699,15 +699,13 @@ public class Driver implements CommandPr
|| op.equals(HiveOperation.QUERY)) {
SemanticAnalyzer querySem = (SemanticAnalyzer) sem;
ParseContext parseCtx = querySem.getParseContext();
- Map<TableScanOperator, Table> tsoTopMap = parseCtx.getTopToTable();
for (Map.Entry<String, Operator<? extends OperatorDesc>> topOpMap :
querySem
.getParseContext().getTopOps().entrySet()) {
Operator<? extends OperatorDesc> topOp = topOpMap.getValue();
- if (topOp instanceof TableScanOperator
- && tsoTopMap.containsKey(topOp)) {
+ if (topOp instanceof TableScanOperator) {
TableScanOperator tableScanOp = (TableScanOperator) topOp;
- Table tbl = tsoTopMap.get(tableScanOp);
+ Table tbl = tableScanOp.getConf().getTableMetadata();
List<Integer> neededColumnIds = tableScanOp.getNeededColumnIDs();
List<FieldSchema> columns = tbl.getCols();
List<String> cols = new ArrayList<String>();
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
Fri Jan 16 23:13:58 2015
@@ -203,7 +203,6 @@ abstract public class AbstractBucketJoin
new LinkedHashMap<String, List<List<String>>>();
HashMap<String, Operator<? extends OperatorDesc>> topOps =
pGraphContext.getTopOps();
- Map<TableScanOperator, Table> topToTable = pGraphContext.getTopToTable();
HashMap<String, String> aliasToNewAliasMap = new HashMap<String, String>();
@@ -270,7 +269,7 @@ abstract public class AbstractBucketJoin
joinKeyOrder = new Integer[keys.size()];
}
- Table tbl = topToTable.get(tso);
+ Table tbl = tso.getConf().getTableMetadata();
if (tbl.isPartitioned()) {
PrunedPartitionList prunedParts =
pGraphContext.getPrunedPartitions(alias, tso);
List<Partition> partitions = prunedParts.getNotDeniedPartns();
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java
Fri Jan 16 23:13:58 2015
@@ -249,8 +249,6 @@ abstract public class AbstractSMBJoinPro
int pos,
List<Order> sortColumnsFirstTable) throws SemanticException {
String alias = aliases[pos];
- Map<TableScanOperator, Table> topToTable = this.pGraphContext
- .getTopToTable();
/*
* Consider a query like:
@@ -313,7 +311,7 @@ abstract public class AbstractSMBJoinPro
return false;
}
- Table tbl = topToTable.get(tso);
+ Table tbl = tso.getConf().getTableMetadata();
if (tbl.isPartitioned()) {
PrunedPartitionList prunedParts =
pGraphContext.getPrunedPartitions(alias, tso);
List<Partition> partitions = prunedParts.getNotDeniedPartns();
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AvgPartitionSizeBasedBigTableSelectorForAutoSMJ.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AvgPartitionSizeBasedBigTableSelectorForAutoSMJ.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AvgPartitionSizeBasedBigTableSelectorForAutoSMJ.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AvgPartitionSizeBasedBigTableSelectorForAutoSMJ.java
Fri Jan 16 23:13:58 2015
@@ -71,7 +71,7 @@ public class AvgPartitionSizeBasedBigTab
int numPartitions = 1; // in case the sizes match, preference is
// given to the table with fewer partitions
- Table table = parseCtx.getTopToTable().get(topOp);
+ Table table = topOp.getConf().getTableMetadata();
long averageSize = 0;
if (!table.isPartitioned()) {
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
Fri Jan 16 23:13:58 2015
@@ -465,7 +465,7 @@ public class BucketingSortingReduceSinkO
if (op instanceof TableScanOperator) {
assert !useBucketSortPositions;
TableScanOperator ts = (TableScanOperator) op;
- Table srcTable = pGraphContext.getTopToTable().get(ts);
+ Table srcTable = ts.getConf().getTableMetadata();
// Find the positions of the bucketed columns in the table
corresponding
// to the select list.
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
Fri Jan 16 23:13:58 2015
@@ -186,7 +186,7 @@ public class DynamicPartitionPruningOpti
String column = extractColName(ctx.parent);
if (ts != null && column != null) {
- Table table = parseContext.getTopToTable().get(ts);
+ Table table = ts.getConf().getTableMetadata();
if (table != null && table.isPartitionKey(column)) {
String alias = ts.getConf().getAlias();
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
Fri Jan 16 23:13:58 2015
@@ -70,7 +70,7 @@ public class GenMRTableScan1 implements
TableScanOperator op = (TableScanOperator) nd;
GenMRProcContext ctx = (GenMRProcContext) opProcCtx;
ParseContext parseCtx = ctx.getParseCtx();
- Class<? extends InputFormat> inputFormat = parseCtx.getTopToTable().get(op)
+ Class<? extends InputFormat> inputFormat = op.getConf().getTableMetadata()
.getInputFormatClass();
Map<Operator<? extends OperatorDesc>, GenMapRedCtx> mapCurrCtx =
ctx.getMapCurrCtx();
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
Fri Jan 16 23:13:58 2015
@@ -521,12 +521,11 @@ public final class GenMapRedUtils {
// The table does not have any partitions
if (aliasPartnDesc == null) {
- aliasPartnDesc = new PartitionDesc(Utilities.getTableDesc(parseCtx
- .getTopToTable().get(topOp)), null);
-
+ aliasPartnDesc = new
PartitionDesc(Utilities.getTableDesc(((TableScanOperator) topOp)
+ .getConf().getTableMetadata()), null);
}
- Map<String, String> props = parseCtx.getTopToProps().get(topOp);
+ Map<String, String> props = topOp.getConf().getOpProps();
if (props != null) {
Properties target = aliasPartnDesc.getProperties();
if (target == null) {
@@ -955,7 +954,7 @@ public final class GenMapRedUtils {
public static TableScanOperator createTemporaryTableScanOperator(RowSchema
rowSchema) {
TableScanOperator tableScanOp =
- (TableScanOperator) OperatorFactory.get(new TableScanDesc(),
rowSchema);
+ (TableScanOperator) OperatorFactory.get(new TableScanDesc(null),
rowSchema);
// Set needed columns for this dummy TableScanOperator
List<Integer> neededColumnIds = new ArrayList<Integer>();
List<String> neededColumnNames = new ArrayList<String>();
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java
Fri Jan 16 23:13:58 2015
@@ -62,10 +62,8 @@ public class GlobalLimitOptimizer implem
GlobalLimitCtx globalLimitCtx = pctx.getGlobalLimitCtx();
Map<TableScanOperator, ExprNodeDesc> opToPartPruner =
pctx.getOpToPartPruner();
Map<String, SplitSample> nameToSplitSample = pctx.getNameToSplitSample();
- Map<TableScanOperator, Table> topToTable = pctx.getTopToTable();
QB qb = pctx.getQB();
- HiveConf conf = pctx.getConf();
QBParseInfo qbParseInfo = qb.getParseInfo();
// determine the query qualifies reduce input size for LIMIT
@@ -93,7 +91,7 @@ public class GlobalLimitOptimizer implem
// query qualify for the optimization
if (tempGlobalLimit != null && tempGlobalLimit != 0) {
TableScanOperator ts = (TableScanOperator)
topOps.values().toArray()[0];
- Table tab = topToTable.get(ts);
+ Table tab = ts.getConf().getTableMetadata();
if (!tab.isPartitioned()) {
if (qbParseInfo.getDestToWhereExpr().isEmpty()) {
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
Fri Jan 16 23:13:58 2015
@@ -304,7 +304,7 @@ public class GroupByOptimizer implements
// Create a mapping from the group by columns to the table columns
Map<String, String> tableColsMapping = new HashMap<String, String>();
Set<String> constantCols = new HashSet<String>();
- Table table = pGraphContext.getTopToTable().get(currOp);
+ Table table = tableScanOp.getConf().getTableMetadata();
for (FieldSchema col : table.getAllCols()) {
tableColsMapping.put(col.getName(), col.getName());
}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
Fri Jan 16 23:13:58 2015
@@ -176,7 +176,7 @@ public class SimpleFetchOptimizer implem
if (!aggressive && qb.hasTableSample(alias)) {
return null;
}
- Table table = pctx.getTopToTable().get(ts);
+ Table table = ts.getConf().getTableMetadata();
if (table == null) {
return null;
}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java
Fri Jan 16 23:13:58 2015
@@ -202,9 +202,7 @@ public class SkewJoinOptimizer implement
for (Entry<String, Operator<? extends OperatorDesc>> topOp :
topOps.entrySet()) {
TableScanOperator tso = (TableScanOperator) topOp.getValue();
- Table origTable =
parseContext.getTopToTable().get(ctx.getCloneTSOpMap().get(tso));
String tabAlias = tso.getConf().getAlias();
- parseContext.getTopToTable().put(tso, origTable);
int initCnt = 1;
String newAlias = "subquery" + initCnt + ":" + tabAlias;
while (origTopOps.containsKey(newAlias)) {
@@ -410,7 +408,7 @@ public class SkewJoinOptimizer implement
if (op instanceof TableScanOperator) {
TableScanOperator tsOp = (TableScanOperator)op;
if (tableScanOpsForJoin.contains(tsOp)) {
- return parseContext.getTopToTable().get(tsOp);
+ return tsOp.getConf().getTableMetadata();
}
}
if ((op.getParentOperators() == null) ||
(op.getParentOperators().isEmpty()) ||
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
Fri Jan 16 23:13:58 2015
@@ -228,7 +228,7 @@ public class StatsOptimizer implements T
return null;
}
- Table tbl = pctx.getTopToTable().get(tsOp);
+ Table tbl = tsOp.getConf().getTableMetadata();
List<Object> oneRow = new ArrayList<Object>();
List<ObjectInspector> ois = new ArrayList<ObjectInspector>();
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java
Fri Jan 16 23:13:58 2015
@@ -60,7 +60,7 @@ implements BigTableSelectorForAutoSMJ {
currentPos++;
continue;
}
- Table table = parseCtx.getTopToTable().get(topOp);
+ Table table = topOp.getConf().getTableMetadata();
long currentSize = 0;
if (!table.isPartitioned()) {
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java
Fri Jan 16 23:13:58 2015
@@ -124,7 +124,7 @@ public class CorrelationOptimizer implem
Set<String> aliases = new LinkedHashSet<String>();
for (TableScanOperator tsop : topOps) {
- Table table = pCtx.getTopToTable().get(tsop);
+ Table table = tsop.getConf().getTableMetadata();
if (table == null) {
// table should not be null.
throw new SemanticException("The table of " +
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java
Fri Jan 16 23:13:58 2015
@@ -161,11 +161,10 @@ public class RewriteGBUsingIndex impleme
* if the optimization can be applied. If yes, we add the name of the top
table to
* the tsOpToProcess to apply rewrite later on.
* */
- Map<TableScanOperator, Table> topToTable = parseContext.getTopToTable();
for (Map.Entry<String, Operator<?>> entry :
parseContext.getTopOps().entrySet()) {
String alias = entry.getKey();
TableScanOperator topOp = (TableScanOperator) entry.getValue();
- Table table = topToTable.get(topOp);
+ Table table = topOp.getConf().getTableMetadata();
List<Index> indexes = tableToIndex.get(table);
if (indexes.isEmpty()) {
continue;
@@ -232,12 +231,16 @@ public class RewriteGBUsingIndex impleme
supportedIndexes.add(AggregateIndexHandler.class.getName());
// query the metastore to know what columns we have indexed
- Collection<Table> topTables = parseContext.getTopToTable().values();
+ Collection<Operator<? extends OperatorDesc>> topTables =
parseContext.getTopOps().values();
Map<Table, List<Index>> indexes = new HashMap<Table, List<Index>>();
- for (Table tbl : topTables){
- List<Index> tblIndexes = IndexUtils.getIndexes(tbl, supportedIndexes);
- if (tblIndexes.size() > 0) {
- indexes.put(tbl, tblIndexes);
+ for (Operator<? extends OperatorDesc> op : topTables) {
+ if (op instanceof TableScanOperator) {
+ TableScanOperator tsOP = (TableScanOperator) op;
+ List<Index> tblIndexes =
IndexUtils.getIndexes(tsOP.getConf().getTableMetadata(),
+ supportedIndexes);
+ if (tblIndexes.size() > 0) {
+ indexes.put(tsOP.getConf().getTableMetadata(), tblIndexes);
+ }
}
}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java
Fri Jan 16 23:13:58 2015
@@ -170,7 +170,6 @@ public final class RewriteQueryUsingAggr
// Need to remove the original TableScanOperators from these data
structures
// and add new ones
- Map<TableScanOperator, Table> topToTable =
rewriteQueryCtx.getParseContext().getTopToTable();
Map<String, Operator<? extends OperatorDesc>> topOps =
rewriteQueryCtx.getParseContext()
.getTopOps();
Map<Operator<? extends OperatorDesc>, OpParseContext> opParseContext =
rewriteQueryCtx
@@ -181,13 +180,8 @@ public final class RewriteQueryUsingAggr
// remove original TableScanOperator
topOps.remove(alias);
- topToTable.remove(scanOperator);
opParseContext.remove(scanOperator);
- // construct a new descriptor for the index table scan
- TableScanDesc indexTableScanDesc = new TableScanDesc();
- indexTableScanDesc.setGatherStats(false);
-
String indexTableName = rewriteQueryCtx.getIndexName();
Table indexTableHandle = null;
try {
@@ -198,6 +192,10 @@ public final class RewriteQueryUsingAggr
throw new SemanticException(e.getMessage(), e);
}
+ // construct a new descriptor for the index table scan
+ TableScanDesc indexTableScanDesc = new TableScanDesc(indexTableHandle);
+ indexTableScanDesc.setGatherStats(false);
+
String k = indexTableName + Path.SEPARATOR;
indexTableScanDesc.setStatsAggPrefix(k);
scanOperator.setConf(indexTableScanDesc);
@@ -227,12 +225,10 @@ public final class RewriteQueryUsingAggr
}
// Scan operator now points to other table
- topToTable.put(scanOperator, indexTableHandle);
scanOperator.getConf().setAlias(newAlias);
scanOperator.setAlias(indexTableName);
topOps.put(newAlias, scanOperator);
opParseContext.put(scanOperator, operatorContext);
-
rewriteQueryCtx.getParseContext().setTopToTable((HashMap<TableScanOperator,
Table>) topToTable);
rewriteQueryCtx.getParseContext().setTopOps(
(HashMap<String, Operator<? extends OperatorDesc>>) topOps);
rewriteQueryCtx.getParseContext().setOpParseCtx(
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java
Fri Jan 16 23:13:58 2015
@@ -139,7 +139,7 @@ public class OpProcFactory {
// Table scan operator.
TableScanOperator top = (TableScanOperator)nd;
- org.apache.hadoop.hive.ql.metadata.Table t =
pctx.getTopToTable().get(top);
+ org.apache.hadoop.hive.ql.metadata.Table t =
top.getConf().getTableMetadata();
Table tab = t.getTTable();
// Generate the mappings
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java
Fri Jan 16 23:13:58 2015
@@ -52,7 +52,7 @@ public class LBPartitionProcFactory exte
protected void generatePredicate(NodeProcessorCtx procCtx, FilterOperator
fop,
TableScanOperator top) throws SemanticException, UDFArgumentException {
LBOpPartitionWalkerCtx owc = (LBOpPartitionWalkerCtx) procCtx;
- Table tbl = owc.getParseContext().getTopToTable().get(top);
+ Table tbl = top.getConf().getTableMetadata();
if (tbl.isPartitioned()) {
// Run partition pruner to get partitions
ParseContext parseCtx = owc.getParseContext();
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
Fri Jan 16 23:13:58 2015
@@ -161,7 +161,7 @@ public class OpTraitsRulesProcFactory {
Object... nodeOutputs) throws SemanticException {
TableScanOperator ts = (TableScanOperator)nd;
AnnotateOpTraitsProcCtx opTraitsCtx = (AnnotateOpTraitsProcCtx)procCtx;
- Table table = opTraitsCtx.getParseContext().getTopToTable().get(ts);
+ Table table = ts.getConf().getTableMetadata();
PrunedPartitionList prunedPartList = null;
try {
prunedPartList =
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java
Fri Jan 16 23:13:58 2015
@@ -28,6 +28,7 @@ import java.util.Map;
import java.util.Stack;
import org.apache.hadoop.hive.metastore.api.Index;
+import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.index.bitmap.BitmapIndexHandler;
@@ -47,6 +48,7 @@ import org.apache.hadoop.hive.ql.optimiz
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.MapredWork;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
/**
*
@@ -115,12 +117,14 @@ public class IndexWhereTaskDispatcher im
supportedIndexes.add(BitmapIndexHandler.class.getName());
// query the metastore to know what columns we have indexed
- Collection<Table> topTables = pctx.getTopToTable().values();
Map<TableScanOperator, List<Index>> indexes = new
HashMap<TableScanOperator, List<Index>>();
- for (Map.Entry<TableScanOperator, Table> entry :
pctx.getTopToTable().entrySet()) {
- List<Index> tblIndexes = IndexUtils.getIndexes(entry.getValue(),
supportedIndexes);
- if (tblIndexes.size() > 0) {
- indexes.put(entry.getKey(), tblIndexes);
+ for (Operator<? extends OperatorDesc> op : pctx.getTopOps().values()) {
+ if (op instanceof TableScanOperator) {
+ List<Index> tblIndexes = IndexUtils.getIndexes(((TableScanOperator)
op).getConf()
+ .getTableMetadata(), supportedIndexes);
+ if (tblIndexes.size() > 0) {
+ indexes.put((TableScanOperator) op, tblIndexes);
+ }
}
}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
Fri Jan 16 23:13:58 2015
@@ -138,7 +138,7 @@ public class PartitionPruner implements
*/
public static PrunedPartitionList prune(TableScanOperator ts, ParseContext
parseCtx,
String alias) throws SemanticException {
- return prune(parseCtx.getTopToTable().get(ts),
parseCtx.getOpToPartPruner().get(ts),
+ return prune(ts.getConf().getTableMetadata(),
parseCtx.getOpToPartPruner().get(ts),
parseCtx.getConf(), alias, parseCtx.getPrunedPartitions());
}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
Fri Jan 16 23:13:58 2015
@@ -103,7 +103,7 @@ public class StatsRulesProcFactory {
AnnotateStatsProcCtx aspCtx = (AnnotateStatsProcCtx) procCtx;
PrunedPartitionList partList =
aspCtx.getParseContext().getPrunedPartitions(tsop.getName(), tsop);
- Table table = aspCtx.getParseContext().getTopToTable().get(tsop);
+ Table table = tsop.getConf().getTableMetadata();
try {
// gather statistics for the first time and the attach it to table
scan operator
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java
Fri Jan 16 23:13:58 2015
@@ -17,16 +17,20 @@
*/
package org.apache.hadoop.hive.ql.parse;
+import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
public class ColumnAccessAnalyzer {
- private static final Log LOG =
LogFactory.getLog(ColumnAccessAnalyzer.class.getName());
+ private static final Log LOG =
LogFactory.getLog(ColumnAccessAnalyzer.class.getName());
private final ParseContext pGraphContext;
public ColumnAccessAnalyzer() {
@@ -39,19 +43,22 @@ public class ColumnAccessAnalyzer {
public ColumnAccessInfo analyzeColumnAccess() throws SemanticException {
ColumnAccessInfo columnAccessInfo = new ColumnAccessInfo();
- Map<TableScanOperator, Table> topOps = pGraphContext.getTopToTable();
- for (TableScanOperator op : topOps.keySet()) {
- Table table = topOps.get(op);
- String tableName = table.getCompleteName();
- List<String> referenced = op.getReferencedColumns();
- for (String column : referenced) {
- columnAccessInfo.add(tableName, column);
- }
- if (table.isPartitioned()) {
- PrunedPartitionList parts =
pGraphContext.getPrunedPartitions(table.getTableName(), op);
- if (parts.getReferredPartCols() != null) {
- for (String partKey : parts.getReferredPartCols()) {
- columnAccessInfo.add(tableName, partKey);
+ Collection<Operator<? extends OperatorDesc>> topOps =
pGraphContext.getTopOps().values();
+ for (Operator<? extends OperatorDesc> op : topOps) {
+ if (op instanceof TableScanOperator) {
+ TableScanOperator top = (TableScanOperator) op;
+ Table table = top.getConf().getTableMetadata();
+ String tableName = table.getCompleteName();
+ List<String> referenced = top.getReferencedColumns();
+ for (String column : referenced) {
+ columnAccessInfo.add(tableName, column);
+ }
+ if (table.isPartitioned()) {
+ PrunedPartitionList parts =
pGraphContext.getPrunedPartitions(table.getTableName(), top);
+ if (parts.getReferredPartCols() != null) {
+ for (String partKey : parts.getReferredPartCols()) {
+ columnAccessInfo.add(tableName, partKey);
+ }
}
}
}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java Fri
Jan 16 23:13:58 2015
@@ -191,10 +191,7 @@ public class GenTezUtils {
setupMapWork(mapWork, context, partitions, root, alias);
- if (context.parseContext != null
- && context.parseContext.getTopToTable() != null
- && context.parseContext.getTopToTable().containsKey(ts)
- && context.parseContext.getTopToTable().get(ts).isDummyTable()) {
+ if (ts.getConf().getTableMetadata() != null &&
ts.getConf().getTableMetadata().isDummyTable()) {
mapWork.setDummyTableScan(true);
}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
Fri Jan 16 23:13:58 2015
@@ -42,7 +42,6 @@ import org.apache.hadoop.hive.ql.exec.Ta
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.hooks.LineageInfo;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
-import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner;
import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
@@ -71,14 +70,11 @@ public class ParseContext {
private HashMap<TableScanOperator, sampleDesc> opToSamplePruner;
private Map<TableScanOperator, Map<String, ExprNodeDesc>>
opToPartToSkewedPruner;
private HashMap<String, Operator<? extends OperatorDesc>> topOps;
- private HashMap<String, Operator<? extends OperatorDesc>> topSelOps;
private LinkedHashMap<Operator<? extends OperatorDesc>, OpParseContext>
opParseCtx;
private Map<JoinOperator, QBJoinTree> joinContext;
private Map<MapJoinOperator, QBJoinTree> mapJoinContext;
private Map<SMBMapJoinOperator, QBJoinTree> smbMapJoinContext;
- private HashMap<TableScanOperator, Table> topToTable;
private List<ReduceSinkOperator>
reduceSinkOperatorsAddedByEnforceBucketingSorting;
- private HashMap<TableScanOperator, Map<String, String>> topToProps;
private HashMap<String, SplitSample> nameToSplitSample;
private List<LoadTableDesc> loadTableWork;
private List<LoadFileDesc> loadFileWork;
@@ -125,15 +121,11 @@ public class ParseContext {
* @param opToPartList
* @param topOps
* list of operators for the top query
- * @param topSelOps
- * list of operators for the selects introduced for column pruning
* @param opParseCtx
* operator parse context - contains a mapping from operator to
* operator parse state (row resolver etc.)
* @param joinContext
* context needed join processing (map join specifically)
- * @param topToTable
- * the top tables being processed
* @param loadTableWork
* list of destination tables being loaded
* @param loadFileWork
@@ -160,12 +152,9 @@ public class ParseContext {
HashMap<TableScanOperator, ExprNodeDesc> opToPartPruner,
HashMap<TableScanOperator, PrunedPartitionList> opToPartList,
HashMap<String, Operator<? extends OperatorDesc>> topOps,
- HashMap<String, Operator<? extends OperatorDesc>> topSelOps,
LinkedHashMap<Operator<? extends OperatorDesc>, OpParseContext>
opParseCtx,
Map<JoinOperator, QBJoinTree> joinContext,
Map<SMBMapJoinOperator, QBJoinTree> smbMapJoinContext,
- HashMap<TableScanOperator, Table> topToTable,
- HashMap<TableScanOperator, Map<String, String>> topToProps,
List<LoadTableDesc> loadTableWork, List<LoadFileDesc> loadFileWork,
Context ctx, HashMap<String, String> idToTableNameMap, int destTableId,
UnionProcContext uCtx, List<AbstractMapJoinOperator<? extends
MapJoinDesc>> listMapJoinOpsNoReducer,
@@ -186,13 +175,10 @@ public class ParseContext {
this.opToPartList = opToPartList;
this.joinContext = joinContext;
this.smbMapJoinContext = smbMapJoinContext;
- this.topToTable = topToTable;
- this.topToProps = topToProps;
this.loadFileWork = loadFileWork;
this.loadTableWork = loadTableWork;
this.opParseCtx = opParseCtx;
this.topOps = topOps;
- this.topSelOps = topSelOps;
this.ctx = ctx;
this.idToTableNameMap = idToTableNameMap;
this.destTableId = destTableId;
@@ -292,21 +278,6 @@ public class ParseContext {
return opToPartList;
}
- /**
- * @return the topToTable
- */
- public HashMap<TableScanOperator, Table> getTopToTable() {
- return topToTable;
- }
-
- /**
- * @param topToTable
- * the topToTable to set
- */
- public void setTopToTable(HashMap<TableScanOperator, Table> topToTable) {
- this.topToTable = topToTable;
- }
-
public List<ReduceSinkOperator>
getReduceSinkOperatorsAddedByEnforceBucketingSorting() {
return reduceSinkOperatorsAddedByEnforceBucketingSorting;
}
@@ -318,21 +289,6 @@ public class ParseContext {
}
/**
- * @return the topToProps
- */
- public HashMap<TableScanOperator, Map<String, String>> getTopToProps() {
- return topToProps;
- }
-
- /**
- * @param topToProps
- * the topToProps to set
- */
- public void setTopToProps(HashMap<TableScanOperator, Map<String, String>>
topToProps) {
- this.topToProps = topToProps;
- }
-
- /**
* @return the topOps
*/
public HashMap<String, Operator<? extends OperatorDesc>> getTopOps() {
@@ -348,22 +304,6 @@ public class ParseContext {
}
/**
- * @return the topSelOps
- */
- public HashMap<String, Operator<? extends OperatorDesc>> getTopSelOps() {
- return topSelOps;
- }
-
- /**
- * @param topSelOps
- * the topSelOps to set
- */
- public void setTopSelOps(
- HashMap<String, Operator<? extends OperatorDesc>> topSelOps) {
- this.topSelOps = topSelOps;
- }
-
- /**
* @return the opParseCtx
*/
public LinkedHashMap<Operator<? extends OperatorDesc>, OpParseContext>
getOpParseCtx() {
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
Fri Jan 16 23:13:58 2015
@@ -75,7 +75,7 @@ public class ProcessAnalyzeTable impleme
TableScanOperator tableScan = (TableScanOperator) nd;
ParseContext parseContext = context.parseContext;
- Class<? extends InputFormat> inputFormat =
parseContext.getTopToTable().get(tableScan)
+ Class<? extends InputFormat> inputFormat =
tableScan.getConf().getTableMetadata()
.getInputFormatClass();
QB queryBlock = parseContext.getQB();
QBParseInfo parseInfo = parseContext.getQB().getParseInfo();
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Fri Jan 16 23:13:58 2015
@@ -375,7 +375,6 @@ public class SemanticAnalyzer extends Ba
opToPartList = pctx.getOpToPartList();
opToSamplePruner = pctx.getOpToSamplePruner();
topOps = pctx.getTopOps();
- topSelOps = pctx.getTopSelOps();
opParseCtx = pctx.getOpParseCtx();
loadTableWork = pctx.getLoadTableWork();
loadFileWork = pctx.getLoadFileWork();
@@ -394,14 +393,13 @@ public class SemanticAnalyzer extends Ba
}
public ParseContext getParseContext() {
- return new ParseContext(conf, qb, ast, opToPartPruner, opToPartList,
topOps,
- topSelOps, opParseCtx, joinContext, smbMapJoinContext, topToTable,
topToTableProps,
- loadTableWork, loadFileWork, ctx, idToTableNameMap, destTableId, uCtx,
+ return new ParseContext(conf, qb, ast, opToPartPruner, opToPartList,
+ topOps, opParseCtx, joinContext, smbMapJoinContext, loadTableWork,
+ loadFileWork, ctx, idToTableNameMap, destTableId, uCtx,
listMapJoinOpsNoReducer, groupOpToInputTables, prunedPartitions,
opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks,
opToPartToSkewedPruner, viewAliasToInput,
- reduceSinkOperatorsAddedByEnforceBucketingSorting,
- queryProperties);
+ reduceSinkOperatorsAddedByEnforceBucketingSorting, queryProperties);
}
@SuppressWarnings("nls")
@@ -9273,7 +9271,7 @@ public class SemanticAnalyzer extends Ba
}
// Create the root of the operator tree
- TableScanDesc tsDesc = new TableScanDesc(alias, vcList);
+ TableScanDesc tsDesc = new TableScanDesc(alias, vcList, tab);
setupStats(tsDesc, qb.getParseInfo(), tab, alias, rwsch);
SplitSample sample = nameToSplitSample.get(alias_id);
@@ -9295,6 +9293,7 @@ public class SemanticAnalyzer extends Ba
Map<String, String> props = qb.getTabPropsForAlias(alias);
if (props != null) {
topToTableProps.put((TableScanOperator) top, props);
+ tsDesc.setOpProps(props);
}
} else {
rwsch = opParseCtx.get(top).getRowResolver();
@@ -9988,12 +9987,13 @@ public class SemanticAnalyzer extends Ba
}
// 4. Generate Parse Context for Optimizer & Physical compiler
- ParseContext pCtx = new ParseContext(conf, qb, plannerCtx.child,
opToPartPruner, opToPartList,
- topOps, topSelOps, opParseCtx, joinContext, smbMapJoinContext,
topToTable, topToTableProps,
- loadTableWork, loadFileWork, ctx, idToTableNameMap, destTableId, uCtx,
- listMapJoinOpsNoReducer, groupOpToInputTables, prunedPartitions,
opToSamplePruner,
- globalLimitCtx, nameToSplitSample, inputs, rootTasks,
opToPartToSkewedPruner,
- viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting,
queryProperties);
+ ParseContext pCtx = new ParseContext(conf, qb, plannerCtx.child,
+ opToPartPruner, opToPartList, topOps, opParseCtx, joinContext,
+ smbMapJoinContext, loadTableWork, loadFileWork, ctx, idToTableNameMap,
+ destTableId, uCtx, listMapJoinOpsNoReducer, groupOpToInputTables,
+ prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample,
+ inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput,
+ reduceSinkOperatorsAddedByEnforceBucketingSorting, queryProperties);
// 5. Take care of view creation
if (createVwDesc != null) {
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java
Fri Jan 16 23:13:58 2015
@@ -143,7 +143,7 @@ public class TableAccessAnalyzer {
// Must be deterministic order map for consistent q-test output across
Java versions
Map<String, List<String>> tableToKeysMap = new LinkedHashMap<String,
List<String>>();
- Table tbl = pGraphContext.getTopToTable().get(tso);
+ Table tbl = tso.getConf().getTableMetadata();
tableToKeysMap.put(tbl.getCompleteName(), keyColNames);
tableAccessCtx.addOperatorTableAccess(op, tableToKeysMap);
@@ -203,7 +203,7 @@ public class TableAccessAnalyzer {
return null;
}
- Table tbl = pGraphContext.getTopToTable().get(tso);
+ Table tbl = tso.getConf().getTableMetadata();
tableToKeysMap.put(tbl.getCompleteName(), keyColNames);
} else {
return null;
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
Fri Jan 16 23:13:58 2015
@@ -387,8 +387,8 @@ public abstract class TaskCompiler {
ParseContext clone = new ParseContext(conf,
pCtx.getQB(), pCtx.getParseTree(),
pCtx.getOpToPartPruner(), pCtx.getOpToPartList(), pCtx.getTopOps(),
- pCtx.getTopSelOps(), pCtx.getOpParseCtx(), pCtx.getJoinContext(),
- pCtx.getSmbMapJoinContext(), pCtx.getTopToTable(),
pCtx.getTopToProps(),
+ pCtx.getOpParseCtx(), pCtx.getJoinContext(),
+ pCtx.getSmbMapJoinContext(),
pCtx.getLoadTableWork(), pCtx.getLoadFileWork(), pCtx.getContext(),
pCtx.getIdToTableNameMap(), pCtx.getDestTableId(), pCtx.getUCtx(),
pCtx.getListMapJoinOpsNoReducer(), pCtx.getGroupOpToInputTables(),
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
Fri Jan 16 23:13:58 2015
@@ -82,7 +82,7 @@ public class SparkProcessAnalyzeTable im
ParseContext parseContext = context.parseContext;
@SuppressWarnings("rawtypes")
- Class<? extends InputFormat> inputFormat =
parseContext.getTopToTable().get(tableScan)
+ Class<? extends InputFormat> inputFormat =
tableScan.getConf().getTableMetadata()
.getInputFormatClass();
QB queryBlock = parseContext.getQB();
QBParseInfo parseInfo = parseContext.getQB().getParseInfo();
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java
Fri Jan 16 23:13:58 2015
@@ -18,12 +18,22 @@
package org.apache.hadoop.hive.ql.plan;
+
+import java.util.Map;
+
+import org.apache.hadoop.hive.ql.exec.PTFUtils;
+
public class AbstractOperatorDesc implements OperatorDesc {
protected boolean vectorMode = false;
protected transient Statistics statistics;
protected transient OpTraits opTraits;
+ protected transient Map<String, String> opProps;
+ static {
+ PTFUtils.makeTransient(AbstractOperatorDesc.class, "opProps");
+ }
+
@Override
@Explain(skipHeader = true, displayName = "Statistics")
public Statistics getStatistics() {
@@ -51,4 +61,12 @@ public class AbstractOperatorDesc implem
public void setOpTraits(OpTraits opTraits) {
this.opTraits = opTraits;
}
+
+ public Map<String, String> getOpProps() {
+ return opProps;
+ }
+
+ public void setOpProps(Map<String, String> props) {
+ this.opProps = props;
+ }
}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java Fri
Jan 16 23:13:58 2015
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
+import java.util.Map;
public interface OperatorDesc extends Serializable, Cloneable {
public Object clone() throws CloneNotSupportedException;
@@ -26,4 +27,5 @@ public interface OperatorDesc extends Se
public void setStatistics(Statistics statistics);
public OpTraits getOpTraits();
public void setOpTraits(OpTraits opTraits);
+ public Map<String, String> getOpProps();
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java Fri
Jan 16 23:13:58 2015
@@ -956,7 +956,7 @@ public final class PlanUtils {
ReadEntity parentViewInfo = getParentViewInfo(alias,
parseCtx.getViewAliasToInput());
// Adds tables only for create view (PPD filter can be appended by outer
query)
- Table table = parseCtx.getTopToTable().get(topOp);
+ Table table = topOp.getConf().getTableMetadata();
PlanUtils.addInput(inputs, new ReadEntity(table, parentViewInfo));
}
}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
Fri Jan 16 23:13:58 2015
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.ql.exec.PTFUtils;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
/**
@@ -36,7 +37,7 @@ public class TableScanDesc extends Abstr
private static final long serialVersionUID = 1L;
static {
- PTFUtils.makeTransient(TableScanDesc.class, "filterObject",
"referencedColumns");
+ PTFUtils.makeTransient(TableScanDesc.class, "filterObject",
"referencedColumns", "tableMetadata");
}
private String alias;
@@ -93,23 +94,32 @@ public class TableScanDesc extends Abstr
private boolean isMetadataOnly = false;
- @SuppressWarnings("nls")
+ private transient final Table tableMetadata;
+
+
public TableScanDesc() {
+ this(null, null);
}
- public TableScanDesc(final String alias) {
- this.alias = alias;
+ @SuppressWarnings("nls")
+ public TableScanDesc(Table tblMetadata) {
+ this(null, tblMetadata);
}
- public TableScanDesc(final String alias, List<VirtualColumn> vcs) {
+ public TableScanDesc(final String alias, Table tblMetadata) {
+ this(alias, null, tblMetadata);
+ }
+
+ public TableScanDesc(final String alias, List<VirtualColumn> vcs, Table
tblMetadata) {
this.alias = alias;
this.virtualCols = vcs;
+ this.tableMetadata = tblMetadata;
}
@Override
public Object clone() {
List<VirtualColumn> vcs = new ArrayList<VirtualColumn>(getVirtualCols());
- return new TableScanDesc(getAlias(), vcs);
+ return new TableScanDesc(getAlias(), vcs, this.tableMetadata);
}
@Explain(displayName = "alias")
@@ -250,8 +260,12 @@ public class TableScanDesc extends Abstr
public void setIsMetadataOnly(boolean metadata_only) {
isMetadataOnly = metadata_only;
}
-
+
public boolean getIsMetadataOnly() {
return isMetadataOnly;
}
+
+ public Table getTableMetadata() {
+ return tableMetadata;
+ }
}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java Fri
Jan 16 23:13:58 2015
@@ -897,7 +897,7 @@ public final class OpProcFactory {
HiveConf hiveConf) {
TableScanDesc tableScanDesc = tableScanOp.getConf();
- Table tbl = owi.getParseContext().getTopToTable().get(tableScanOp);
+ Table tbl = tableScanDesc.getTableMetadata();
if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTINDEXFILTER)) {
// attach the original predicate to the table scan operator for index
// optimizations that require the pushed predicate before pcr & later
Modified:
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java?rev=1652542&r1=1652541&r2=1652542&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java
(original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java
Fri Jan 16 23:13:58 2015
@@ -95,7 +95,7 @@ public class TestGenTezWork {
rs = new ReduceSinkOperator();
rs.setConf(new ReduceSinkDesc());
ts = new TableScanOperator();
- ts.setConf(new TableScanDesc());
+ ts.setConf(new TableScanDesc(null));
ts.getChildOperators().add(rs);
rs.getParentOperators().add(ts);
rs.getChildOperators().add(fs);
