HIVE-10546 : genFileSinkPlan should use the generated SEL's RR for the partition col of FS (Pengcheng Xiong via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <hashut...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/27c42cf2 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/27c42cf2 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/27c42cf2 Branch: refs/heads/branch-1.2 Commit: 27c42cf2f4a4402e510c9c877977c7552e70907a Parents: 2f5bf19 Author: Pengcheng Xiong <pxi...@hortonworks.com> Authored: Wed Apr 29 21:36:00 2015 -0700 Committer: Ashutosh Chauhan <hashut...@apache.org> Committed: Fri May 1 12:09:42 2015 -0700 ---------------------------------------------------------------------- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 42 +++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/27c42cf2/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 1d2c764..dec0e38 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -6043,7 +6043,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { boolean enforceBucketing = false; boolean enforceSorting = false; ArrayList<ExprNodeDesc> partnCols = new ArrayList<ExprNodeDesc>(); - ArrayList<ExprNodeDesc> partnColsNoConvert = new ArrayList<ExprNodeDesc>(); ArrayList<ExprNodeDesc> sortCols = new ArrayList<ExprNodeDesc>(); ArrayList<Integer> sortOrders = new ArrayList<Integer>(); boolean multiFileSpray = false; @@ -6055,11 +6054,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { enforceBucketing = true; if (updating() || deleting()) { partnCols = getPartitionColsFromBucketColsForUpdateDelete(input, true); - partnColsNoConvert = getPartitionColsFromBucketColsForUpdateDelete(input, false); } else { partnCols = getPartitionColsFromBucketCols(dest, qb, dest_tab, table_desc, input, true); - partnColsNoConvert = getPartitionColsFromBucketCols(dest, qb, dest_tab, table_desc, input, - false); } } @@ -6071,7 +6067,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { sortOrders = getSortOrders(dest, qb, dest_tab, input); if (!enforceBucketing) { partnCols = sortCols; - partnColsNoConvert = getSortCols(dest, qb, dest_tab, table_desc, input, false); } } @@ -6107,12 +6102,41 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { reduceSinkOperatorsAddedByEnforceBucketingSorting.add((ReduceSinkOperator)input.getParentOperators().get(0)); ctx.setMultiFileSpray(multiFileSpray); ctx.setNumFiles(numFiles); - ctx.setPartnCols(partnColsNoConvert); ctx.setTotalFiles(totalFiles); } return input; } + private void genPartnCols(String dest, Operator input, QB qb, + TableDesc table_desc, Table dest_tab, SortBucketRSCtx ctx) throws SemanticException { + boolean enforceBucketing = false; + boolean enforceSorting = false; + ArrayList<ExprNodeDesc> partnColsNoConvert = new ArrayList<ExprNodeDesc>(); + + if ((dest_tab.getNumBuckets() > 0) && + (conf.getBoolVar(HiveConf.ConfVars.HIVEENFORCEBUCKETING))) { + enforceBucketing = true; + if (updating() || deleting()) { + partnColsNoConvert = getPartitionColsFromBucketColsForUpdateDelete(input, false); + } else { + partnColsNoConvert = getPartitionColsFromBucketCols(dest, qb, dest_tab, table_desc, input, + false); + } + } + + if ((dest_tab.getSortCols() != null) && + (dest_tab.getSortCols().size() > 0) && + (conf.getBoolVar(HiveConf.ConfVars.HIVEENFORCESORTING))) { + enforceSorting = true; + if (!enforceBucketing) { + partnColsNoConvert = getSortCols(dest, qb, dest_tab, table_desc, input, false); + } + } + + if (enforceBucketing || enforceSorting) { + ctx.setPartnCols(partnColsNoConvert); + } + } /** * Check for HOLD_DDLTIME hint. * @@ -6556,6 +6580,12 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // If this table is working with ACID semantics, turn off merging canBeMerged &= !destTableIsAcid; + // Generate the partition columns from the parent input + if (dest_type.intValue() == QBMetaData.DEST_TABLE + || dest_type.intValue() == QBMetaData.DEST_PARTITION) { + genPartnCols(dest, input, qb, table_desc, dest_tab, rsCtx); + } + FileSinkDesc fileSinkDesc = new FileSinkDesc( queryTmpdir, table_desc,