http://git-wip-us.apache.org/repos/asf/hive/blob/c50ebb34/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index b67a03f..9ccb4e5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -52,6 +52,7 @@ import org.antlr.runtime.tree.TreeVisitorAction; import org.antlr.runtime.tree.TreeWizard; import org.antlr.runtime.tree.TreeWizard.ContextVisitor; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.util.ImmutableBitSet; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; @@ -127,6 +128,7 @@ import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.InvalidTableException; +import org.apache.hadoop.hive.ql.metadata.NotNullConstraint; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient; import org.apache.hadoop.hive.ql.metadata.Table; @@ -210,6 +212,7 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.ResourceType; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCardinalityViolation; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF; @@ -574,7 +577,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { HashMap<String, ASTNode> wExprsInDest = qb.getParseInfo().getWindowingExprsForClause(dest); int wColIdx = spec.getWindowExpressions() == null ? 0 : spec.getWindowExpressions().size(); WindowFunctionSpec wFnSpec = processWindowFunction(wdwFn, - (ASTNode)wdwFn.getChild(wdwFn.getChildCount()-1)); + (ASTNode)wdwFn.getChild(wdwFn.getChildCount()-1)); // If this is a duplicate invocation of a function; don't add to WindowingSpec. if ( wExprsInDest != null && wExprsInDest.containsKey(wFnSpec.getExpression().toStringTree())) { @@ -611,8 +614,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * @throws SemanticException */ private void doPhase1GetAllAggregations(ASTNode expressionTree, - HashMap<String, ASTNode> aggregations, List<ASTNode> wdwFns, - ASTNode wndParent) throws SemanticException { + HashMap<String, ASTNode> aggregations, List<ASTNode> wdwFns, + ASTNode wndParent) throws SemanticException { int exprTokenType = expressionTree.getToken().getType(); if(exprTokenType == HiveParser.TOK_SUBQUERY_EXPR) { //since now we have scalar subqueries we can get subquery expression in having @@ -792,9 +795,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { ErrorMsg.SAMPLE_RESTRICTION.getMsg())); } TableSample tabSample = new TableSample( - unescapeIdentifier(sampleClause.getChild(0).getText()), - unescapeIdentifier(sampleClause.getChild(1).getText()), - sampleCols); + unescapeIdentifier(sampleClause.getChild(0).getText()), + unescapeIdentifier(sampleClause.getChild(1).getText()), + sampleCols); qb.getParseInfo().setTabSample(alias, tabSample); if (unparseTranslator.isEnabled()) { for (ASTNode sampleCol : sampleCols) { @@ -876,8 +879,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { private void assertCombineInputFormat(Tree numerator, String message) throws SemanticException { String inputFormat = conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez") ? - HiveConf.getVar(conf, HiveConf.ConfVars.HIVETEZINPUTFORMAT): - HiveConf.getVar(conf, HiveConf.ConfVars.HIVEINPUTFORMAT); + HiveConf.getVar(conf, HiveConf.ConfVars.HIVETEZINPUTFORMAT): + HiveConf.getVar(conf, HiveConf.ConfVars.HIVEINPUTFORMAT); if (!inputFormat.equals(CombineHiveInputFormat.class.getName())) { throw new SemanticException(generateErrorMessage((ASTNode) numerator, message + " sampling is not supported in " + inputFormat)); @@ -1164,8 +1167,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { static private boolean isOuterJoinToken(ASTNode node) { return (node.getToken().getType() == HiveParser.TOK_LEFTOUTERJOIN) - || (node.getToken().getType() == HiveParser.TOK_RIGHTOUTERJOIN) - || (node.getToken().getType() == HiveParser.TOK_FULLOUTERJOIN); + || (node.getToken().getType() == HiveParser.TOK_RIGHTOUTERJOIN) + || (node.getToken().getType() == HiveParser.TOK_FULLOUTERJOIN); } /** @@ -1197,7 +1200,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { processPTF(qb, child); PTFInvocationSpec ptfInvocationSpec = qb.getPTFInvocationSpec(child); String inputAlias = ptfInvocationSpec == null ? null : - ptfInvocationSpec.getFunction().getAlias();; + ptfInvocationSpec.getFunction().getAlias();; if ( inputAlias == null ) { throw new SemanticException(generateErrorMessage(child, "PTF invocation in a Join must have an alias")); @@ -1328,7 +1331,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { doPhase1GetColumnAliasesFromSelect(ast, qbp); qbp.setAggregationExprsForClause(ctx_1.dest, aggregations); qbp.setDistinctFuncExprsForClause(ctx_1.dest, - doPhase1GetDistinctFuncExprs(aggregations)); + doPhase1GetDistinctFuncExprs(aggregations)); break; case HiveParser.TOK_WHERE: @@ -1449,7 +1452,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { break; case HiveParser.TOK_SORTBY: - // Get the sort by aliases - these are aliased to the entries in the + // Get the sort by aliases - these are aliased to the entries in the // select list queryProperties.setHasSortBy(true); qbp.setSortByExprForClause(ctx_1.dest, ast); @@ -1645,17 +1648,17 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { List<String> targetColNames = new ArrayList<String>(); for(Node col : tabColName.getChildren()) { assert ((ASTNode)col).getType() == HiveParser.Identifier : - "expected token " + HiveParser.Identifier + " found " + ((ASTNode)col).getType(); + "expected token " + HiveParser.Identifier + " found " + ((ASTNode)col).getType(); targetColNames.add(((ASTNode)col).getText().toLowerCase()); } String fullTableName = getUnescapedName((ASTNode) ast.getChild(0).getChild(0), - SessionState.get().getCurrentDatabase()); + SessionState.get().getCurrentDatabase()); qbp.setDestSchemaForClause(ctx_1.dest, targetColNames); Set<String> targetColumns = new HashSet<String>(); targetColumns.addAll(targetColNames); if(targetColNames.size() != targetColumns.size()) { throw new SemanticException(generateErrorMessage(tabColName, - "Duplicate column name detected in " + fullTableName + " table schema specification")); + "Duplicate column name detected in " + fullTableName + " table schema specification")); } Table targetTable = null; try { @@ -1667,7 +1670,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } if(targetTable == null) { throw new SemanticException(generateErrorMessage(ast, - "Unable to access metadata for table " + fullTableName)); + "Unable to access metadata for table " + fullTableName)); } for(FieldSchema f : targetTable.getCols()) { //parser only allows foo(a,b), not foo(foo.a, foo.b) @@ -1701,7 +1704,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } if(tokPartVal != null && tokPartVal.getType() == HiveParser.TOK_PARTVAL && tokPartVal.getChildCount() == 1) { assert tokPartVal.getChild(0).getType() == HiveParser.Identifier : - "Expected column name; found tokType=" + tokPartVal.getType(); + "Expected column name; found tokType=" + tokPartVal.getType(); dynamicPartitionColumns.add(tokPartVal.getChild(0).getText()); } } @@ -1713,10 +1716,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { if(!targetColumns.isEmpty()) { //Found some columns in user specified schema which are neither regular not dynamic partition columns throw new SemanticException(generateErrorMessage(tabColName, - "'" + (targetColumns.size() == 1 ? targetColumns.iterator().next() : targetColumns) + - "' in insert schema specification " + (targetColumns.size() == 1 ? "is" : "are") + - " not found among regular columns of " + - fullTableName + " nor dynamic partition columns.")); + "'" + (targetColumns.size() == 1 ? targetColumns.iterator().next() : targetColumns) + + "' in insert schema specification " + (targetColumns.size() == 1 ? "is" : "are") + + " not found among regular columns of " + + fullTableName + " nor dynamic partition columns.")); } } } @@ -1761,8 +1764,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { if (cte != null) { if (ctesExpanded.contains(cteName)) { throw new SemanticException("Recursive cte " + cteName + - " detected (cycle: " + StringUtils.join(ctesExpanded, " -> ") + - " -> " + cteName + ")."); + " detected (cycle: " + StringUtils.join(ctesExpanded, " -> ") + + " -> " + cteName + ")."); } cte.reference++; current.parents.add(cte); @@ -1804,7 +1807,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } private void getMetaData(QBExpr qbexpr, ReadEntity parentInput) - throws HiveException { + throws HiveException { if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) { getMetaData(qbexpr.getQB(), parentInput); } else { @@ -1815,7 +1818,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { @SuppressWarnings("nls") private void getMetaData(QB qb, ReadEntity parentInput) - throws HiveException { + throws HiveException { LOG.info("Get metadata for source tables"); // Go over the tables and populate the related structures. @@ -1847,7 +1850,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { tab = new Table(tab.getTTable().deepCopy()); } if (tab == null || - tab.getDbName().equals(SessionState.get().getCurrentDatabase())) { + tab.getDbName().equals(SessionState.get().getCurrentDatabase())) { Table materializedTab = ctx.getMaterializedTable(cteName); if (materializedTab == null) { // we first look for this alias from CTE, and then from catalog. @@ -1877,7 +1880,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } } - if (tab.isView()) { + if (tab.isView()) { if (qb.getParseInfo().isAnalyzeCommand()) { throw new SemanticException(ErrorMsg.ANALYZE_VIEW.getMsg()); } @@ -1901,7 +1904,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { String aliasId = getAliasId(alias, qb); if (aliasId != null) { aliasId = aliasId.replace(SemanticAnalyzer.SUBQUERY_TAG_1, "") - .replace(SemanticAnalyzer.SUBQUERY_TAG_2, ""); + .replace(SemanticAnalyzer.SUBQUERY_TAG_2, ""); } viewAliasToInput.put(aliasId, viewInput); continue; @@ -1909,8 +1912,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { if (!InputFormat.class.isAssignableFrom(tab.getInputFormatClass())) { throw new SemanticException(generateErrorMessage( - qb.getParseInfo().getSrcForAlias(alias), - ErrorMsg.INVALID_INPUT_FORMAT_TYPE.getMsg())); + qb.getParseInfo().getSrcForAlias(alias), + ErrorMsg.INVALID_INPUT_FORMAT_TYPE.getMsg())); } qb.getMetaData().setSrcForAlias(alias, tab); @@ -1923,8 +1926,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { ts.partitions = db.getPartitionsByNames(ts.tableHandle, ts.partSpec); } catch (HiveException e) { throw new SemanticException(generateErrorMessage( - qb.getParseInfo().getSrcForAlias(alias), - "Cannot get partitions for " + ts.partSpec), e); + qb.getParseInfo().getSrcForAlias(alias), + "Cannot get partitions for " + ts.partSpec), e); } } @@ -1936,7 +1939,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // Temporary tables created during the execution are not the input sources if (!PlanUtils.isValuesTempTable(alias)) { PlanUtils.addInput(inputs, - new ReadEntity(tab, parentViewInfo, parentViewInfo == null),mergeIsDirect); + new ReadEntity(tab, parentViewInfo, parentViewInfo == null),mergeIsDirect); } } @@ -1972,84 +1975,84 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { for (String name : qbp.getClauseNamesForDest()) { ASTNode ast = qbp.getDestForClause(name); switch (ast.getToken().getType()) { - case HiveParser.TOK_TAB: { - TableSpec ts = new TableSpec(db, conf, ast); - if (ts.tableHandle.isView() || ts.tableHandle.isMaterializedView()) { - throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg()); - } + case HiveParser.TOK_TAB: { + TableSpec ts = new TableSpec(db, conf, ast); + if (ts.tableHandle.isView() || ts.tableHandle.isMaterializedView()) { + throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg()); + } - Class<?> outputFormatClass = ts.tableHandle.getOutputFormatClass(); - if (!ts.tableHandle.isNonNative() && - !HiveOutputFormat.class.isAssignableFrom(outputFormatClass)) { - throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE - .getMsg(ast, "The class is " + outputFormatClass.toString())); - } + Class<?> outputFormatClass = ts.tableHandle.getOutputFormatClass(); + if (!ts.tableHandle.isNonNative() && + !HiveOutputFormat.class.isAssignableFrom(outputFormatClass)) { + throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE + .getMsg(ast, "The class is " + outputFormatClass.toString())); + } - boolean isTableWrittenTo = qb.getParseInfo().isInsertIntoTable(ts.tableHandle.getDbName(), + boolean isTableWrittenTo = qb.getParseInfo().isInsertIntoTable(ts.tableHandle.getDbName(), ts.tableHandle.getTableName()); - isTableWrittenTo |= (qb.getParseInfo().getInsertOverwriteTables(). + isTableWrittenTo |= (qb.getParseInfo().getInsertOverwriteTables(). get(getUnescapedName((ASTNode) ast.getChild(0), ts.tableHandle.getDbName()).toLowerCase()) != null); - assert isTableWrittenTo : + assert isTableWrittenTo : "Inconsistent data structure detected: we are writing to " + ts.tableHandle + " in " + - name + " but it's not in isInsertIntoTable() or getInsertOverwriteTables()"; - // Disallow update and delete on non-acid tables - boolean isAcid = AcidUtils.isAcidTable(ts.tableHandle); - if ((updating(name) || deleting(name)) && !isAcid) { - // Whether we are using an acid compliant transaction manager has already been caught in - // UpdateDeleteSemanticAnalyzer, so if we are updating or deleting and getting nonAcid - // here, it means the table itself doesn't support it. - throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, ts.tableName); + name + " but it's not in isInsertIntoTable() or getInsertOverwriteTables()"; + // Disallow update and delete on non-acid tables + boolean isAcid = AcidUtils.isAcidTable(ts.tableHandle); + if ((updating(name) || deleting(name)) && !isAcid) { + // Whether we are using an acid compliant transaction manager has already been caught in + // UpdateDeleteSemanticAnalyzer, so if we are updating or deleting and getting nonAcid + // here, it means the table itself doesn't support it. + throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, ts.tableName); + } + // TableSpec ts is got from the query (user specified), + // which means the user didn't specify partitions in their query, + // but whether the table itself is partitioned is not know. + if (ts.specType != SpecType.STATIC_PARTITION) { + // This is a table or dynamic partition + qb.getMetaData().setDestForAlias(name, ts.tableHandle); + // has dynamic as well as static partitions + if (ts.partSpec != null && ts.partSpec.size() > 0) { + qb.getMetaData().setPartSpecForAlias(name, ts.partSpec); } - // TableSpec ts is got from the query (user specified), - // which means the user didn't specify partitions in their query, - // but whether the table itself is partitioned is not know. - if (ts.specType != SpecType.STATIC_PARTITION) { - // This is a table or dynamic partition - qb.getMetaData().setDestForAlias(name, ts.tableHandle); - // has dynamic as well as static partitions - if (ts.partSpec != null && ts.partSpec.size() > 0) { - qb.getMetaData().setPartSpecForAlias(name, ts.partSpec); - } - } else { - // This is a partition - qb.getMetaData().setDestForAlias(name, ts.partHandle); - } - if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - // Add the table spec for the destination table. - qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts); - } - break; + } else { + // This is a partition + qb.getMetaData().setDestForAlias(name, ts.partHandle); + } + if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { + // Add the table spec for the destination table. + qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts); } + break; + } - case HiveParser.TOK_DIR: { - // This is a dfs file - String fname = stripQuotes(ast.getChild(0).getText()); - if ((!qb.getParseInfo().getIsSubQ()) - && (((ASTNode) ast.getChild(0)).getToken().getType() == HiveParser.TOK_TMP_FILE)) { + case HiveParser.TOK_DIR: { + // This is a dfs file + String fname = stripQuotes(ast.getChild(0).getText()); + if ((!qb.getParseInfo().getIsSubQ()) + && (((ASTNode) ast.getChild(0)).getToken().getType() == HiveParser.TOK_TMP_FILE)) { - if (qb.isCTAS() || qb.isMaterializedView()) { - qb.setIsQuery(false); - ctx.setResDir(null); - ctx.setResFile(null); + if (qb.isCTAS() || qb.isMaterializedView()) { + qb.setIsQuery(false); + ctx.setResDir(null); + ctx.setResFile(null); - Path location; - // If the CTAS query does specify a location, use the table location, else use the db location - if (qb.getTableDesc() != null && qb.getTableDesc().getLocation() != null) { - location = new Path(qb.getTableDesc().getLocation()); - } else { - // allocate a temporary output dir on the location of the table - String tableName = getUnescapedName((ASTNode) ast.getChild(0)); - String[] names = Utilities.getDbTableName(tableName); - try { - Warehouse wh = new Warehouse(conf); - //Use destination table's db location. - String destTableDb = qb.getTableDesc() != null ? qb.getTableDesc().getDatabaseName() : null; - if (destTableDb == null) { - destTableDb = names[0]; - } - location = wh.getDatabasePath(db.getDatabase(destTableDb)); - } catch (MetaException e) { - throw new SemanticException(e); + Path location; + // If the CTAS query does specify a location, use the table location, else use the db location + if (qb.getTableDesc() != null && qb.getTableDesc().getLocation() != null) { + location = new Path(qb.getTableDesc().getLocation()); + } else { + // allocate a temporary output dir on the location of the table + String tableName = getUnescapedName((ASTNode) ast.getChild(0)); + String[] names = Utilities.getDbTableName(tableName); + try { + Warehouse wh = new Warehouse(conf); + //Use destination table's db location. + String destTableDb = qb.getTableDesc() != null ? qb.getTableDesc().getDatabaseName() : null; + if (destTableDb == null) { + destTableDb = names[0]; + } + location = wh.getDatabasePath(db.getDatabase(destTableDb)); + } catch (MetaException e) { + throw new SemanticException(e); } } try { @@ -2071,66 +2074,66 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // Add the table spec for the destination table. qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts); } - } else { - // This is the only place where isQuery is set to true; it defaults to false. - qb.setIsQuery(true); - Path stagingPath = getStagingDirectoryPathname(qb); - fname = stagingPath.toString(); - ctx.setResDir(stagingPath); - } + } else { + // This is the only place where isQuery is set to true; it defaults to false. + qb.setIsQuery(true); + Path stagingPath = getStagingDirectoryPathname(qb); + fname = stagingPath.toString(); + ctx.setResDir(stagingPath); } + } - boolean isDfsFile = true; - if (ast.getChildCount() >= 2 && ast.getChild(1).getText().toLowerCase().equals("local")) { - isDfsFile = false; - } - // Set the destination for the SELECT query inside the CTAS - qb.getMetaData().setDestForAlias(name, fname, isDfsFile); - - CreateTableDesc directoryDesc = new CreateTableDesc(); - boolean directoryDescIsSet = false; - int numCh = ast.getChildCount(); - for (int num = 1; num < numCh ; num++){ - ASTNode child = (ASTNode) ast.getChild(num); - if (child != null) { - if (storageFormat.fillStorageFormat(child)) { - directoryDesc.setOutputFormat(storageFormat.getOutputFormat()); - directoryDesc.setSerName(storageFormat.getSerde()); - directoryDescIsSet = true; - continue; - } - switch (child.getToken().getType()) { - case HiveParser.TOK_TABLEROWFORMAT: - rowFormatParams.analyzeRowFormat(child); - directoryDesc.setFieldDelim(rowFormatParams.fieldDelim); - directoryDesc.setLineDelim(rowFormatParams.lineDelim); - directoryDesc.setCollItemDelim(rowFormatParams.collItemDelim); - directoryDesc.setMapKeyDelim(rowFormatParams.mapKeyDelim); - directoryDesc.setFieldEscape(rowFormatParams.fieldEscape); - directoryDesc.setNullFormat(rowFormatParams.nullFormat); - directoryDescIsSet=true; - break; - case HiveParser.TOK_TABLESERIALIZER: - ASTNode serdeChild = (ASTNode) child.getChild(0); - storageFormat.setSerde(unescapeSQLString(serdeChild.getChild(0).getText())); - directoryDesc.setSerName(storageFormat.getSerde()); - if (serdeChild.getChildCount() > 1) { - directoryDesc.setSerdeProps(new HashMap<String, String>()); - readProps((ASTNode) serdeChild.getChild(1).getChild(0), directoryDesc.getSerdeProps()); - } - directoryDescIsSet = true; - break; + boolean isDfsFile = true; + if (ast.getChildCount() >= 2 && ast.getChild(1).getText().toLowerCase().equals("local")) { + isDfsFile = false; + } + // Set the destination for the SELECT query inside the CTAS + qb.getMetaData().setDestForAlias(name, fname, isDfsFile); + + CreateTableDesc directoryDesc = new CreateTableDesc(); + boolean directoryDescIsSet = false; + int numCh = ast.getChildCount(); + for (int num = 1; num < numCh ; num++){ + ASTNode child = (ASTNode) ast.getChild(num); + if (child != null) { + if (storageFormat.fillStorageFormat(child)) { + directoryDesc.setOutputFormat(storageFormat.getOutputFormat()); + directoryDesc.setSerName(storageFormat.getSerde()); + directoryDescIsSet = true; + continue; + } + switch (child.getToken().getType()) { + case HiveParser.TOK_TABLEROWFORMAT: + rowFormatParams.analyzeRowFormat(child); + directoryDesc.setFieldDelim(rowFormatParams.fieldDelim); + directoryDesc.setLineDelim(rowFormatParams.lineDelim); + directoryDesc.setCollItemDelim(rowFormatParams.collItemDelim); + directoryDesc.setMapKeyDelim(rowFormatParams.mapKeyDelim); + directoryDesc.setFieldEscape(rowFormatParams.fieldEscape); + directoryDesc.setNullFormat(rowFormatParams.nullFormat); + directoryDescIsSet=true; + break; + case HiveParser.TOK_TABLESERIALIZER: + ASTNode serdeChild = (ASTNode) child.getChild(0); + storageFormat.setSerde(unescapeSQLString(serdeChild.getChild(0).getText())); + directoryDesc.setSerName(storageFormat.getSerde()); + if (serdeChild.getChildCount() > 1) { + directoryDesc.setSerdeProps(new HashMap<String, String>()); + readProps((ASTNode) serdeChild.getChild(1).getChild(0), directoryDesc.getSerdeProps()); } + directoryDescIsSet = true; + break; } } - if (directoryDescIsSet){ - qb.setDirectoryDesc(directoryDesc); - } - break; } - default: - throw new SemanticException(generateErrorMessage(ast, - "Unknown Token Type " + ast.getToken().getType())); + if (directoryDescIsSet){ + qb.setDirectoryDesc(directoryDesc); + } + break; + } + default: + throw new SemanticException(generateErrorMessage(ast, + "Unknown Token Type " + ast.getToken().getType())); } } } @@ -2151,7 +2154,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } } } catch (Exception e) { - throw new HiveException("Unable to determine if " + path + " is encrypted: " + e, e); + throw new HiveException("Unable to determine if " + path + " is encrypted: " + e, e); } return false; @@ -2283,12 +2286,12 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } private void replaceViewReferenceWithDefinition(QB qb, Table tab, - String tab_name, String alias) throws SemanticException { + String tab_name, String alias) throws SemanticException { ASTNode viewTree; final ASTNodeOrigin viewOrigin = new ASTNodeOrigin("VIEW", tab.getTableName(), tab.getViewExpandedText(), alias, qb.getParseInfo().getSrcForAlias( - alias)); + alias)); try { // Reparse text, passing null for context to avoid clobbering // the top-level token stream. @@ -2305,7 +2308,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { Dispatcher nodeOriginDispatcher = new Dispatcher() { @Override public Object dispatch(Node nd, java.util.Stack<Node> stack, - Object... nodeOutputs) { + Object... nodeOutputs) { ((ASTNode) nd).setOrigin(viewOrigin); return null; } @@ -2363,7 +2366,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { */ @SuppressWarnings("rawtypes") private String findAlias(ASTNode columnRef, - Map<String, Operator> aliasToOpInfo) throws SemanticException { + Map<String, Operator> aliasToOpInfo) throws SemanticException { String colName = unescapeIdentifier(columnRef.getChild(0).getText() .toLowerCase()); String tabAlias = null; @@ -2391,9 +2394,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { @SuppressWarnings("nls") void parseJoinCondPopulateAlias(QBJoinTree joinTree, ASTNode condn, - ArrayList<String> leftAliases, ArrayList<String> rightAliases, - ArrayList<String> fields, - Map<String, Operator> aliasToOpInfo) throws SemanticException { + ArrayList<String> leftAliases, ArrayList<String> rightAliases, + ArrayList<String> fields, + Map<String, Operator> aliasToOpInfo) throws SemanticException { // String[] allAliases = joinTree.getAllAliases(); switch (condn.getToken().getType()) { case HiveParser.TOK_TABLE_OR_COL: @@ -2497,8 +2500,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } private void populateAliases(List<String> leftAliases, - List<String> rightAliases, ASTNode condn, QBJoinTree joinTree, - List<String> leftSrc) throws SemanticException { + List<String> rightAliases, ASTNode condn, QBJoinTree joinTree, + List<String> leftSrc) throws SemanticException { if ((leftAliases.size() != 0) && (rightAliases.size() != 0)) { joinTree.addPostJoinFilter(condn); return; @@ -2525,21 +2528,21 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * only left sources are referenced in a Predicate */ void applyEqualityPredicateToQBJoinTree(QBJoinTree joinTree, - JoinType type, - List<String> leftSrc, - ASTNode joinCond, - ASTNode leftCondn, - ASTNode rightCondn, - List<String> leftCondAl1, - List<String> leftCondAl2, - List<String> rightCondAl1, - List<String> rightCondAl2) throws SemanticException { + JoinType type, + List<String> leftSrc, + ASTNode joinCond, + ASTNode leftCondn, + ASTNode rightCondn, + List<String> leftCondAl1, + List<String> leftCondAl2, + List<String> rightCondAl1, + List<String> rightCondAl2) throws SemanticException { if (leftCondAl1.size() != 0) { if ((rightCondAl1.size() != 0) || ((rightCondAl1.size() == 0) && (rightCondAl2.size() == 0))) { if (type.equals(JoinType.LEFTOUTER) || type.equals(JoinType.FULLOUTER)) { - joinTree.getFilters().get(0).add(joinCond); + joinTree.getFilters().get(0).add(joinCond); } else { /* * If the rhs references table sources and this QBJoinTree has a leftTree; @@ -2623,7 +2626,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { || ((rightCondAl1.size() == 0) && (rightCondAl2.size() == 0))) { if (type.equals(JoinType.RIGHTOUTER) || type.equals(JoinType.FULLOUTER)) { - joinTree.getFilters().get(1).add(joinCond); + joinTree.getFilters().get(1).add(joinCond); } else { joinTree.getFiltersForPushing().get(1).add(joinCond); } @@ -2638,14 +2641,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } else if (rightCondAl1.size() != 0) { if (type.equals(JoinType.LEFTOUTER) || type.equals(JoinType.FULLOUTER)) { - joinTree.getFilters().get(0).add(joinCond); + joinTree.getFilters().get(0).add(joinCond); } else { joinTree.getFiltersForPushing().get(0).add(joinCond); } } else { if (type.equals(JoinType.RIGHTOUTER) || type.equals(JoinType.FULLOUTER)) { - joinTree.getFilters().get(1).add(joinCond); + joinTree.getFilters().get(1).add(joinCond); } else if (type.equals(JoinType.LEFTSEMI)) { joinTree.getExpressions().get(0).add(leftCondn); joinTree.getExpressions().get(1).add(rightCondn); @@ -2661,7 +2664,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { @SuppressWarnings("rawtypes") private void parseJoinCondition(QBJoinTree joinTree, ASTNode joinCond, List<String> leftSrc, - Map<String, Operator> aliasToOpInfo) + Map<String, Operator> aliasToOpInfo) throws SemanticException { if (joinCond == null) { return; @@ -2700,8 +2703,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { */ @SuppressWarnings("rawtypes") private void parseJoinCondition(QBJoinTree joinTree, ASTNode joinCond, - List<String> leftSrc, JoinType type, - Map<String, Operator> aliasToOpInfo) throws SemanticException { + List<String> leftSrc, JoinType type, + Map<String, Operator> aliasToOpInfo) throws SemanticException { if (joinCond == null) { return; } @@ -2797,14 +2800,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { if (!leftAliasNull) { if (type.equals(JoinType.LEFTOUTER) || type.equals(JoinType.FULLOUTER)) { - joinTree.getFilters().get(0).add(joinCond); + joinTree.getFilters().get(0).add(joinCond); } else { joinTree.getFiltersForPushing().get(0).add(joinCond); } } else { if (type.equals(JoinType.RIGHTOUTER) || type.equals(JoinType.FULLOUTER)) { - joinTree.getFilters().get(1).add(joinCond); + joinTree.getFilters().get(1).add(joinCond); } else { joinTree.getFiltersForPushing().get(1).add(joinCond); } @@ -2817,7 +2820,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { @SuppressWarnings("rawtypes") private void extractJoinCondsFromWhereClause(QBJoinTree joinTree, QB qb, String dest, ASTNode predicate, - Map<String, Operator> aliasToOpInfo) throws SemanticException { + Map<String, Operator> aliasToOpInfo) throws SemanticException { switch (predicate.getType()) { case HiveParser.KW_AND: @@ -2834,7 +2837,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { ArrayList<String> leftCondAl2 = new ArrayList<String>(); try { parseJoinCondPopulateAlias(joinTree, leftCondn, leftCondAl1, leftCondAl2, - null, aliasToOpInfo); + null, aliasToOpInfo); } catch(SemanticException se) { // suppress here; if it is a real issue will get caught in where clause handling. return; @@ -2884,7 +2887,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { @SuppressWarnings("nls") public <T extends OperatorDesc> Operator<T> putOpInsertMap(Operator<T> op, - RowResolver rr) { + RowResolver rr) { OpParseContext ctx = new OpParseContext(rr); opParseCtx.put(op, ctx); op.augmentPlan(); @@ -2893,7 +2896,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { @SuppressWarnings("nls") private Operator genHavingPlan(String dest, QB qb, Operator input, - Map<String, Operator> aliasToOpInfo) + Map<String, Operator> aliasToOpInfo) throws SemanticException { ASTNode havingExpr = qb.getParseInfo().getHavingForClause(dest); @@ -2912,8 +2915,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // If CBO did not optimize the query, we might need to replace grouping function final String destClauseName = qb.getParseInfo().getClauseNames().iterator().next(); final boolean cubeRollupGrpSetPresent = (!qb.getParseInfo().getDestRollups().isEmpty() - || !qb.getParseInfo().getDestGroupingSets().isEmpty() - || !qb.getParseInfo().getDestCubes().isEmpty()); + || !qb.getParseInfo().getDestGroupingSets().isEmpty() + || !qb.getParseInfo().getDestCubes().isEmpty()); // Special handling of grouping function condn = rewriteGroupingFunctionAST(getGroupByForClause(qb.getParseInfo(), destClauseName), condn, !cubeRollupGrpSetPresent); @@ -2930,7 +2933,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } protected static ASTNode rewriteGroupingFunctionAST(final List<ASTNode> grpByAstExprs, ASTNode targetNode, - final boolean noneSet) throws SemanticException { + final boolean noneSet) throws SemanticException { TreeVisitorAction action = new TreeVisitorAction() { @@ -2953,9 +2956,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { ASTNode newRoot = new ASTNode(); // Rewritten grouping function ASTNode groupingFunc = (ASTNode) ParseDriver.adaptor.create( - HiveParser.Identifier, "grouping"); + HiveParser.Identifier, "grouping"); ParseDriver.adaptor.addChild(groupingFunc, ParseDriver.adaptor.create( - HiveParser.Identifier, "rewritten")); + HiveParser.Identifier, "rewritten")); newRoot.addChild(groupingFunc); // Grouping ID reference ASTNode childGroupingID; @@ -2963,13 +2966,13 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // Query does not contain CUBE, ROLLUP, or GROUPING SETS, and thus, // grouping should return 0 childGroupingID = (ASTNode) ParseDriver.adaptor.create(HiveParser.IntegralLiteral, - String.valueOf(0)); + String.valueOf(0)); } else { // We refer to grouping_id column childGroupingID = (ASTNode) ParseDriver.adaptor.create( - HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL"); + HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL"); ParseDriver.adaptor.addChild(childGroupingID, ParseDriver.adaptor.create( - HiveParser.Identifier, VirtualColumn.GROUPINGID.getName())); + HiveParser.Identifier, VirtualColumn.GROUPINGID.getName())); } newRoot.addChild(childGroupingID); // Indices @@ -2981,7 +2984,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // Create and add AST node with position of grouping function input // in group by clause ASTNode childN = (ASTNode) ParseDriver.adaptor.create(HiveParser.IntegralLiteral, - String.valueOf(IntMath.mod(-j-1, grpByAstExprs.size()))); + String.valueOf(IntMath.mod(-j-1, grpByAstExprs.size()))); newRoot.addChild(childN); break; } @@ -3013,8 +3016,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { @SuppressWarnings("nls") private Operator genFilterPlan(ASTNode searchCond, QB qb, Operator input, - Map<String, Operator> aliasToOpInfo, - boolean forHavingClause, boolean forGroupByClause) + Map<String, Operator> aliasToOpInfo, + boolean forHavingClause, boolean forGroupByClause) throws SemanticException { OpParseContext inputCtx = opParseCtx.get(input); @@ -3114,7 +3117,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { if ( subQuery.getOperator().getType() != SubQueryType.EXISTS && subQuery.getOperator().getType() != SubQueryType.NOT_EXISTS && sqRR.getColumnInfos().size() - - subQuery.getNumOfCorrelationExprsAddedToSQSelect() > 1 ) { + subQuery.getNumOfCorrelationExprsAddedToSQSelect() > 1 ) { throw new SemanticException(ErrorMsg.INVALID_SUBQUERY_EXPRESSION.getMsg( subQueryAST, "SubQuery can contain only 1 item in Select List.")); } @@ -3205,7 +3208,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * every non nullSafe predicate. */ private Operator genNotNullFilterForJoinSourcePlan(QB qb, Operator input, - QBJoinTree joinTree, ExprNodeDesc[] joinKeys) throws SemanticException { + QBJoinTree joinTree, ExprNodeDesc[] joinKeys) throws SemanticException { if (qb == null || joinTree == null) { return input; @@ -3226,7 +3229,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { List<Boolean> nullSafes = joinTree.getNullSafes(); for (int i = 0; i < joinKeys.length; i++) { if (nullSafes.get(i) || (joinKeys[i] instanceof ExprNodeColumnDesc && - ((ExprNodeColumnDesc)joinKeys[i]).getIsPartitionColOrVirtualCol())) { + ((ExprNodeColumnDesc)joinKeys[i]).getIsPartitionColOrVirtualCol())) { // no need to generate is not null predicate for partitioning or // virtual column, since those columns can never be null. continue; @@ -3273,11 +3276,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } @SuppressWarnings("nls") - // TODO: make aliases unique, otherwise needless rewriting takes place + // TODO: make aliases unique, otherwise needless rewriting takes place Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, - ArrayList<ExprNodeDesc> col_list, HashSet<ColumnInfo> excludeCols, RowResolver input, - RowResolver colSrcRR, Integer pos, RowResolver output, List<String> aliases, - boolean ensureUniqueCols) throws SemanticException { + ArrayList<ExprNodeDesc> col_list, HashSet<ColumnInfo> excludeCols, RowResolver input, + RowResolver colSrcRR, Integer pos, RowResolver output, List<String> aliases, + boolean ensureUniqueCols) throws SemanticException { if (colSrcRR == null) { colSrcRR = input; @@ -3514,7 +3517,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } private TableDesc getTableDescFromSerDe(ASTNode child, String cols, - String colTypes, boolean defaultCols) throws SemanticException { + String colTypes, boolean defaultCols) throws SemanticException { if (child.getType() == HiveParser.TOK_SERDENAME) { String serdeName = unescapeSQLString(child.getChild(0).getText()); Class<? extends Deserializer> serdeClass = null; @@ -3723,7 +3726,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // Input and Output Serdes if (trfm.getChild(inputSerDeNum).getChildCount() > 0) { inInfo = getTableDescFromSerDe((ASTNode) (((ASTNode) trfm - .getChild(inputSerDeNum))).getChild(0), inpColumns.toString(), + .getChild(inputSerDeNum))).getChild(0), inpColumns.toString(), inpColumnTypes.toString(), false); } else { inInfo = PlanUtils.getTableDesc(serde, Integer @@ -3733,7 +3736,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { if (trfm.getChild(outputSerDeNum).getChildCount() > 0) { outInfo = getTableDescFromSerDe((ASTNode) (((ASTNode) trfm - .getChild(outputSerDeNum))).getChild(0), columns.toString(), + .getChild(outputSerDeNum))).getChild(0), columns.toString(), columnTypes.toString(), false); // This is for backward compatibility. If the user did not specify the // output column list, we assume that there are 2 columns: key and value. @@ -3857,7 +3860,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } protected List<Integer> getGroupingSets(List<ASTNode> groupByExpr, QBParseInfo parseInfo, - String dest) throws SemanticException { + String dest) throws SemanticException { Map<String, Integer> exprPos = new HashMap<String, Integer>(); for (int i = 0; i < groupByExpr.size(); ++i) { ASTNode node = groupByExpr.get(i); @@ -3947,7 +3950,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } static String[] getColAlias(ASTNode selExpr, String defaultName, - RowResolver inputRR, boolean includeFuncName, int colNum) { + RowResolver inputRR, boolean includeFuncName, int colNum) { String colAlias = null; String tabAlias = null; String[] colRef = new String[2]; @@ -3955,7 +3958,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { //for queries with a windowing expressions, the selexpr may have a third child if (selExpr.getChildCount() == 2 || (selExpr.getChildCount() == 3 && - selExpr.getChild(2).getType() == HiveParser.TOK_WINDOWSPEC)) { + selExpr.getChild(2).getType() == HiveParser.TOK_WINDOWSPEC)) { // return zz for "xx + yy AS zz" colAlias = unescapeIdentifier(selExpr.getChild(1).getText().toLowerCase()); colRef[0] = tabAlias; @@ -4038,7 +4041,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { private Operator<?> genSelectPlan(String dest, QB qb, Operator<?> input, - Operator<?> inputForSelectStar) throws SemanticException { + Operator<?> inputForSelectStar) throws SemanticException { ASTNode selExprList = qb.getParseInfo().getSelForClause(dest); Operator<?> op = genSelectPlan(dest, selExprList, qb, input, inputForSelectStar, false); @@ -4051,7 +4054,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { @SuppressWarnings("nls") private Operator<?> genSelectPlan(String dest, ASTNode selExprList, QB qb, Operator<?> input, - Operator<?> inputForSelectStar, boolean outerLV) throws SemanticException { + Operator<?> inputForSelectStar, boolean outerLV) throws SemanticException { if (LOG.isDebugEnabled()) { LOG.debug("tree: " + selExprList.toStringTree()); @@ -4174,8 +4177,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } final boolean cubeRollupGrpSetPresent = (!qb.getParseInfo().getDestRollups().isEmpty() - || !qb.getParseInfo().getDestGroupingSets().isEmpty() - || !qb.getParseInfo().getDestCubes().isEmpty()); + || !qb.getParseInfo().getDestGroupingSets().isEmpty() + || !qb.getParseInfo().getDestCubes().isEmpty()); Set<String> colAliases = new HashSet<String>(); ASTNode[] exprs = new ASTNode[exprList.getChildCount()]; String[][] aliases = new String[exprList.getChildCount()][]; @@ -4229,7 +4232,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { if (expr.getType() == HiveParser.TOK_ALLCOLREF) { int initPos = pos; pos = genColListRegex(".*", expr.getChildCount() == 0 ? null - : getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(), + : getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(), expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); if (unparseTranslator.isEnabled()) { offset += pos - initPos - 1; @@ -4246,7 +4249,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } else if (expr.getType() == HiveParser.DOT && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL && inputRR.hasTableAlias(unescapeIdentifier(expr.getChild(0) - .getChild(0).getText().toLowerCase())) && !hasAsClause + .getChild(0).getText().toLowerCase())) && !hasAsClause && !inputRR.getIsExprResolver() && isRegex(unescapeIdentifier(expr.getChild(1).getText()), conf)) { // In case the expression is TABLE.COL (col can be regex). @@ -4254,7 +4257,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(1).getText()), unescapeIdentifier(expr.getChild(0).getChild(0).getText().toLowerCase()), - expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); + expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); } else { // Case when this is an expression TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR, true, isCBOExecuted()); @@ -4341,8 +4344,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * @throws SemanticException */ public RowResolver handleInsertStatementSpec(List<ExprNodeDesc> col_list, String dest, - RowResolver outputRR, RowResolver inputRR, QB qb, - ASTNode selExprList) throws SemanticException { + RowResolver outputRR, RowResolver inputRR, QB qb, + ASTNode selExprList) throws SemanticException { //(z,x) List<String> targetTableSchema = qb.getParseInfo().getDestSchemaForClause(dest);//specified in the query if(targetTableSchema == null) { @@ -4353,9 +4356,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { Table target = qb.getMetaData().getDestTableForAlias(dest); Partition partition = target == null ? qb.getMetaData().getDestPartitionForAlias(dest) : null; throw new SemanticException(generateErrorMessage(selExprList, - "Expected " + targetTableSchema.size() + " columns for " + dest + - (target != null ? "/" + target.getCompleteName() : (partition != null ? "/" + partition.getCompleteName() : "")) + - "; select produces " + col_list.size() + " columns")); + "Expected " + targetTableSchema.size() + " columns for " + dest + + (target != null ? "/" + target.getCompleteName() : (partition != null ? "/" + partition.getCompleteName() : "")) + + "; select produces " + col_list.size() + " columns")); } //e.g. map z->expr for a Map<String, ExprNodeDesc> targetCol2Projection = new HashMap<String, ExprNodeDesc>(); @@ -4370,7 +4373,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { Partition partition = target == null ? qb.getMetaData().getDestPartitionForAlias(dest) : null; if(target == null && partition == null) { throw new SemanticException(generateErrorMessage(selExprList, - "No table/partition found in QB metadata for dest='" + dest + "'")); + "No table/partition found in QB metadata for dest='" + dest + "'")); } ArrayList<ExprNodeDesc> new_col_list = new ArrayList<ExprNodeDesc>(); colListPos = 0; @@ -4410,7 +4413,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { new_col_list.add(exp); final String tableAlias = null;//this column doesn't come from any table ColumnInfo colInfo = new ColumnInfo(getColumnInternalName(colListPos), - exp.getWritableObjectInspector(), tableAlias, false); + exp.getWritableObjectInspector(), tableAlias, false); newOutputRR.put(colInfo.getTabAlias(), colInfo.getInternalName(), colInfo); } colListPos++; @@ -4486,8 +4489,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * for each GroupBy aggregation. */ public static GenericUDAFEvaluator getGenericUDAFEvaluator(String aggName, - ArrayList<ExprNodeDesc> aggParameters, ASTNode aggTree, - boolean isDistinct, boolean isAllColumns) + ArrayList<ExprNodeDesc> aggParameters, ASTNode aggTree, + boolean isDistinct, boolean isAllColumns) throws SemanticException { ArrayList<ObjectInspector> originalParameterTypeInfos = getWritableObjectInspector(aggParameters); @@ -4514,7 +4517,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * when the UDAF is not found or has problems. */ public static GenericUDAFInfo getGenericUDAFInfo(GenericUDAFEvaluator evaluator, - GenericUDAFEvaluator.Mode emode, ArrayList<ExprNodeDesc> aggParameters) + GenericUDAFEvaluator.Mode emode, ArrayList<ExprNodeDesc> aggParameters) throws SemanticException { GenericUDAFInfo r = new GenericUDAFInfo(); @@ -4586,7 +4589,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * a constant parameter; otherwise, return null */ public static ExprNodeDesc isConstantParameterInAggregationParameters(String internalName, - List<ExprNodeDesc> reduceValues) { + List<ExprNodeDesc> reduceValues) { // only the pattern of "VALUE._col([0-9]+)" should be handled. String[] terms = internalName.split("\\."); @@ -4624,8 +4627,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { */ @SuppressWarnings("nls") private Operator genGroupByPlanGroupByOperator(QBParseInfo parseInfo, - String dest, Operator input, ReduceSinkOperator rs, GroupByDesc.Mode mode, - Map<String, GenericUDAFEvaluator> genericUDAFEvaluators) + String dest, Operator input, ReduceSinkOperator rs, GroupByDesc.Mode mode, + Map<String, GenericUDAFEvaluator> genericUDAFEvaluators) throws SemanticException { RowResolver groupByInputRowResolver = opParseCtx .get(input).getRowResolver(); @@ -4758,10 +4761,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // This function is called for GroupBy2 to pass the additional grouping keys introduced by // GroupBy1 for the grouping set (corresponding to the rollup). private void addGroupingSetKey(List<ExprNodeDesc> groupByKeys, - RowResolver groupByInputRowResolver, - RowResolver groupByOutputRowResolver, - List<String> outputColumnNames, - Map<String, ExprNodeDesc> colExprMap) throws SemanticException { + RowResolver groupByInputRowResolver, + RowResolver groupByOutputRowResolver, + List<String> outputColumnNames, + Map<String, ExprNodeDesc> colExprMap) throws SemanticException { // For grouping sets, add a dummy grouping key String groupingSetColumnName = groupByInputRowResolver.get(null, VirtualColumn.GROUPINGID.getName()).getInternalName(); @@ -4789,10 +4792,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // This function is called for ReduceSink to add the additional grouping keys introduced by // GroupBy1 into the reduce keys. private void processGroupingSetReduceSinkOperator(RowResolver reduceSinkInputRowResolver, - RowResolver reduceSinkOutputRowResolver, - List<ExprNodeDesc> reduceKeys, - List<String> outputKeyColumnNames, - Map<String, ExprNodeDesc> colExprMap) throws SemanticException { + RowResolver reduceSinkOutputRowResolver, + List<ExprNodeDesc> reduceKeys, + List<String> outputKeyColumnNames, + Map<String, ExprNodeDesc> colExprMap) throws SemanticException { // add a key for reduce sink String groupingSetColumnName = reduceSinkInputRowResolver.get(null, VirtualColumn.GROUPINGID.getName()).getInternalName(); @@ -4832,11 +4835,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { */ @SuppressWarnings("nls") private Operator genGroupByPlanGroupByOperator1(QBParseInfo parseInfo, - String dest, Operator reduceSinkOperatorInfo, GroupByDesc.Mode mode, - Map<String, GenericUDAFEvaluator> genericUDAFEvaluators, - List<Integer> groupingSets, - boolean groupingSetsPresent, - boolean groupingSetsNeedAdditionalMRJob) throws SemanticException { + String dest, Operator reduceSinkOperatorInfo, GroupByDesc.Mode mode, + Map<String, GenericUDAFEvaluator> genericUDAFEvaluators, + List<Integer> groupingSets, + boolean groupingSetsPresent, + boolean groupingSetsNeedAdditionalMRJob) throws SemanticException { ArrayList<String> outputColumnNames = new ArrayList<String>(); RowResolver groupByInputRowResolver = opParseCtx .get(reduceSinkOperatorInfo).getRowResolver(); @@ -4974,7 +4977,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { assert (paraExpression != null); aggParameters.add(new ExprNodeColumnDesc(paraExprInfo.getType(), paraExpression, paraExprInfo.getTabAlias(), paraExprInfo - .getIsVirtualCol())); + .getIsVirtualCol())); } if (isDistinct) { numDistinctUDFs++; @@ -5023,9 +5026,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * creates 'n' rows per input row, where 'n' is the number of grouping sets. */ private void createNewGroupingKey(List<ExprNodeDesc> groupByKeys, - List<String> outputColumnNames, - RowResolver groupByOutputRowResolver, - Map<String, ExprNodeDesc> colExprMap) { + List<String> outputColumnNames, + RowResolver groupByOutputRowResolver, + Map<String, ExprNodeDesc> colExprMap) { // The value for the constant does not matter. It is replaced by the grouping set // value for the actual implementation ExprNodeConstantDesc constant = new ExprNodeConstantDesc(0); @@ -5056,13 +5059,13 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { */ @SuppressWarnings("nls") private Operator genGroupByPlanMapGroupByOperator(QB qb, - String dest, - List<ASTNode> grpByExprs, - Operator inputOperatorInfo, - GroupByDesc.Mode mode, - Map<String, GenericUDAFEvaluator> genericUDAFEvaluators, - List<Integer> groupingSetKeys, - boolean groupingSetsPresent) throws SemanticException { + String dest, + List<ASTNode> grpByExprs, + Operator inputOperatorInfo, + GroupByDesc.Mode mode, + Map<String, GenericUDAFEvaluator> genericUDAFEvaluators, + List<Integer> groupingSetKeys, + boolean groupingSetsPresent) throws SemanticException { RowResolver groupByInputRowResolver = opParseCtx.get(inputOperatorInfo) .getRowResolver(); @@ -5206,14 +5209,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { */ @SuppressWarnings("nls") private ReduceSinkOperator genGroupByPlanReduceSinkOperator(QB qb, - String dest, - Operator inputOperatorInfo, - List<ASTNode> grpByExprs, - int numPartitionFields, - boolean changeNumPartitionFields, - int numReducers, - boolean mapAggrDone, - boolean groupingSetsPresent) throws SemanticException { + String dest, + Operator inputOperatorInfo, + List<ASTNode> grpByExprs, + int numPartitionFields, + boolean changeNumPartitionFields, + int numReducers, + boolean mapAggrDone, + boolean groupingSetsPresent) throws SemanticException { RowResolver reduceSinkInputRowResolver = opParseCtx.get(inputOperatorInfo) .getRowResolver(); @@ -5295,8 +5298,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } private ArrayList<ExprNodeDesc> getReduceKeysForReduceSink(List<ASTNode> grpByExprs, String dest, - RowResolver reduceSinkInputRowResolver, RowResolver reduceSinkOutputRowResolver, - List<String> outputKeyColumnNames, Map<String, ExprNodeDesc> colExprMap) + RowResolver reduceSinkInputRowResolver, RowResolver reduceSinkOutputRowResolver, + List<String> outputKeyColumnNames, Map<String, ExprNodeDesc> colExprMap) throws SemanticException { ArrayList<ExprNodeDesc> reduceKeys = new ArrayList<ExprNodeDesc>(); @@ -5332,10 +5335,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } private List<List<Integer>> getDistinctColIndicesForReduceSink(QBParseInfo parseInfo, - String dest, - List<ExprNodeDesc> reduceKeys, RowResolver reduceSinkInputRowResolver, - RowResolver reduceSinkOutputRowResolver, List<String> outputKeyColumnNames, - Map<String, ExprNodeDesc> colExprMap) + String dest, + List<ExprNodeDesc> reduceKeys, RowResolver reduceSinkInputRowResolver, + RowResolver reduceSinkOutputRowResolver, List<String> outputKeyColumnNames, + Map<String, ExprNodeDesc> colExprMap) throws SemanticException { List<List<Integer>> distinctColIndices = new ArrayList<List<Integer>>(); @@ -5384,9 +5387,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } private void getReduceValuesForReduceSinkNoMapAgg(QBParseInfo parseInfo, String dest, - RowResolver reduceSinkInputRowResolver, RowResolver reduceSinkOutputRowResolver, - List<String> outputValueColumnNames, ArrayList<ExprNodeDesc> reduceValues, - Map<String, ExprNodeDesc> colExprMap) throws SemanticException { + RowResolver reduceSinkInputRowResolver, RowResolver reduceSinkOutputRowResolver, + List<String> outputValueColumnNames, ArrayList<ExprNodeDesc> reduceValues, + Map<String, ExprNodeDesc> colExprMap) throws SemanticException { HashMap<String, ASTNode> aggregationTrees = parseInfo .getAggregationExprsForClause(dest); @@ -5414,7 +5417,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { @SuppressWarnings("nls") private ReduceSinkOperator genCommonGroupByPlanReduceSinkOperator(QB qb, List<String> dests, - Operator inputOperatorInfo) throws SemanticException { + Operator inputOperatorInfo) throws SemanticException { RowResolver reduceSinkInputRowResolver = opParseCtx.get(inputOperatorInfo) .getRowResolver(); @@ -5503,7 +5506,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // from mapping if it's already on RS keys. // Remaining column expressions would be a candidate for an RS value private void removeMappingForKeys(ASTNode predicate, Map<ASTNode, ExprNodeDesc> mapping, - List<ExprNodeDesc> keys) { + List<ExprNodeDesc> keys) { ExprNodeDesc expr = mapping.get(predicate); if (expr != null && ExprNodeDescUtils.indexOf(expr, keys) >= 0) { removeRecursively(predicate, mapping); @@ -5540,11 +5543,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { */ @SuppressWarnings("nls") private Operator genGroupByPlanReduceSinkOperator2MR(QBParseInfo parseInfo, - String dest, - Operator groupByOperatorInfo, - int numPartitionFields, - int numReducers, - boolean groupingSetsPresent) throws SemanticException { + String dest, + Operator groupByOperatorInfo, + int numPartitionFields, + int numReducers, + boolean groupingSetsPresent) throws SemanticException { RowResolver reduceSinkInputRowResolver2 = opParseCtx.get( groupByOperatorInfo).getRowResolver(); RowResolver reduceSinkOutputRowResolver2 = new RowResolver(); @@ -5628,11 +5631,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { */ @SuppressWarnings("nls") private Operator genGroupByPlanGroupByOperator2MR(QBParseInfo parseInfo, - String dest, - Operator reduceSinkOperatorInfo2, - GroupByDesc.Mode mode, - Map<String, GenericUDAFEvaluator> genericUDAFEvaluators, - boolean groupingSetsPresent) throws SemanticException { + String dest, + Operator reduceSinkOperatorInfo2, + GroupByDesc.Mode mode, + Map<String, GenericUDAFEvaluator> genericUDAFEvaluators, + boolean groupingSetsPresent) throws SemanticException { RowResolver groupByInputRowResolver2 = opParseCtx.get( reduceSinkOperatorInfo2).getRowResolver(); @@ -5688,7 +5691,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { assert (paraExpression != null); aggParameters.add(new ExprNodeColumnDesc(paraExprInfo.getType(), paraExpression, paraExprInfo.getTabAlias(), paraExprInfo - .getIsVirtualCol())); + .getIsVirtualCol())); String aggName = unescapeIdentifier(value.getChild(0).getText()); @@ -5707,7 +5710,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { udaf.genericUDAFEvaluator, udaf.convertedParameters, (mode != GroupByDesc.Mode.FINAL && value.getToken().getType() == - HiveParser.TOK_FUNCTIONDI), + HiveParser.TOK_FUNCTIONDI), amode)); String field = getColumnInternalName(groupByKeys.size() + aggregations.size() - 1); @@ -5797,7 +5800,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { @SuppressWarnings({"nls"}) private Operator genGroupByPlan1ReduceMultiGBY(List<String> dests, QB qb, Operator input, - Map<String, Operator> aliasToOpInfo) + Map<String, Operator> aliasToOpInfo) throws SemanticException { QBParseInfo parseInfo = qb.getParseInfo(); @@ -5861,7 +5864,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { orFilterDesc.setGenerated(true); selectInput = putOpInsertMap(OperatorFactory.getAndMakeChild(orFilterDesc, new RowSchema( - inputRR.getColumnInfos()), input), inputRR); + inputRR.getColumnInfos()), input), inputRR); } // insert a select operator here used by the ColumnPruner to reduce @@ -6025,7 +6028,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } static private void extractColumns(Set<String> colNamesExprs, - ExprNodeDesc exprNode) throws SemanticException { + ExprNodeDesc exprNode) throws SemanticException { if (exprNode instanceof ExprNodeColumnDesc) { colNamesExprs.add(((ExprNodeColumnDesc) exprNode).getColumn()); return; @@ -6050,9 +6053,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } void checkExpressionsForGroupingSet(List<ASTNode> grpByExprs, - List<ASTNode> distinctGrpByExprs, - Map<String, ASTNode> aggregationTrees, - RowResolver inputRowResolver) throws SemanticException { + List<ASTNode> distinctGrpByExprs, + Map<String, ASTNode> aggregationTrees, + RowResolver inputRowResolver) throws SemanticException { Set<String> colNamesGroupByExprs = new HashSet<String>(); Set<String> colNamesGroupByDistinctExprs = new HashSet<String>(); @@ -6149,7 +6152,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { */ @SuppressWarnings("nls") private Operator genGroupByPlanMapAggrNoSkew(String dest, QB qb, - Operator inputOperatorInfo) throws SemanticException { + Operator inputOperatorInfo) throws SemanticException { QBParseInfo parseInfo = qb.getParseInfo(); ObjectPair<List<ASTNode>, List<Integer>> grpByExprsGroupingSets = @@ -6317,7 +6320,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { */ @SuppressWarnings("nls") private Operator genGroupByPlanMapAggr2MR(String dest, QB qb, - Operator inputOperatorInfo) throws SemanticException { + Operator inputOperatorInfo) throws SemanticException { QBParseInfo parseInfo = qb.getParseInfo(); @@ -6501,7 +6504,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { @SuppressWarnings("nls") private Operator genBucketingSortingDest(String dest, Operator input, QB qb, - TableDesc table_desc, Table dest_tab, SortBucketRSCtx ctx) throws SemanticException { + TableDesc table_desc, Table dest_tab, SortBucketRSCtx ctx) throws SemanticException { // If the table is bucketed, and bucketing is enforced, do the following: // If the number of buckets is smaller than the number of maximum reducers, @@ -6571,7 +6574,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { nullOrder.append(sortOrder == BaseSemanticAnalyzer.HIVE_COLUMN_ORDER_ASC ? 'a' : 'z'); } input = genReduceSinkPlan(input, partnCols, sortCols, order.toString(), nullOrder.toString(), - maxReducers, (AcidUtils.isAcidTable(dest_tab) ? + maxReducers, (AcidUtils.isAcidTable(dest_tab) ? getAcidType(table_desc.getOutputFileFormatClass(), dest) : AcidUtils.Operation.NOT_ACID)); reduceSinkOperatorsAddedByEnforceBucketingSorting.add((ReduceSinkOperator)input.getParentOperators().get(0)); ctx.setMultiFileSpray(multiFileSpray); @@ -6582,7 +6585,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } private void genPartnCols(String dest, Operator input, QB qb, - TableDesc table_desc, Table dest_tab, SortBucketRSCtx ctx) throws SemanticException { + TableDesc table_desc, Table dest_tab, SortBucketRSCtx ctx) throws SemanticException { boolean enforceBucketing = false; ArrayList<ExprNodeDesc> partnColsNoConvert = new ArrayList<ExprNodeDesc>(); @@ -6627,6 +6630,128 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { this.rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTblDesc), conf)); } + private ImmutableBitSet getEnabledNotNullConstraints(Table tbl) throws HiveException{ + List<Boolean> nullConstraints = new ArrayList<>(); + final NotNullConstraint nnc = Hive.get().getEnabledNotNullConstraints( + tbl.getDbName(), tbl.getTableName()); + ImmutableBitSet bitSet = null; + if(nnc == null || nnc.getNotNullConstraints().isEmpty()) { + return bitSet; + } + // Build the bitset with not null columns + ImmutableBitSet.Builder builder = ImmutableBitSet.builder(); + for (String nnCol : nnc.getNotNullConstraints().values()) { + int nnPos = -1; + for (int i = 0; i < tbl.getCols().size(); i++) { + if (tbl.getCols().get(i).getName().equals(nnCol)) { + nnPos = i; + builder.set(nnPos); + break; + } + } + } + bitSet = builder.build(); + return bitSet; + } + + private boolean mergeCardinalityViolationBranch(final Operator input) { + if(input instanceof SelectOperator) { + SelectOperator selectOp = (SelectOperator)input; + if(selectOp.getConf().getColList().size() == 1) { + ExprNodeDesc colExpr = selectOp.getConf().getColList().get(0); + if(colExpr instanceof ExprNodeGenericFuncDesc) { + ExprNodeGenericFuncDesc func = (ExprNodeGenericFuncDesc)colExpr ; + if(func.getGenericUDF() instanceof GenericUDFCardinalityViolation){ + return true; + } + } + } + } + return false; + } + + private Operator + genIsNotNullConstraint(String dest, QB qb, Operator input) + throws SemanticException { + + boolean forceNotNullConstraint = conf.getBoolVar(ConfVars.HIVE_ENFORCE_NOT_NULL_CONSTRAINT); + if(!forceNotNullConstraint) { + return input; + } + + if(deleting(dest)) { + // for DELETE statements NOT NULL constraint need not be checked + return input; + } + + //MERGE statements could have inserted a cardinality violation branch, we need to avoid that + if(mergeCardinalityViolationBranch(input)){ + return input; + } + + // if this is an insert into statement we might need to add constraint check + Table targetTable = null; + Integer dest_type = qb.getMetaData().getDestTypeForAlias(dest); + if(dest_type == QBMetaData.DEST_TABLE) { + targetTable= qb.getMetaData().getDestTableForAlias(dest); + + } + else if(dest_type == QBMetaData.DEST_PARTITION){ + Partition dest_part = qb.getMetaData().getDestPartitionForAlias(dest); + targetTable = dest_part.getTable(); + + } + else { + throw new SemanticException("Generating NOT NULL constraint check: Invalid target type: " + dest); + } + ImmutableBitSet nullConstraintBitSet = null; + try { + nullConstraintBitSet = getEnabledNotNullConstraints(targetTable); + } catch (Exception e) { + if (e instanceof SemanticException) { + throw (SemanticException) e; + } else { + throw (new RuntimeException(e)); + } + } + if(nullConstraintBitSet == null) { + return input; + } + List<ColumnInfo> colInfos = input.getSchema().getSignature(); + + ExprNodeDesc currUDF = null; + int constraintIdx = 0; + for(int colExprIdx=0; colExprIdx < colInfos.size(); colExprIdx++) { + if(updating(dest) && colExprIdx == 0) { + // for updates first column is _rowid + continue; + } + if (nullConstraintBitSet.indexOf(constraintIdx) != -1) { + ExprNodeDesc currExpr = TypeCheckProcFactory.toExprNodeDesc(colInfos.get(colExprIdx)); + ExprNodeDesc isNotNullUDF = TypeCheckProcFactory.DefaultExprProcessor. + getFuncExprNodeDesc("isnotnull", currExpr); + ExprNodeDesc constraintUDF = TypeCheckProcFactory.DefaultExprProcessor. + getFuncExprNodeDesc("enforce_constraint", isNotNullUDF); + if (currUDF != null) { + currUDF = TypeCheckProcFactory.DefaultExprProcessor. + getFuncExprNodeDesc("and", currUDF, constraintUDF); + } else { + currUDF = constraintUDF; + } + } + constraintIdx++; + } + if (currUDF != null) { + assert (input.getParentOperators().size() == 1); + RowResolver inputRR = opParseCtx.get(input).getRowResolver(); + Operator newConstraintFilter = putOpInsertMap(OperatorFactory.getAndMakeChild( + new FilterDesc(currUDF, false), new RowSchema( + inputRR.getColumnInfos()), input), inputRR); + + return newConstraintFilter; + } + return input; + } @SuppressWarnings("nls") protected Operator genFileSinkPlan(String dest, QB qb, Operator input) throws SemanticException { @@ -6712,6 +6837,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // this table_desc does not contain the partitioning columns table_desc = Utilities.getTableDesc(dest_tab); + // Add NOT NULL constraint check + input = genIsNotNullConstraint(dest, qb, input); + // Add sorting/bucketing if needed input = genBucketingSortingDest(dest, input, qb, table_desc, dest_tab, rsCtx); @@ -6736,7 +6864,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { txnId = SessionState.get().getTxnMgr().getCurrentTxnId(); } else { txnId = acidOp == Operation.NOT_ACID ? null : - SessionState.get().getTxnMgr().getCurrentTxnId(); + SessionState.get().getTxnMgr().getCurrentTxnId(); } boolean isReplace = !qb.getParseInfo().isInsertIntoTable( dest_tab.getDbName(), dest_tab.getTableName()); @@ -6744,8 +6872,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // For Acid table, Insert Overwrite shouldn't replace the table content. We keep the old // deltas and base and leave them up to the cleaner to clean up LoadFileType loadType = (!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(), - dest_tab.getTableName()) && !destTableIsTransactional) - ? LoadFileType.REPLACE_ALL : LoadFileType.KEEP_EXISTING; + dest_tab.getTableName()) && !destTableIsTransactional) + ? LoadFileType.REPLACE_ALL : LoadFileType.KEEP_EXISTING; ltd.setLoadFileType(loadType); ltd.setLbCtx(lbCtx); loadTableWork.add(ltd); @@ -6755,7 +6883,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { setStatsForNonNativeTable(dest_tab); // true if it is insert overwrite. boolean overwrite = !qb.getParseInfo().isInsertIntoTable( - String.format("%s.%s", dest_tab.getDbName(), dest_tab.getTableName())); + String.format("%s.%s", dest_tab.getDbName(), dest_tab.getTableName())); createInsertDesc(dest_tab, overwrite); } @@ -6791,6 +6919,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } table_desc = Utilities.getTableDesc(dest_tab); + // Add NOT NULL constraint check + input = genIsNotNullConstraint(dest, qb, input); + // Add sorting/bucketing if needed input = genBucketingSortingDest(dest, input, qb, table_desc, dest_tab, rsCtx); @@ -6811,20 +6942,20 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { txnId = SessionState.get().getTxnMgr().getCurrentTxnId(); } else { txnId = (acidOp == Operation.NOT_ACID) ? null : - SessionState.get().getTxnMgr().getCurrentTxnId(); + SessionState.get().getTxnMgr().getCurrentTxnId(); } ltd = new LoadTableDesc(queryTmpdir, table_desc, dest_part.getSpec(), acidOp, txnId); // For Acid table, Insert Overwrite shouldn't replace the table content. We keep the old // deltas and base and leave them up to the cleaner to clean up LoadFileType loadType = (!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(), - dest_tab.getTableName()) && !destTableIsTransactional) // // Both Full-acid and MM tables are excluded. - ? LoadFileType.REPLACE_ALL : LoadFileType.KEEP_EXISTING; + dest_tab.getTableName()) && !destTableIsTransactional) // // Both Full-acid and MM tables are excluded. + ? LoadFileType.REPLACE_ALL : LoadFileType.KEEP_EXISTING; ltd.setLoadFileType(loadType); ltd.setLbCtx(lbCtx); loadTableWork.add(ltd); if (!outputs.add(new WriteEntity(dest_part, - determineWriteType(ltd, dest_tab.isNonNative(), dest)))) { + determineWriteType(ltd, dest_tab.isNonNative(), dest)))) { throw new SemanticException(ErrorMsg.OUTPUT_SPECIFIED_MULTIPLE_TIMES .getMsg(dest_tab.getTableName() + "@" + dest_part.getName())); @@ -6872,7 +7003,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { queryTmpdir = isMmTable ? qPath : ctx.getTempDirForFinalJobPath(qPath); if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("Setting query directory " + queryTmpdir - + " from " + dest_path + " (" + isMmTable + ")"); + + " from " + dest_path + " (" + isMmTable + ")"); } } catch (Exception e) { throw new SemanticException("Error creating temporary folder on: " @@ -6905,8 +7036,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // Create LFD even for MM CTAS - it's a no-op move, but it still seems to be used for stats. loadFileWork.add(new LoadFileDesc(tblDesc, viewDesc, queryTmpdir, dest_path, isDfsDir, cols, colTypes, - destTableIsFullAcid ?//there is a change here - prev version had 'transadtional', one beofre' acid' - Operation.INSERT : Operation.NOT_ACID, + destTableIsFullAcid ?//there is a change here - prev version had 'transadtional', one beofre' acid' + Operation.INSERT : Operation.NOT_ACID, isMmCtas)); if (tblDesc == null) { if (viewDesc != null) { @@ -6914,23 +7045,23 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } else if (qb.getIsQuery()) { String fileFormat; if (SessionState.get().getIsUsingThriftJDBCBinarySerDe()) { - fileFormat = "SequenceFile"; - HiveConf.setVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT, fileFormat); - table_desc= - PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, fileFormat, - ThriftJDBCBinarySerDe.class); - // Set the fetch formatter to be a no-op for the ListSinkOperator, since we'll - // write out formatted thrift objects to SequenceFile - conf.set(SerDeUtils.LIST_SINK_OUTPUT_FORMATTER, NoOpFetchFormatter.class.getName()); + fileFormat = "SequenceFile"; + HiveConf.setVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT, fileFormat); + table_desc= + PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, fileFormat, + ThriftJDBCBinarySerDe.class); + // Set the fetch formatter to be a no-op for the ListSinkOperator, since we'll + // write out formatted thrift objects to SequenceFile + conf.set(SerDeUtils.LIST_SINK_OUTPUT_FORMATTER, NoOpFetchFormatter.class.getName()); } else { - fileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT); - Class<? extends Deserializer> serdeClass = LazySimpleSerDe.class; - if (fileFormat.equals(PlanUtils.LLAP_OUTPUT_FORMAT_KEY)) { - serdeClass = LazyBinarySerDe2.class; - } - table_desc = - PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, fileFormat, - serdeClass); + fileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT); + Class<? extends Deserializer> serdeClass = LazySimpleSerDe.class; + if (fileFormat.equals(PlanUtils.LLAP_OUTPUT_FORMAT_KEY)) { + serdeClass = LazyBinarySerDe2.class; + } + table_desc = + PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, fileFormat, + serdeClass); } } else { table_desc = PlanUtils.getDefaultTableDesc(qb.getDirectoryDesc(), cols, colTypes); @@ -7009,12 +7140,12 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } if (SessionState.get().isHiveServerQuery() && - null != table_desc && - table_desc.getSerdeClassName().equalsIgnoreCase(ThriftJDBCBinarySerDe.class.getName()) && - HiveConf.getBoolVar(conf,HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_SERIALIZE_IN_TASKS)) { - fileSinkDesc.setIsUsingThriftJDBCBinarySerDe(true); + null != table_desc && + table_desc.getSerdeClassName().equalsIgnoreCase(ThriftJDBCBinarySerDe.class.getName()) && + HiveConf.getBoolVar(conf,HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_SERIALIZE_IN_TASKS)) { + fileSinkDesc.setIsUsingThriftJDBCBinarySerDe(true); } else { - fileSinkDesc.setIsUsingThriftJDBCBinarySerDe(false); + fileSinkDesc.setIsUsingThriftJDBCBinarySerDe(false); } Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( @@ -7072,7 +7203,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { String typeName = colInfo.getType().getTypeName(); // CTAS should NOT create a VOID type if (typeName.equals(serdeConstants.VOID_TYPE_NAME)) { - throw new SemanticException(ErrorMsg.CTAS_CREATES_VOID_TYPE.getMsg(colName)); + throw new SemanticException(ErrorMsg.CTAS_CREATES_VOID_TYPE.getMsg(colName)); } col.setType(typeName); field_schemas.add(col); @@ -7108,31 +7239,31 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } private FileSinkDesc createFileSinkDesc(String dest, TableDesc table_desc, - Partition dest_part, Path dest_path, int currentTableId, - boolean destTableIsAcid, boolean destTableIsTemporary, - boolean destTableIsMaterialization, Path queryTmpdir, - SortBucketRSCtx rsCtx, DynamicPartitionCtx dpCtx, ListBucketingCtx lbCtx, - RowSchema fsRS, boolean canBeMerged, Table dest_tab, Long mmWriteId, boolean isMmCtas, - Integer dest_type, QB qb) throws SemanticException { + Partition dest_part, Path dest_path, int currentTableId, + boolean destTableIsAcid, boolean destTableIsTemporary, + boolean destTableIsMaterialization, Path queryTmpdir, + SortBucketRSCtx rsCtx, DynamicPartitionCtx dpCtx, ListBucketingCtx lbCtx, + RowSchema fsRS, boolean canBeMerged, Table dest_tab, Long mmWriteId, boolean isMmCtas, + Integer dest_type,
<TRUNCATED>