pengzhiwei2018 commented on a change in pull request #1138: [CALCITE-1581] UDTF
like in hive
URL: https://github.com/apache/calcite/pull/1138#discussion_r270616194
##########
File path:
core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
##########
@@ -1320,6 +1331,214 @@ protected SqlNode performUnconditionalRewrites(
return node;
}
+ /**
+ * Rewrite Hive like udtf grammar to the LATERAL TABLE.
+ * <p> eg. rewrite the
+ * "select a.id, table_func(a.id) as (f0,f1) from a"
+ * to
+ * "select a.id, _table_function_0.f0,_table_function_0.f1 from a,
+ * lateral table(table_func(a.id)) as _table_function_0(f0,f1)"
+ *
+ * @param node the SqlNode to rewrite
+ * @return the SqlNode after rewrite
+ */
+ private SqlNode performHiveUdtfRewrite(SqlNode node) {
+ // Mapping of SqlSelect and it's TableFunction Info
+ Map<SqlSelect, TableFunctionInfo> select2TableFunctionInfos = new
HashMap<>();
+ return performHiveUdtfRewriteInternal(node, select2TableFunctionInfos);
+ }
+
+ private SqlNode performHiveUdtfRewriteInternal(SqlNode current,
+ Map<SqlSelect, TableFunctionInfo>
select2TableFunctionInfos) {
+ // do the rewrite for SqlSelect
+ if (current instanceof SqlSelect) {
+ SqlSelect select = (SqlSelect) current;
+ // rewrite select items
+ SqlNodeList newSelectItem =
+ performRewriteForSelectItem(select,
+ select.getSelectList(), select2TableFunctionInfos);
+
+ TableFunctionInfo tableFunctionInfo =
select2TableFunctionInfos.get(select);
+ // if the select items contain a table function,
+ // join the from node with the table function.
+ if (tableFunctionInfo != null && select.getFrom() != null) {
+ SqlBasicCall joinRight = createLateralTable(tableFunctionInfo);
+ SqlNode newFrom = new SqlJoin(
+ SqlParserPos.ZERO,
+ select.getFrom(),
+ SqlLiteral.createBoolean(false, SqlParserPos.ZERO),
+ SqlLiteral.createSymbol(JoinType.COMMA, SqlParserPos.ZERO),
+ joinRight,
+ SqlLiteral.createSymbol(JoinConditionType.NONE,
SqlParserPos.ZERO), null);
+ select.setSelectList(newSelectItem);
+ select.setFrom(newFrom);
+ }
+ }
+ // recursive all sub-node of the node,ensure all
+ // SqlSelect can be rewrite.
+ if (current instanceof SqlCall) {
+ SqlCall call = (SqlCall) current;
+ List<SqlNode> newOperands = new ArrayList<>();
+ for (int i = 0; i < call.getOperandList().size(); i++) {
+ newOperands.add(performHiveUdtfRewriteInternal
+ (call.getOperandList().get(i), select2TableFunctionInfos));
+ }
+
+ for (int i = 0; i < newOperands.size(); i++) {
+ if (newOperands.get(i) != null) {
+ call.setOperand(i, newOperands.get(i));
+ }
+ }
+ } else if (current instanceof SqlNodeList) {
+ SqlNodeList nodeList = (SqlNodeList) current;
+ List<SqlNode> newNodes = new ArrayList<>();
+ for (int i = 0; i < nodeList.size(); i++) {
+ newNodes.add(
+ performHiveUdtfRewriteInternal(
+ nodeList.get(i), select2TableFunctionInfos));
+ }
+
+ for (int i = 0; i < newNodes.size(); i++) {
+ if (newNodes.get(i) != null) {
+ nodeList.set(i, newNodes.get(i));
+ }
+ }
+ }
+ return current;
+ }
+
+ /**
+ * Rewrite the "select a.id table_func(a.id) as (f0,f1)" to
+ * "select a.id,_table_function_0.f0, _table_function_0.f1"
+ * @param select SqlSelect Node
+ * @param selectItems select items to rewrite
+ * @param select2TableFunctionInfos Mapping of SqlSelect and it's
TableFunction
+ * @return new SelectItems after rewrite
+ */
+ private SqlNodeList performRewriteForSelectItem(SqlSelect select,
SqlNodeList selectItems,
+ Map<SqlSelect, TableFunctionInfo>
select2TableFunctionInfos) {
+ // step1. find the table function in the select items.
+ for (int i = 0; i < selectItems.size(); i++) {
+ SqlNode selectItem = selectItems.get(i);
+ if (selectItem.getKind() == SqlKind.AS) {
+ SqlNode udtfNode = ((SqlBasicCall) selectItem).getOperands()[0];
+ SqlNode aliasNode = ((SqlBasicCall) selectItem).getOperands()[1];
+
+ // test if this is a "table_func() as (f0,f1)" select item.
+ if (udtfNode instanceof SqlBasicCall
+ && ((SqlBasicCall) udtfNode).getOperator() instanceof SqlFunction
+ && aliasNode instanceof SqlNodeList) {
+
+ SqlFunction function = (SqlFunction)
+ ((SqlBasicCall) udtfNode).getOperator();
+ List<SqlOperator> overloads = new ArrayList<>();
+ opTab.lookupOperatorOverloads(function.getNameAsId(),
+ SqlFunctionCategory.USER_DEFINED_TABLE_FUNCTION,
SqlSyntax.FUNCTION, overloads);
+
+ if (overloads.size() == 0) {
+ throw newValidationError(udtfNode,
+ RESOURCE.exceptTableFunction(function.getName()));
+ }
+ // this is a table function
+ if (overloads.size() == 1 && overloads.get(0)
+ instanceof SqlUserDefinedTableFunction) {
+ //Only one table function allowed in select
+ if (select2TableFunctionInfos.containsKey(select)) {
+ throw newValidationError(udtfNode,
RESOURCE.onlyOneTableFunctionAllowedInSelect());
+ }
+ TableFunctionInfo tableFunctionInfo = new TableFunctionInfo();
+ tableFunctionInfo.node = (SqlBasicCall) udtfNode;
+ tableFunctionInfo.selectIndex = i;
+ tableFunctionInfo.fieldNames = (SqlNodeList) aliasNode;
+ tableFunctionInfo.tableName = "_table_function_" +
nextTableFunctionNameId++;
+
+ select2TableFunctionInfos.put(select, tableFunctionInfo);
+ }
+ }
+ }
+ }
+ // step2. rewrite the select items
+ TableFunctionInfo tableFunctionInfo =
select2TableFunctionInfos.get(select);
+ if (tableFunctionInfo != null) {
+ SqlNodeList newSelectItems = new SqlNodeList(SqlParserPos.ZERO);
+
+ for (int k = 0; k < tableFunctionInfo.selectIndex; k++) {
+ newSelectItems.add(selectItems.get(k));
+ }
+ // add "(f0,f1)" to the select list
+ for (int k = 0; k < tableFunctionInfo.fieldNames.size(); k++) {
+ SqlIdentifier field = new SqlIdentifier(
+ Lists.newArrayList(tableFunctionInfo.tableName,
+ tableFunctionInfo.fieldNames.get(k).toString()),
+ SqlParserPos.ZERO);
+ newSelectItems.add(field);
+ }
+
+ for (int k = tableFunctionInfo.selectIndex + 1; k < selectItems.size();
k++) {
+ newSelectItems.add(selectItems.get(k));
+ }
+ return newSelectItems;
+ }
+ return selectItems;
+ }
+
+ /**
+ * Create LateralTableAs node for table function
+ * @param info information for table function in SqlSelect
+ * @return
+ */
+ private SqlBasicCall createLateralTable(TableFunctionInfo info) {
+ SqlCollectionTableOperator to =
+ new SqlCollectionTableOperator("LATERAL TABLE", SqlModality.RELATION);
+
+ SqlBasicCall tableFunctionNode = info.node;
+ // change the function category to USER_DEFINED_TABLE_FUNCTION
+ if (info.node.getOperator() instanceof SqlUnresolvedFunction) {
+ SqlUnresolvedFunction function = (SqlUnresolvedFunction)
info.node.getOperator();
+ if (!function.getFunctionType().isTableFunction()) {
+ tableFunctionNode = new SqlBasicCall(
+ new SqlUnresolvedFunction(function.getNameAsId(),
+ function.getReturnTypeInference(),
+ function.getOperandTypeInference(),
+ function.getOperandTypeChecker(),
+ function.getParamTypes(),
+ SqlFunctionCategory.USER_DEFINED_TABLE_FUNCTION),
+ info.node.getOperands(),
+ info.node.getParserPosition());
+ }
+ }
+ SqlBasicCall tableCall = new SqlBasicCall(to,
+ new SqlNode[]{ tableFunctionNode }, SqlParserPos.ZERO);
+ SqlLateralOperator lateralOp = new SqlLateralOperator(SqlKind.LATERAL);
+ SqlBasicCall lateralCall = new SqlBasicCall(lateralOp,
+ new SqlNode[]{ tableCall }, SqlParserPos.ZERO);
+
+ SqlAsOperator asOp = SqlStdOperatorTable.AS;
+ SqlNode[] operands = new SqlNode[2 + info.fieldNames.size()];
+ SqlIdentifier tableName = new SqlIdentifier(info.tableName,
SqlParserPos.ZERO);
+ operands[0] = lateralCall;
+ operands[1] = tableName;
+ for (int i = 0; i < info.fieldNames.size(); i++) {
+ operands[2 + i] = info.fieldNames.get(i);
+ }
+ SqlBasicCall lateralTableAs = new SqlBasicCall(asOp, operands,
SqlParserPos.ZERO);
Review comment:
agreed! Thanks for you review!
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
