InvisibleProgrammer commented on code in PR #6043:
URL: https://github.com/apache/hive/pull/6043#discussion_r2353567180
##########
ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java:
##########
@@ -676,6 +689,249 @@ public Object process(Node nd, Stack<Node> stack,
NodeProcessorCtx procCtx,
}
}
+ /**
+ * PTF processor
+ */
+ public static class PTFLineage implements SemanticNodeProcessor {
+
+ @Override
+ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx
procCtx, Object... nodeOutputs) throws SemanticException {
+ // LineageCTx
+ LineageCtx lCtx = (LineageCtx) procCtx;
+
+ // The operators
+ @SuppressWarnings("unchecked")
+ PTFOperator op = (PTFOperator)nd;
+ Operator<? extends OperatorDesc> inpOp = getParent(stack);
+ lCtx.getIndex().copyPredicates(inpOp, op);
+
+ Dependency dep = new Dependency();
+ DependencyType new_type = DependencyType.EXPRESSION;
+ dep.setType(new_type);
+
+ Set<String> columns = new HashSet<>();
+ PartitionedTableFunctionDef funcDef = op.getConf().getFuncDef();
+ StringBuilder sb = new StringBuilder();
+ WindowFrameDef windowFrameDef = null;
+
+ if (!(funcDef.getTFunction() instanceof Noop)) {
+
+ if (funcDef instanceof WindowTableFunctionDef) {
+ // function name
+ WindowFunctionDef windowFunctionDef = ((WindowTableFunctionDef)
funcDef).getWindowFunctions().getFirst();
+ sb.append(windowFunctionDef.getName()).append("(");
+
+ addArgs(sb, columns, lCtx, inpOp, op.getSchema(),
windowFunctionDef.getArgs());
+
+ } else /* PartitionedTableFunctionDef */ {
+ // function name
+ sb.append(funcDef.getName()).append("(");
+ addArgs(sb, columns, lCtx, inpOp,
funcDef.getRawInputShape().getRr().getRowSchema(), funcDef.getArgs());
+ }
+
+ if (funcDef instanceof WindowTableFunctionDef) {
+ WindowFunctionDef windowFunctionDef = ((WindowTableFunctionDef)
funcDef).getWindowFunctions().getFirst();
+ windowFrameDef = windowFunctionDef.getWindowFrame();
+
+ if (sb.charAt(sb.length() - 2) == ',') {
+ sb.delete(sb.length() - 2, sb.length());
+ }
+ sb.append(")");
+ sb.append(" over (");
+ } else {
+ // matchpath has argument pattern like matchpath(<input expression>,
<argument methods: arg1(), arg2()...>)
+ if (funcDef.getInput() != null) {
+ sb.append("on ").append(funcDef.getInput().getAlias()).append(" ");
+
+ int counter = 1;
+ for (PTFExpressionDef arg : funcDef.getArgs()) {
+ ExprNodeDesc exprNode = arg.getExprNode();
+
+ addIfNotNull(columns, exprNode.getCols());
+
+ sb.append("arg").append(counter++).append("(");
+
sb.append(ExprProcFactory.getExprString(funcDef.getRawInputShape().getRr().getRowSchema(),
arg.getExprNode(), lCtx, inpOp, null));
+ sb.append("), ");
+ }
+
+ sb.delete(sb.length() - 2, sb.length());
+ }
+
+ }
+ }
+
+ /*
+ Collect partition by and distribute by information.
+ Please note, at the expression node level, there is no difference
between those.
+ That means distribute by gets a string partition by in the expression
string.
+ */
+ if (funcDef.getPartition() != null ) {
+ List<PTFExpressionDef> partitionExpressions =
funcDef.getPartition().getExpressions();
+
+ boolean isPartitionByAdded = false;
+ for (PTFExpressionDef partitionExpr : partitionExpressions) {
+ ExprNodeDesc partitionExprNode = partitionExpr.getExprNode();
+
+ if (partitionExprNode.getCols() != null &&
!partitionExprNode.getCols().isEmpty()) {
+ if (!isPartitionByAdded) {
+ sb.append("partition by ");
+ isPartitionByAdded = true;
+ }
+
+ addIfNotNull(columns, partitionExprNode.getCols());
+
+ if (partitionExprNode instanceof ExprNodeColumnDesc) {
+
sb.append(ExprProcFactory.getExprString(funcDef.getRawInputShape().getRr().getRowSchema(),
partitionExprNode, lCtx, inpOp, null));
+ sb.append(", ");
+ }
+
+ sb.delete(sb.length() - 2, sb.length());
+ }
+ }
+
+ }
+
+ /*
+ Collects the order by and sort by information.
+ Please note, at the expression node level, there is no difference
between those.
+ That means sort by gets a string partition by in the expression string.
+ */
+ if (funcDef.getOrder() != null) {
+ /*
+ Order by is sometimes added by the compiler to make the PTF call
deterministic.
+ At this point of the code execution, we don't know if it is added by
the compiler or
+ it was originally part of the query string.
+ */
+ List<OrderExpressionDef> orderExpressions =
funcDef.getOrder().getExpressions();
+
+ if (!sb.isEmpty() && sb.charAt(sb.length() - 1) != '(') {
+ sb.append(" ");
+ }
+ sb.append("order by ");
+
+ for (OrderExpressionDef orderExpr : orderExpressions) {
+ ExprNodeDesc orderExprNode = orderExpr.getExprNode();
+ addIfNotNull(columns, orderExprNode.getCols());
+
+
sb.append(ExprProcFactory.getExprString(funcDef.getRawInputShape().getRr().getRowSchema(),
orderExprNode, lCtx, inpOp, null));
+ if (PTFInvocationSpec.Order.DESC.equals(orderExpr.getOrder())) {
+ sb.append(" desc");
+ }
+ sb.append(", ");
+ }
+
+ sb.delete(sb.length() - 2, sb.length());
+ }
+
+ /*
+ Window frame is sometimes added by the compiler to make the PTF call
deterministic.
+ At this point of the code execution, we don't know if it is added by the
compiler or
+ it was originally part of the query string.
+ */
+ if (windowFrameDef != null) {
+ sb.append(" ").append(windowFrameDef.getWindowType()).append(" between
");
+
+ if (windowFrameDef.getStart().isCurrentRow()) {
+ sb.append("current_row");
+ } else {
+ sb.append(windowFrameDef.getStart().isUnbounded() ? "unbounded" :
windowFrameDef.getStart().getAmt() + " preceding");
+ }
+
+ sb.append(" and ");
+
+ if (windowFrameDef.getStart().isCurrentRow()) {
+ sb.append("current_row");
+ } else {
+ sb.append(windowFrameDef.getStart().isUnbounded() ? "unbounded" :
windowFrameDef.getStart().getAmt() + " following");
+ }
Review Comment:
Ok.
##########
ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java:
##########
@@ -676,6 +689,249 @@ public Object process(Node nd, Stack<Node> stack,
NodeProcessorCtx procCtx,
}
}
+ /**
+ * PTF processor
+ */
+ public static class PTFLineage implements SemanticNodeProcessor {
+
+ @Override
+ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx
procCtx, Object... nodeOutputs) throws SemanticException {
+ // LineageCTx
+ LineageCtx lCtx = (LineageCtx) procCtx;
+
+ // The operators
+ @SuppressWarnings("unchecked")
+ PTFOperator op = (PTFOperator)nd;
+ Operator<? extends OperatorDesc> inpOp = getParent(stack);
+ lCtx.getIndex().copyPredicates(inpOp, op);
+
+ Dependency dep = new Dependency();
+ DependencyType new_type = DependencyType.EXPRESSION;
+ dep.setType(new_type);
+
+ Set<String> columns = new HashSet<>();
+ PartitionedTableFunctionDef funcDef = op.getConf().getFuncDef();
+ StringBuilder sb = new StringBuilder();
+ WindowFrameDef windowFrameDef = null;
+
+ if (!(funcDef.getTFunction() instanceof Noop)) {
+
+ if (funcDef instanceof WindowTableFunctionDef) {
+ // function name
+ WindowFunctionDef windowFunctionDef = ((WindowTableFunctionDef)
funcDef).getWindowFunctions().getFirst();
+ sb.append(windowFunctionDef.getName()).append("(");
+
+ addArgs(sb, columns, lCtx, inpOp, op.getSchema(),
windowFunctionDef.getArgs());
+
+ } else /* PartitionedTableFunctionDef */ {
+ // function name
+ sb.append(funcDef.getName()).append("(");
+ addArgs(sb, columns, lCtx, inpOp,
funcDef.getRawInputShape().getRr().getRowSchema(), funcDef.getArgs());
+ }
+
+ if (funcDef instanceof WindowTableFunctionDef) {
+ WindowFunctionDef windowFunctionDef = ((WindowTableFunctionDef)
funcDef).getWindowFunctions().getFirst();
+ windowFrameDef = windowFunctionDef.getWindowFrame();
+
+ if (sb.charAt(sb.length() - 2) == ',') {
+ sb.delete(sb.length() - 2, sb.length());
+ }
+ sb.append(")");
+ sb.append(" over (");
+ } else {
+ // matchpath has argument pattern like matchpath(<input expression>,
<argument methods: arg1(), arg2()...>)
+ if (funcDef.getInput() != null) {
+ sb.append("on ").append(funcDef.getInput().getAlias()).append(" ");
+
+ int counter = 1;
+ for (PTFExpressionDef arg : funcDef.getArgs()) {
+ ExprNodeDesc exprNode = arg.getExprNode();
+
+ addIfNotNull(columns, exprNode.getCols());
+
+ sb.append("arg").append(counter++).append("(");
+
sb.append(ExprProcFactory.getExprString(funcDef.getRawInputShape().getRr().getRowSchema(),
arg.getExprNode(), lCtx, inpOp, null));
+ sb.append("), ");
+ }
+
+ sb.delete(sb.length() - 2, sb.length());
+ }
+
+ }
+ }
+
+ /*
+ Collect partition by and distribute by information.
+ Please note, at the expression node level, there is no difference
between those.
+ That means distribute by gets a string partition by in the expression
string.
+ */
+ if (funcDef.getPartition() != null ) {
+ List<PTFExpressionDef> partitionExpressions =
funcDef.getPartition().getExpressions();
+
+ boolean isPartitionByAdded = false;
+ for (PTFExpressionDef partitionExpr : partitionExpressions) {
+ ExprNodeDesc partitionExprNode = partitionExpr.getExprNode();
+
+ if (partitionExprNode.getCols() != null &&
!partitionExprNode.getCols().isEmpty()) {
+ if (!isPartitionByAdded) {
+ sb.append("partition by ");
+ isPartitionByAdded = true;
+ }
+
+ addIfNotNull(columns, partitionExprNode.getCols());
+
+ if (partitionExprNode instanceof ExprNodeColumnDesc) {
+
sb.append(ExprProcFactory.getExprString(funcDef.getRawInputShape().getRr().getRowSchema(),
partitionExprNode, lCtx, inpOp, null));
+ sb.append(", ");
+ }
+
+ sb.delete(sb.length() - 2, sb.length());
+ }
+ }
+
+ }
+
+ /*
+ Collects the order by and sort by information.
+ Please note, at the expression node level, there is no difference
between those.
+ That means sort by gets a string partition by in the expression string.
+ */
+ if (funcDef.getOrder() != null) {
+ /*
+ Order by is sometimes added by the compiler to make the PTF call
deterministic.
+ At this point of the code execution, we don't know if it is added by
the compiler or
+ it was originally part of the query string.
+ */
+ List<OrderExpressionDef> orderExpressions =
funcDef.getOrder().getExpressions();
+
+ if (!sb.isEmpty() && sb.charAt(sb.length() - 1) != '(') {
+ sb.append(" ");
+ }
+ sb.append("order by ");
+
+ for (OrderExpressionDef orderExpr : orderExpressions) {
+ ExprNodeDesc orderExprNode = orderExpr.getExprNode();
+ addIfNotNull(columns, orderExprNode.getCols());
+
+
sb.append(ExprProcFactory.getExprString(funcDef.getRawInputShape().getRr().getRowSchema(),
orderExprNode, lCtx, inpOp, null));
+ if (PTFInvocationSpec.Order.DESC.equals(orderExpr.getOrder())) {
+ sb.append(" desc");
+ }
+ sb.append(", ");
+ }
+
+ sb.delete(sb.length() - 2, sb.length());
+ }
+
+ /*
+ Window frame is sometimes added by the compiler to make the PTF call
deterministic.
+ At this point of the code execution, we don't know if it is added by the
compiler or
+ it was originally part of the query string.
+ */
+ if (windowFrameDef != null) {
+ sb.append(" ").append(windowFrameDef.getWindowType()).append(" between
");
+
+ if (windowFrameDef.getStart().isCurrentRow()) {
+ sb.append("current_row");
+ } else {
+ sb.append(windowFrameDef.getStart().isUnbounded() ? "unbounded" :
windowFrameDef.getStart().getAmt() + " preceding");
+ }
+
+ sb.append(" and ");
+
+ if (windowFrameDef.getStart().isCurrentRow()) {
+ sb.append("current_row");
+ } else {
+ sb.append(windowFrameDef.getStart().isUnbounded() ? "unbounded" :
windowFrameDef.getStart().getAmt() + " following");
+ }
+ }
+
+ sb.append(")");
+ dep.setExpr(sb.toString());
+
+ LinkedHashSet<BaseColumnInfo> col_set = new LinkedHashSet<>();
Review Comment:
Ok
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
