[ https://issues.apache.org/jira/browse/DRILL-5260?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15883712#comment-15883712 ]
ASF GitHub Bot commented on DRILL-5260: --------------------------------------- Github user sohami commented on a diff in the pull request: https://github.com/apache/drill/pull/753#discussion_r103052237 --- Diff: exec/java-exec/src/test/java/org/apache/drill/test/ProfileParser.java --- @@ -138,9 +414,208 @@ public long getMetric(int id) { } } - public Map<Integer,OpInfo> getOpInfo( ) { + /** + * Information about an operator definition: the plan-time information + * that appears in the plan portion of the profile. Also holds the + * "actuals" from the minor fragment portion of the profile. + * Allows integrating the "planned" vs. "actual" performance of the + * query. + */ + + public static class OpDefInfo { + public String opName; + public boolean isInferred; + public int majorId; + public int stepId; + public String args; + public List<FieldDef> columns; + public int globalLevel; + public int localLevel; + public int id; + public int branchId; + public boolean isBranchRoot; + public double estMemoryCost; + public double estNetCost; + public double estIOCost; + public double estCpuCost; + public double estRowCost; + public double estRows; + public String name; + public long actualMemory; + public int actualBatches; + public long actualRows; + public OpDefInfo inferredParent; + public List<OperatorProfile> opExecs = new ArrayList<>( ); + public List<OpDefInfo> children = new ArrayList<>( ); + + // 00-00 Screen : rowType = RecordType(VARCHAR(10) Year, VARCHAR(65536) Month, VARCHAR(100) Devices, VARCHAR(100) Tier, VARCHAR(100) LOB, CHAR(10) Gateway, BIGINT Day, BIGINT Hour, INTEGER Week, VARCHAR(100) Week_end_date, BIGINT Usage_Cnt): \ + // rowcount = 100.0, cumulative cost = {7.42124276972414E9 rows, 7.663067406383167E10 cpu, 0.0 io, 2.24645048816E10 network, 2.692766612982188E8 memory}, id = 129302 + // + // 00-01 Project(Year=[$0], Month=[$1], Devices=[$2], Tier=[$3], LOB=[$4], Gateway=[$5], Day=[$6], Hour=[$7], Week=[$8], Week_end_date=[$9], Usage_Cnt=[$10]) : + // rowType = RecordType(VARCHAR(10) Year, VARCHAR(65536) Month, VARCHAR(100) Devices, VARCHAR(100) Tier, VARCHAR(100) LOB, CHAR(10) Gateway, BIGINT Day, BIGINT Hour, INTEGER Week, VARCHAR(100) Week_end_date, BIGINT Usage_Cnt): rowcount = 100.0, cumulative cost = {7.42124275972414E9 rows, 7.663067405383167E10 cpu, 0.0 io, 2.24645048816E10 network, 2.692766612982188E8 memory}, id = 129301 + + public OpDefInfo(String plan) { + Pattern p = Pattern.compile( "^(\\d+)-(\\d+)(\\s+)(\\w+)(?:\\((.*)\\))?\\s*:\\s*(.*)$" ); + Matcher m = p.matcher(plan); + if (!m.matches()) { + throw new IllegalStateException( "Could not parse plan: " + plan ); + } + majorId = Integer.parseInt(m.group(1)); + stepId = Integer.parseInt(m.group(2)); + name = m.group(4); + args = m.group(5); + String tail = m.group(6); + String indent = m.group(3); + globalLevel = (indent.length() - 4) / 2; + + p = Pattern.compile("rowType = RecordType\\((.*)\\): (rowcount .*)"); + m = p.matcher(tail); + if ( m.matches() ) { + columns = parseCols(m.group(1)); + tail = m.group(2); + } + + p = Pattern.compile( "rowcount = ([\\d.E]+), cumulative cost = \\{([\\d.E]+) rows, ([\\d.E]+) cpu, ([\\d.E]+) io, ([\\d.E]+) network, ([\\d.E]+) memory\\}, id = (\\d+)"); + m = p.matcher(tail); + if (! m.matches()) { + throw new IllegalStateException("Could not parse costs: " + tail ); + } + estRows = Double.parseDouble(m.group(1)); + estRowCost = Double.parseDouble(m.group(2)); + estCpuCost = Double.parseDouble(m.group(3)); + estIOCost = Double.parseDouble(m.group(4)); + estNetCost = Double.parseDouble(m.group(5)); + estMemoryCost = Double.parseDouble(m.group(6)); + id = Integer.parseInt(m.group(7)); + } + + public void printTree(String indent) { + new TreePrinter().visit(this); + } + + public OpDefInfo(int major, int id) { + majorId = major; + stepId = id; + } + + @Override + public String toString() { + String head = "[OpDefInfo " + majorId + "-" + stepId + ": " + name; + if ( isInferred ) { + head += " (" + opName + ")"; + } + return head + "]"; + } + } + + /** + * Visit a tree of operator definitions to support printing, + * analysis and other tasks. + */ + + public static class TreeVisitor + { + public void visit(OpDefInfo root) { + visit(root, 0); + } + public void visit(OpDefInfo node, int indent) { + visitOp( node, indent ); + if (node.children.isEmpty()) { + return; + } + if ( node.children.size() == 1) { + visit(node.children.get(0), indent); + return; + } + indent++; + int i = 0; + for (OpDefInfo child : node.children) { + visitSubtree(node, i++, indent); + visit(child, indent+1); + } + } + + protected void visitOp(OpDefInfo node, int indent) { + } + + protected void visitSubtree(OpDefInfo node, int i, int indent) { + } + + public String indentString(int indent, String pad) { + StringBuilder buf = new StringBuilder(); + for (int i = 0; i < indent; i++) { + buf.append( pad ); + } + return buf.toString(); + } + + public String indentString(int indent) { + return indentString(indent, " "); + } + + public String subtreeLabel(OpDefInfo node, int branch) { + if (node.name.equals("HashJoin")) { + return (branch == 0) ? "Probe" : "Build"; + } else { + return "Input " + (branch + 1); + } + } + } + + /** + * Print the operator tree for analysis. + */ + + public static class TreePrinter extends TreeVisitor + { + @Override + protected void visitOp(OpDefInfo node, int indent) { + System.out.print( indentString(indent) ); + System.out.println( node.toString() ); + } + + @Override + protected void visitSubtree(OpDefInfo node, int i, int indent) { + System.out.print( indentString(indent) ); + System.out.println(subtreeLabel(node, i)); + } + } + + /** + * Print out the tree showing a comparison of estimated vs. + * actual costs. + */ + + public static class CostPrinter extends TreeVisitor + { + @Override + protected void visitOp(OpDefInfo node, int indentLevel) { + System.out.print(String.format("%02d-%02d ", node.majorId, node.stepId)); + String indent = indentString(indentLevel, ". "); + System.out.print( indent + node.name ); + if (node.opName != null) { + System.out.print( " (" + node.opName + ")" ); + } + System.out.println( ); + indent = indentString(15); + System.out.print( indent ); + System.out.println(String.format(" Estimate: %,15.0f rows, %,7.0f MB", + node.estRows, node.estMemoryCost / 1024 / 1024) ); + System.out.print( indent ); + System.out.println(String.format(" Actual: %,15d rows, %,7d MB", + node.actualRows, node.actualMemory / 1024 / 1024)); + } + + @Override + protected void visitSubtree(OpDefInfo node, int i, int indent) { + System.out.print( indentString(indent) + " " ); + System.out.println(subtreeLabel(node, i)); + } + } + + public Map<Integer,OperatorProfile> getOpInfo( ) { Map<Integer,String> ops = getOperators( ); --- End diff -- Thanks! > Refinements to new "Cluster Fixture" test framework > --------------------------------------------------- > > Key: DRILL-5260 > URL: https://issues.apache.org/jira/browse/DRILL-5260 > Project: Apache Drill > Issue Type: Improvement > Affects Versions: 1.10 > Reporter: Paul Rogers > Assignee: Paul Rogers > Priority: Minor > Fix For: 1.10 > > > Roll-up of a number of enhancements to the cluster fixture framework. > * Config option to suppress printing of CSV and other output. (Allows > printing for single tests, not printing when running from Maven.) > * Parsing of query profiles to extract plan and run time information. > * Fix bug in log fixture when enabling logging for a package. > * Improved ZK support. > * Set up the new CTTAS default temporary workspace for tests. > * Revise TestDrillbitResiliance to use the new framework. > * Revise TestWindowFrame to to use the new framework. > * Revise TestMergeJoinWithSchemaChanges to use the new framework. -- This message was sent by Atlassian JIRA (v6.3.15#6346)