This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit b9a61b04b622ff4be0fa87091d3c879148d9b1ed
Author: Kunal Khatua <kkha...@maprtech.com>
AuthorDate: Thu Aug 22 10:00:58 2019 -0700
DRILL-7222: Visualize estimated and actual row counts for a query
With statistics in place, it is useful to have the estimated rowcount along
side the actual rowcount query profile's operator overview. A toggle button
allows this with the estimated rows hidden by default
We can extract this from the Physical Plan section of the profile.
Added a toggle-ready table-column header
closes #1779
---
.../java/org/apache/drill/exec/ExecConstants.java | 2 +
.../exec/server/rest/profile/HtmlAttribute.java | 2 +
.../exec/server/rest/profile/OperatorWrapper.java | 8 ++-
.../exec/server/rest/profile/ProfileWrapper.java | 6 ++
.../java-exec/src/main/resources/drill-module.conf | 1 +
.../src/main/resources/rest/profile/profile.ftl | 64 +++++++++++++++++++---
6 files changed, 74 insertions(+), 9 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 549d374..463b0a0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -875,6 +875,8 @@ public final class ExecConstants {
public static final BooleanValidator DYNAMIC_UDF_SUPPORT_ENABLED_VALIDATOR =
new BooleanValidator(DYNAMIC_UDF_SUPPORT_ENABLED,
new OptionDescription("Enables users to dynamically upload UDFs. Users
must upload their UDF (source and binary) JAR files to a staging directory in
the distributed file system before issuing the CREATE FUNCTION USING JAR
command to register a UDF. Default is true. (Drill 1.9+)"));
+ //Display estimated rows in operator overview by default
+ public static final String PROFILE_STATISTICS_ESTIMATED_ROWS_SHOW =
"drill.exec.http.profile.statistics.estimated_rows.show";
//Trigger warning in UX if fragments appear to be doing no work (units are
in seconds).
public static final String PROFILE_WARNING_PROGRESS_THRESHOLD =
"drill.exec.http.profile.warning.progress.threshold";
//Trigger warning in UX if slowest fragment operator crosses min threshold
and exceeds ratio with average (units are in seconds).
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/HtmlAttribute.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/HtmlAttribute.java
index 75db298..95c0507 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/HtmlAttribute.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/HtmlAttribute.java
@@ -23,6 +23,7 @@ package org.apache.drill.exec.server.rest.profile;
public class HtmlAttribute {
//Attributes
public static final String CLASS = "class";
+ public static final String KEY = "key";
public static final String DATA_ORDER = "data-order";
public static final String TITLE = "title";
public static final String SPILLS = "spills";
@@ -33,5 +34,6 @@ public class HtmlAttribute {
public static final String CLASS_VALUE_NO_PROGRESS_TAG = "no-progress-tag";
public static final String CLASS_VALUE_TIME_SKEW_TAG = "time-skew-tag";
public static final String CLASS_VALUE_SCAN_WAIT_TAG = "scan-wait-tag";
+ public static final String CLASS_VALUE_EST_ROWS_ANCHOR = "estRowsAnchor";
public static final String STYLE_VALUE_CURSOR_HELP = "cursor:help;";
}
\ No newline at end of file
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
index 0f61170..2e593b6 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
@@ -143,7 +143,7 @@ public class OperatorWrapper {
OverviewTblTxt.AVG_SETUP_TIME, OverviewTblTxt.MAX_SETUP_TIME,
OverviewTblTxt.AVG_PROCESS_TIME, OverviewTblTxt.MAX_PROCESS_TIME,
OverviewTblTxt.MIN_WAIT_TIME, OverviewTblTxt.AVG_WAIT_TIME,
OverviewTblTxt.MAX_WAIT_TIME,
- OverviewTblTxt.PERCENT_FRAGMENT_TIME, OverviewTblTxt.PERCENT_QUERY_TIME,
OverviewTblTxt.ROWS,
+ OverviewTblTxt.PERCENT_FRAGMENT_TIME, OverviewTblTxt.PERCENT_QUERY_TIME,
OverviewTblTxt.ROWS.concat(OverviewTblTxt.ESTIMATED_ROWS),
OverviewTblTxt.AVG_PEAK_MEMORY, OverviewTblTxt.MAX_PEAK_MEMORY
};
@@ -269,7 +269,10 @@ public class OperatorWrapper {
tb.appendPercent(processSum / majorBusyNanos);
tb.appendPercent(processSum / majorFragmentBusyTallyTotal);
- tb.appendFormattedInteger(recordSum);
+ Map<String, String> estRowcountMap = new HashMap<>();
+ estRowcountMap.put(HtmlAttribute.CLASS,
HtmlAttribute.CLASS_VALUE_EST_ROWS_ANCHOR);
+ estRowcountMap.put(HtmlAttribute.KEY, path.replaceAll("-xx-", "-"));
+ tb.appendFormattedInteger(recordSum, estRowcountMap);
final ImmutablePair<OperatorProfile, Integer> peakMem =
Collections.max(opList, Comparators.operatorPeakMemory);
@@ -419,6 +422,7 @@ public class OperatorWrapper {
static final String PERCENT_FRAGMENT_TIME = "% Fragment Time";
static final String PERCENT_QUERY_TIME = "% Query Time";
static final String ROWS = "Rows";
+ static final String ESTIMATED_ROWS = "<div class='estRows'
title='Estimated'>(Estimated)</div>";
static final String AVG_PEAK_MEMORY = "Avg Peak Memory";
static final String MAX_PEAK_MEMORY = "Max Peak Memory";
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java
index e6e6318..5c144eb 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java
@@ -65,6 +65,7 @@ public class ProfileWrapper {
private Map<String, String> physicalOperatorMap;
private final String noProgressWarningThreshold;
private final int defaultAutoLimit;
+ private boolean showEstimatedRows;
public ProfileWrapper(final QueryProfile profile, DrillConfig drillConfig) {
this.profile = profile;
@@ -134,6 +135,7 @@ public class ProfileWrapper {
this.onlyImpersonationEnabled =
WebServer.isOnlyImpersonationEnabled(drillConfig);
this.noProgressWarningThreshold =
String.valueOf(drillConfig.getInt(ExecConstants.PROFILE_WARNING_PROGRESS_THRESHOLD));
+ this.showEstimatedRows =
drillConfig.getBoolean(ExecConstants.PROFILE_STATISTICS_ESTIMATED_ROWS_SHOW);
}
private long tallyMajorFragmentCost(List<MajorFragmentProfile>
majorFragments) {
@@ -390,4 +392,8 @@ public class ProfileWrapper {
physicalOperatorMap.put(operatorPath, extractedOperatorName);
}
}
+
+ public boolean showEstimatedRows() {
+ return showEstimatedRows;
+ }
}
diff --git a/exec/java-exec/src/main/resources/drill-module.conf
b/exec/java-exec/src/main/resources/drill-module.conf
index 422d850..dd062ea 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -143,6 +143,7 @@ drill.exec: {
}
max_profiles: 100,
profiles_per_page: [10, 25, 50, 100],
+ profile.statistics.estimated_rows.show : false,
profile.warning: {
progress.threshold: 300,
time.skew: {
diff --git a/exec/java-exec/src/main/resources/rest/profile/profile.ftl
b/exec/java-exec/src/main/resources/rest/profile/profile.ftl
index 97f03e9..303d83d 100644
--- a/exec/java-exec/src/main/resources/rest/profile/profile.ftl
+++ b/exec/java-exec/src/main/resources/rest/profile/profile.ftl
@@ -42,7 +42,7 @@
};
$(document).ready(function() {
- $(".sortable").DataTable( {
+ $(".sortable").DataTable({
"searching": false,
"lengthChange": false,
"paging": false,
@@ -84,6 +84,29 @@
document.getElementById(warningElemId).style.display="none";
}
+ //Injects Estimated Rows
+ function injectEstimatedRows() {
+ Object.keys(opRowCountMap).forEach(key => {
+ var tgtElem = $("td.estRowsAnchor[key='" + key + "']");
+ var status = tgtElem.append("<div class='estRows' title='Estimated'>("
+ opRowCountMap[key] + ")</div>");
+ });
+ }
+
+ //Toggle Estimates' visibility
+ function toggleEstimates(tgtColumn) {
+ var colClass = '.est' + tgtColumn;
+ var estimates = $(colClass);
+ if (estimates.filter(":visible").length > 0) {
+ $(colClass).each(function () {
+ $(this).attr("style", "display:none");
+ });
+ } else {
+ $(colClass).each(function () {
+ $(this).attr("style", "display:block");
+ });
+ }
+ }
+
//Close the cancellation status popup
function refreshStatus() {
//Close PopUp Modal
@@ -94,7 +117,7 @@
//Cancel query & show cancellation status
function cancelQuery() {
document.getElementById("cancelTitle").innerHTML = "Drillbit on " +
location.hostname + " says";
- $.get("/profiles/cancel/"+globalconfig.queryid, function(data,
status){/*Not Tracking Response*/});
+ $.get("/profiles/cancel/" + globalconfig.queryid, function(data,
status){/*Not Tracking Response*/});
//Show PopUp Modal
$("#queryCancelModal").modal("show");
};
@@ -389,8 +412,17 @@
</div>
<div class="page-header"></div>
- <h3>Operator Profiles</h3>
-
+ <h3>Operator Profiles
+ <button onclick="toggleEstimates('Rows')" class="btn" style="font-size:60%;
float:right">Show/Hide Estimated Rows</button></h3>
+
+ <style>
+ .estRows {
+ color:navy;
+ font-style:italic;
+ font-size: 80%;
+ display:<#if model.showEstimatedRows()>block<#else>none</#if>;
+ }
+</style>
<div class="panel-group" id="operator-accordion">
<div class="panel panel-default">
<div class="panel-heading">
@@ -476,6 +508,9 @@
injectIconByClass("spill-tag","glyphicon-download-alt");
injectIconByClass("time-skew-tag","glyphicon-time");
injectSlowScanIcon();
+ //Building RowCount
+ buildRowCountMap();
+ injectEstimatedRows();
});
//Inject Glyphicon by Class tag
@@ -485,7 +520,7 @@
var i;
for (i = 0; i < tagElemList.length; i++) {
var content = tagElemList[i].innerHTML;
- tagElemList[i].innerHTML = "<span class=\"glyphicon
"+tagIcon+"\"> </span>"+content;
+ tagElemList[i].innerHTML = "<span class=\"glyphicon " + tagIcon +
"\"> </span>" + content;
}
}
@@ -496,7 +531,7 @@
var i;
for (i = 0; i < tagElemList.length; i++) {
var content = tagElemList[i].innerHTML;
- tagElemList[i].innerHTML = "<img src='/static/img/turtle.png'
alt='slow'> "+content;
+ tagElemList[i].innerHTML = "<img src='/static/img/turtle.png'
alt='slow'> " + content;
}
}
@@ -567,7 +602,7 @@
var popUpAndPrintPlan = function() {
var srcSvg = $('#query-visual-canvas');
var screenRatio=0.9;
- let printWindow = window.open('', 'PlanPrint', 'width=' +
(screenRatio*screen.width) + ',height=' + (screenRatio*screen.height) );
+ let printWindow = window.open('', 'PlanPrint', 'width=' +
(screenRatio*screen.width) + ',height=' + (screenRatio*screen.height));
printWindow.document.writeln($(srcSvg).parent().html());
printWindow.print();
};
@@ -587,6 +622,21 @@
if (e.target.form)
<#if
model.isOnlyImpersonationEnabled()>doSubmitQueryWithUserName()<#else>doSubmitQueryWithAutoLimit()</#if>;
});
+
+ // Extract estimated rowcount map
+ var opRowCountMap = {};
+ // Get OpId-Rowocunt Map
+ function buildRowCountMap() {
+ var phyText = $('#query-physical').find('pre').text();
+ var opLines = phyText.split("\n");
+ opLines.forEach(line => {
+ if (line.trim().length > 0) {
+ var opId = line.match(/\d+-\d+/g)[0];
+ var opRowCount = line.match(/rowcount = ([^,]+)/)[1];
+ opRowCountMap[opId] = Number(opRowCount).toLocaleString('en');
+ }
+ });
+ }
</script>
</#macro>
