DRILL-1217: Enhance execution stats web UI Added Gantt chart Improved organization Decrease number of tables and overall page size Implement general javascript functions for profile processing
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/6d5ccc81 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/6d5ccc81 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/6d5ccc81 Branch: refs/heads/master Commit: 6d5ccc81881ad779f276bed03cf8c5246e214e1f Parents: 204aa5d Author: Cliff Buchanan <[email protected]> Authored: Mon Jul 28 10:01:15 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Tue Jul 29 16:49:33 2014 -0700 ---------------------------------------------------------------------- .../drill/exec/server/rest/ProfileWrapper.java | 361 +++++++++-------- .../src/main/resources/rest/profile/profile.ftl | 177 ++++++--- .../src/main/resources/rest/www/graph.js | 383 +++++++++++++++---- .../src/main/resources/rest/www/style.css | 27 -- 4 files changed, 640 insertions(+), 308 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6d5ccc81/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java index 706f9a3..777779f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java @@ -25,23 +25,24 @@ import org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile; import org.apache.drill.exec.proto.UserBitShared.OperatorProfile; import org.apache.drill.exec.proto.UserBitShared.QueryProfile; import org.apache.drill.exec.proto.UserBitShared.StreamProfile; +import org.apache.drill.exec.proto.helper.QueryIdHelper; import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.collect.Collections2; import org.apache.drill.exec.proto.helper.QueryIdHelper; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import java.text.DateFormat; import java.text.NumberFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.Iterator; import java.util.List; import java.util.Locale; -import java.util.TreeMap; +import java.util.Map; public class ProfileWrapper { public QueryProfile profile; @@ -60,78 +61,70 @@ public class ProfileWrapper { return id; } - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - ArrayList<MajorFragmentProfile> majors = new ArrayList<MajorFragmentProfile>(profile.getFragmentProfileList()); + public String getQueryId() { + return QueryIdHelper.getQueryId(profile.getId()); + } + + public List<OperatorWrapper> getOperatorProfiles() { + List<OperatorWrapper> ows = Lists.newArrayList(); + Map<ImmutablePair<Integer, Integer>, List<ImmutablePair<OperatorProfile, Integer>>> opmap = Maps.newHashMap(); + List<MajorFragmentProfile> majors = new ArrayList<>(profile.getFragmentProfileList()); Collections.sort(majors, Comparators.majorIdCompare); - builder.append(queryTimingProfile(majors)); - for (MajorFragmentProfile m : majors) { - builder.append(majorFragmentOperatorProfile(m)); - } - for (MajorFragmentProfile m : majors) { - builder.append(majorFragmentTimingProfile(m)); - } - for (MajorFragmentProfile m : majors) { - Collection<MinorFragmentProfile> minors = Collections2.filter(m.getMinorFragmentProfileList(), Filters.hasOperators); - for (MinorFragmentProfile mi : minors) { - builder.append(minorFragmentOperatorProfile(m.getMajorFragmentId(), mi)); + for (MajorFragmentProfile major : majors) { + + List<MinorFragmentProfile> minors = new ArrayList<>(major.getMinorFragmentProfileList()); + Collections.sort(minors, Comparators.minorIdCompare); + for (MinorFragmentProfile minor : minors) { + + List<OperatorProfile> ops = new ArrayList<>(minor.getOperatorProfileList()); + Collections.sort(ops, Comparators.operatorIdCompare); + for (OperatorProfile op : ops) { + + ImmutablePair<Integer, Integer> ip = new ImmutablePair<>( + major.getMajorFragmentId(), op.getOperatorId()); + if (!opmap.containsKey(ip)) { + List<ImmutablePair<OperatorProfile, Integer>> l = Lists.newArrayList(); + opmap.put(ip, l); + } + opmap.get(ip).add(new ImmutablePair<>(op, minor.getMinorFragmentId())); + } } } - return builder.toString(); + + List<ImmutablePair<Integer, Integer>> keys = new ArrayList<>(opmap.keySet()); + Collections.sort(keys); + for (ImmutablePair<Integer, Integer> ip : keys) { + ows.add(new OperatorWrapper(ip.getLeft(), opmap.get(ip))); + } + + return ows; } - - public String queryTimingProfile(ArrayList<MajorFragmentProfile> majors) { - final String[] columns = {"major id", "fragments reporting", "first start", "last start", "first end", "last end", "tmin", "tavg", "tmax"}; - TableBuilder builder = new TableBuilder("Query Timing Profile", "QueryTimingProfile", columns); - + + public List<FragmentWrapper> getFragmentProfiles() { + List<FragmentWrapper> fws = Lists.newArrayList(); - long t0 = profile.getStart(); - for (MajorFragmentProfile m : majors) { - final String fmt = " (<a href=\"#MinorFragment" + m.getMajorFragmentId() + "_%1$dOperatorProfile\">%1$d</a>)"; - - ArrayList<MinorFragmentProfile> complete = new ArrayList<MinorFragmentProfile>( - Collections2.filter(m.getMinorFragmentProfileList(), Filters.hasOperatorsAndTimes)); - - builder.appendInteger(m.getMajorFragmentId(), null); - builder.appendCell(complete.size() + " / " + m.getMinorFragmentProfileCount(), null); - - if (complete.size() < 1) { - builder.appendRepeated("", null, 7); - continue; - } - - int li = complete.size() - 1; - - Collections.sort(complete, Comparators.startTimeCompare); - builder.appendMillis(complete.get(0).getStartTime() - t0, String.format(fmt, complete.get(0).getMinorFragmentId())); - builder.appendMillis(complete.get(li).getStartTime() - t0, String.format(fmt, complete.get(li).getMinorFragmentId())); + List<MajorFragmentProfile> majors = new ArrayList<>(profile.getFragmentProfileList()); + Collections.sort(majors, Comparators.majorIdCompare); + for (MajorFragmentProfile major : majors) { + fws.add(new FragmentWrapper(major)); + } + + return fws; + } - Collections.sort(complete, Comparators.endTimeCompare); - builder.appendMillis(complete.get(0).getEndTime() - t0, String.format(fmt, complete.get(0).getMinorFragmentId())); - builder.appendMillis(complete.get(li).getEndTime() - t0, String.format(fmt, complete.get(li).getMinorFragmentId())); - - long total = 0; - for (MinorFragmentProfile p : complete) { - total += p.getEndTime() - p.getStartTime(); - } - Collections.sort(complete, Comparators.runTimeCompare); - builder.appendMillis(complete.get(0).getEndTime() - complete.get(0).getStartTime(), - String.format(fmt, complete.get(0).getMinorFragmentId())); - builder.appendMillis((long) (total / complete.size()), null); - builder.appendMillis(complete.get(li).getEndTime() - complete.get(li).getStartTime(), - String.format(fmt, complete.get(li).getMinorFragmentId())); + public String getFragmentsOverview() { + final String[] columns = {"Fragment", "Minor Fragments Reporting", "First Start", "Last Start", "First End", "Last End", "tmin", "tavg", "tmax"}; + TableBuilder tb = new TableBuilder(columns); + for (FragmentWrapper fw : getFragmentProfiles()) { + fw.addSummary(tb); } - return builder.toString(); + return tb.toString(); } public String majorFragmentTimingProfile(MajorFragmentProfile majorFragmentProfile) { - final String[] columns = {"id", "start", "end", "total time", "max records", "max batches"}; - TableBuilder builder = new TableBuilder( - "Major Fragment #" + majorFragmentProfile.getMajorFragmentId() + " Timing Profile", - "MajorFragment" + majorFragmentProfile.getMajorFragmentId() + "TimingProfile", - columns); + final String[] columns = {"Minor", "Start", "End", "Total Time", "Max Records", "Max Batches"}; + TableBuilder builder = new TableBuilder(columns); ArrayList<MinorFragmentProfile> complete, incomplete; complete = new ArrayList<MinorFragmentProfile>(Collections2.filter( @@ -158,14 +151,11 @@ public class ProfileWrapper { biggestBatches = Math.max(biggestBatches, batches); } - builder.appendCell( - majorFragmentProfile.getMajorFragmentId() + "-" - + m.getMinorFragmentId(), null); + builder.appendCell(String.format("#%d - %d", majorFragmentProfile.getMajorFragmentId(), m.getMinorFragmentId()), null); builder.appendMillis(m.getStartTime() - t0, null); builder.appendMillis(m.getEndTime() - t0, null); builder.appendMillis(m.getEndTime() - m.getStartTime(), null); - Collections.sort(ops, Comparators.incomingRecordCompare); builder.appendInteger(biggestIncomingRecords, null); builder.appendInteger(biggestBatches, null); } @@ -177,155 +167,213 @@ public class ProfileWrapper { } return builder.toString(); } + + public String getOperatorsOverview() { + final String [] columns = {"Operator", "Type", "Setup (min)", "Setup (avg)", "Setup (max)", "Process (min)", "Process (avg)", "Process (max)", "Wait (min)", "Wait (avg)", "Wait (max)"}; + TableBuilder tb = new TableBuilder(columns); + for (OperatorWrapper ow : getOperatorProfiles()) { + ow.addSummary(tb); + } + return tb.toString(); + } + + public String getOperatorsJSON() { + StringBuilder sb = new StringBuilder("{"); + String sep = ""; + for (CoreOperatorType op : CoreOperatorType.values()) { + sb.append(String.format("%s\"%d\" : \"%s\"", sep, op.ordinal(), op)); + sep = ", "; + } + return sb.append("}").toString(); + } + + public class FragmentWrapper { + MajorFragmentProfile major; + public FragmentWrapper(MajorFragmentProfile major) { + this.major = major; + } + + public String getDisplayName() { + return "Fragment #" + major.getMajorFragmentId(); + } + + public String getId() { + return "fragment-" + major.getMajorFragmentId(); + } + + public void addSummary(TableBuilder tb) { + final String fmt = " (%d)"; + long t0 = profile.getStart(); + + ArrayList<MinorFragmentProfile> complete = new ArrayList<MinorFragmentProfile>( + Collections2.filter(major.getMinorFragmentProfileList(), Filters.hasOperatorsAndTimes)); - public String majorFragmentOperatorProfile(MajorFragmentProfile major) { - TreeMap<Integer, ArrayList<Pair<OperatorProfile, Integer>>> opmap = - new TreeMap<Integer, ArrayList<Pair<OperatorProfile, Integer>>>(); + tb.appendCell("#" + major.getMajorFragmentId(), null); + tb.appendCell(complete.size() + " / " + major.getMinorFragmentProfileCount(), null); + + if (complete.size() < 1) { + tb.appendRepeated("", null, 7); + return; + } + + int li = complete.size() - 1; + + Collections.sort(complete, Comparators.startTimeCompare); + tb.appendMillis(complete.get(0).getStartTime() - t0, String.format(fmt, complete.get(0).getMinorFragmentId())); + tb.appendMillis(complete.get(li).getStartTime() - t0, String.format(fmt, complete.get(li).getMinorFragmentId())); + Collections.sort(complete, Comparators.endTimeCompare); + tb.appendMillis(complete.get(0).getEndTime() - t0, String.format(fmt, complete.get(0).getMinorFragmentId())); + tb.appendMillis(complete.get(li).getEndTime() - t0, String.format(fmt, complete.get(li).getMinorFragmentId())); + + long total = 0; + for (MinorFragmentProfile p : complete) { + total += p.getEndTime() - p.getStartTime(); + } + Collections.sort(complete, Comparators.runTimeCompare); + tb.appendMillis(complete.get(0).getEndTime() - complete.get(0).getStartTime(), + String.format(fmt, complete.get(0).getMinorFragmentId())); + tb.appendMillis((long) (total / complete.size()), null); + tb.appendMillis(complete.get(li).getEndTime() - complete.get(li).getStartTime(), + String.format(fmt, complete.get(li).getMinorFragmentId())); + } + public String getContent() { + return majorFragmentTimingProfile(major); + } + } + + public class OperatorWrapper { + int major; + List<ImmutablePair<OperatorProfile, Integer>> ops; - final String [] columns = {"id", "type", "setup min", "setup avg", "setup max", "process min", "process avg", "process max", "wait min", "wait avg", "wait max"}; - TableBuilder builder = new TableBuilder( - String.format("Major Fragment #%d Operator Profile", major.getMajorFragmentId()), - String.format("MajorFragment%dOperatorProfile", major.getMajorFragmentId()), - columns); + public OperatorWrapper(int major, List<ImmutablePair<OperatorProfile, Integer>> ops) { + assert ops.size() > 0; + this.major = major; + this.ops = ops; + } + + public String getDisplayName() { + return String.format("Fragment #%d - Operator %d (%s)", + major, ops.get(0).getLeft().getOperatorId(), + CoreOperatorType.valueOf(ops.get(0).getLeft().getOperatorType() ).toString()); + } + public String getId() { + return String.format("operator-%d-%d", major, ops.get(0).getLeft().getOperatorId()); + } - for (MinorFragmentProfile m : major.getMinorFragmentProfileList()) { - int mid = m.getMinorFragmentId(); + public String getContent() { + final String [] columns = {"Fragment", "Setup", "Process", "Wait", "Max Batches", "Max Records"}; + TableBuilder builder = new TableBuilder(columns); - for (OperatorProfile op : m.getOperatorProfileList()) { - int opid = op.getOperatorId(); + for (ImmutablePair<OperatorProfile, Integer> ip : ops) { + int minor = ip.getRight(); + OperatorProfile op = ip.getLeft(); - if (!opmap.containsKey(opid)) { - opmap.put(opid, new ArrayList<Pair<OperatorProfile, Integer>>()); + builder.appendCell(String.format("#%d - %d", major, minor), null); + builder.appendNanos(op.getSetupNanos(), null); + builder.appendNanos(op.getProcessNanos(), null); + builder.appendNanos(op.getWaitNanos(), null); + + long maxBatches = Long.MIN_VALUE; + long maxRecords = Long.MIN_VALUE; + for (StreamProfile sp : op.getInputProfileList()) { + maxBatches = Math.max(sp.getBatches(), maxBatches); + maxRecords = Math.max(sp.getRecords(), maxRecords); } - opmap.get(opid).add(new ImmutablePair<OperatorProfile, Integer>(op, mid)); + + builder.appendInteger(maxBatches, null); + builder.appendInteger(maxRecords, null); } + return builder.toString(); } - for (Integer opid : opmap.keySet()) { - ArrayList<Pair<OperatorProfile, Integer>> oplist = opmap.get(opid); - final String fmt = " (<a href=\"#MinorFragment" + major.getMajorFragmentId() + "_%1$dOperatorProfile\">%1$d</a>)"; - int li = oplist.size() - 1; - double totalsetup = 0; - double totalprocess = 0; - double totalwait = 0; + public void addSummary(TableBuilder tb) { + tb.appendCell(String.format("#%d - Op %d", major, ops.get(0).getLeft().getOperatorId()), null); + tb.appendCell(CoreOperatorType.valueOf(ops.get(0).getLeft().getOperatorType() ).toString(), null); - for (Pair<OperatorProfile, Integer> opint : oplist) { - totalsetup += opint.getLeft().getSetupNanos(); - totalprocess += opint.getLeft().getProcessNanos(); - totalwait += opint.getLeft().getWaitNanos(); + int li = ops.size() - 1; + String fmt = " (%s)"; + + double setupSum = 0.0; + double processSum = 0.0; + double waitSum = 0.0; + for (ImmutablePair<OperatorProfile, Integer> ip : ops) { + setupSum += ip.getLeft().getSetupNanos(); + processSum += ip.getLeft().getProcessNanos(); + waitSum += ip.getLeft().getWaitNanos(); } - builder.appendInteger(oplist.get(0).getLeft().getOperatorId(), null); - builder.appendCell(CoreOperatorType.valueOf(oplist.get(0).getLeft().getOperatorType()).toString(), null); + Collections.sort(ops, Comparators.setupTimeSort); + tb.appendNanos(ops.get(0).getLeft().getSetupNanos(), String.format(fmt, ops.get(0).getRight())); + tb.appendNanos((long) (setupSum / ops.size()), null); + tb.appendNanos(ops.get(li).getLeft().getSetupNanos(), String.format(fmt, ops.get(li).getRight())); - Collections.sort(oplist, Comparators.setupTimeSort); - builder.appendNanos(oplist.get(0).getLeft().getSetupNanos(), String.format(fmt, oplist.get(0).getRight())); - builder.appendNanos((long) (totalsetup / oplist.size()), null); - builder.appendNanos(oplist.get(li).getLeft().getSetupNanos(), String.format(fmt, oplist.get(li).getRight())); - - Collections.sort(opmap.get(opid), Comparators.processTimeSort); - builder.appendNanos(oplist.get(0).getLeft().getProcessNanos(), String.format(fmt, oplist.get(0).getRight())); - builder.appendNanos((long) (totalprocess / oplist.size()), null); - builder.appendNanos(oplist.get(li).getLeft().getProcessNanos(), String.format(fmt, oplist.get(li).getRight())); + Collections.sort(ops, Comparators.processTimeSort); + tb.appendNanos(ops.get(0).getLeft().getProcessNanos(), String.format(fmt, ops.get(0).getRight())); + tb.appendNanos((long) (processSum / ops.size()), null); + tb.appendNanos(ops.get(li).getLeft().getProcessNanos(), String.format(fmt, ops.get(li).getRight())); - Collections.sort(opmap.get(opid), Comparators.waitTimeSort); - builder.appendNanos(oplist.get(0).getLeft().getWaitNanos(), String.format(fmt, oplist.get(0).getRight())); - builder.appendNanos((long) (totalwait / oplist.size()), null); - builder.appendNanos(oplist.get(li).getLeft().getWaitNanos(), String.format(fmt, oplist.get(li).getRight())); + Collections.sort(ops, Comparators.waitTimeSort); + tb.appendNanos(ops.get(0).getLeft().getWaitNanos(), String.format(fmt, ops.get(0).getRight())); + tb.appendNanos((long) (waitSum / ops.size()), null); + tb.appendNanos(ops.get(li).getLeft().getWaitNanos(), String.format(fmt, ops.get(li).getRight())); } - return builder.toString(); } - - public String minorFragmentOperatorProfile(int majorId, MinorFragmentProfile minorFragmentProfile) { - ArrayList<OperatorProfile> oplist = new ArrayList<OperatorProfile>(minorFragmentProfile.getOperatorProfileList()); - - final String[] columns = {"id", "type", "setup", "process", "wait"}; - TableBuilder builder = new TableBuilder( - String.format("Minor Fragment #%d-%d Operator Profile", majorId, minorFragmentProfile.getMinorFragmentId()), - String.format("MinorFragment%d_%dOperatorProfile", majorId, minorFragmentProfile.getMinorFragmentId()), - columns); - Collections.sort(oplist, Comparators.operatorIdCompare); - for (OperatorProfile op : oplist) { - builder.appendInteger(op.getOperatorId(), null); - builder.appendCell(CoreOperatorType.valueOf(op.getOperatorType()).toString(), null); - builder.appendNanos(op.getSetupNanos(), null); - builder.appendNanos(op.getProcessNanos(), null); - builder.appendNanos(op.getWaitNanos(), null); - } - - return builder.toString(); - } - - private static class Comparators { + static class Comparators { final static Comparator<MajorFragmentProfile> majorIdCompare = new Comparator<MajorFragmentProfile>() { public int compare(MajorFragmentProfile o1, MajorFragmentProfile o2) { - return o1.getMajorFragmentId() < o2.getMajorFragmentId() ? -1 : 1; + return Long.compare(o1.getMajorFragmentId(), o2.getMajorFragmentId()); } }; final static Comparator<MinorFragmentProfile> minorIdCompare = new Comparator<MinorFragmentProfile>() { public int compare(MinorFragmentProfile o1, MinorFragmentProfile o2) { - return o1.getMinorFragmentId() < o2.getMinorFragmentId() ? -1 : 1; + return Long.compare(o1.getMinorFragmentId(), o2.getMinorFragmentId()); } }; final static Comparator<MinorFragmentProfile> startTimeCompare = new Comparator<MinorFragmentProfile>() { public int compare(MinorFragmentProfile o1, MinorFragmentProfile o2) { - return o1.getStartTime() < o2.getStartTime() ? -1 : 1; + return Long.compare(o1.getStartTime(), o2.getStartTime()); } }; final static Comparator<MinorFragmentProfile> endTimeCompare = new Comparator<MinorFragmentProfile>() { public int compare(MinorFragmentProfile o1, MinorFragmentProfile o2) { - return o1.getEndTime() < o2.getEndTime() ? -1 : 1; + return Long.compare(o1.getEndTime(), o2.getEndTime()); } }; final static Comparator<MinorFragmentProfile> runTimeCompare = new Comparator<MinorFragmentProfile>() { public int compare(MinorFragmentProfile o1, MinorFragmentProfile o2) { - return o1.getEndTime() - o1.getStartTime() < o2.getEndTime() - o2.getStartTime() ? -1 : 1; + return Long.compare(o1.getEndTime() - o1.getStartTime(), o2.getEndTime() - o2.getStartTime()); } }; final static Comparator<OperatorProfile> operatorIdCompare = new Comparator<OperatorProfile>() { public int compare(OperatorProfile o1, OperatorProfile o2) { - return o1.getOperatorId() < o2.getOperatorId() ? -1 : 1; - } - }; - - final static Comparator<OperatorProfile> incomingRecordCompare = new Comparator<OperatorProfile>() { - public long incomingRecordCount(OperatorProfile op) { - long count = 0; - for (StreamProfile sp : op.getInputProfileList()) { - count += sp.getRecords(); - } - return count; - } - - public int compare(OperatorProfile o1, OperatorProfile o2) { - return incomingRecordCount(o1) > incomingRecordCount(o2) ? -1 : 1; + return Long.compare(o1.getOperatorId(), o2.getOperatorId()); } }; final static Comparator<Pair<OperatorProfile, Integer>> setupTimeSort = new Comparator<Pair<OperatorProfile, Integer>>() { public int compare(Pair<OperatorProfile, Integer> o1, Pair<OperatorProfile, Integer> o2) { - return o1.getLeft().getSetupNanos() < o2.getLeft().getSetupNanos() ? -1 : 1; + return Long.compare(o1.getLeft().getSetupNanos(), o2.getLeft().getSetupNanos()); } }; final static Comparator<Pair<OperatorProfile, Integer>> processTimeSort = new Comparator<Pair<OperatorProfile, Integer>>() { public int compare(Pair<OperatorProfile, Integer> o1, Pair<OperatorProfile, Integer> o2) { - return o1.getLeft().getProcessNanos() < o2.getLeft().getProcessNanos() ? -1 : 1; + return Long.compare(o1.getLeft().getProcessNanos(), o2.getLeft().getProcessNanos()); } }; final static Comparator<Pair<OperatorProfile, Integer>> waitTimeSort = new Comparator<Pair<OperatorProfile, Integer>>() { public int compare(Pair<OperatorProfile, Integer> o1, Pair<OperatorProfile, Integer> o2) { - return o1.getLeft().getWaitNanos() < o2.getLeft().getWaitNanos() ? -1 : 1; + return Long.compare(o1.getLeft().getWaitNanos(), o2.getLeft().getWaitNanos()); } }; } @@ -356,14 +404,13 @@ public class ProfileWrapper { int w = 0; int width; - public TableBuilder(String title, String id, String[] columns) { + public TableBuilder(String[] columns) { sb = new StringBuilder(); width = columns.length; format.setMaximumFractionDigits(3); format.setMinimumFractionDigits(3); - sb.append(String.format("<h3 id=\"%s\">%s</h3>\n", id, title)); sb.append("<table class=\"table table-bordered text-right\">\n<tr>"); for (String cn : columns) { sb.append("<th>" + cn + "</th>"); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6d5ccc81/exec/java-exec/src/main/resources/rest/profile/profile.ftl ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/rest/profile/profile.ftl b/exec/java-exec/src/main/resources/rest/profile/profile.ftl index ffe7e46..3cd214c 100644 --- a/exec/java-exec/src/main/resources/rest/profile/profile.ftl +++ b/exec/java-exec/src/main/resources/rest/profile/profile.ftl @@ -11,68 +11,153 @@ <#include "*/generic.ftl"> <#macro page_head> -<link href="/www/style.css" rel="stylesheet"> - -<script src="http://d3js.org/d3.v3.min.js";></script> +<script src="http://d3js.org/d3.v3.js";></script> <script src="http://cpettitt.github.io/project/dagre-d3/latest/dagre-d3.js";></script> <script src="/www/graph.js"></script> +<script> + var globalconfig = { + "queryid" : "${model.getQueryId()}", + "operators" : ${model.getOperatorsJSON()} + }; +</script> </#macro> <#macro page_body> <a href="/queries">back</a><br/> <div class="page-header"> </div> - <h3>Query</h3> - <form role="form" action="/query" method="POST"> - <div class="form-group"> - <textarea class="form-control" id="query" name="query" style="font-family: Courier;">${model.getProfile().query}</textarea> + <h3>Query and Planning</h3> + <ul id="query-tabs" class="nav nav-tabs" role="tablist"> + <li><a href="#query-query" role="tab" data-toggle="tab">Query</a></li> + <li><a href="#query-physical" role="tab" data-toggle="tab">Physical Plan</a></li> + <li><a href="#query-visual" role="tab" data-toggle="tab">Visualized Plan</a></li> + <li><a href="#query-edit" role="tab" data-toggle="tab">Edit Query</a></li> + </ul> + <div id="query-content" class="tab-content"> + <div id="query-query" class="tab-pane"> + <p><pre>${model.getProfile().query}</pre></p> + </div> + <div id="query-physical" class="tab-pane"> + <p><pre>${model.profile.plan}</pre></p> + </div> + <div id="query-visual" class="tab-pane"> + <svg id="query-visual-canvas" class="center-block"></svg> </div> - <div class="form-group"> - <div class="radio-inline"> - <label> - <input type="radio" name="queryType" id="sql" value="SQL" checked> - SQL - </label> + <div id="query-edit" class="tab-pane"> + <form role="form" action="/query" method="POST"> + <div class="form-group"> + <textarea class="form-control" id="query" name="query" style="font-family: Courier;">${model.getProfile().query}</textarea> + </div> + <div class="form-group"> + <div class="radio-inline"> + <label> + <input type="radio" name="queryType" id="sql" value="SQL" checked> + SQL + </label> + </div> + <div class="radio-inline"> + <label> + <input type="radio" name="queryType" id="physical" value="PHYSICAL"> + PHYSICAL + </label> + </div> + <div class="radio-inline"> + <label> + <input type="radio" name="queryType" id="logical" value="LOGICAL"> + LOGICAL + </label> + </div> + </div> + <button type="submit" class="btn btn-default">Re-run query</button> + </form> + <form action="/profiles/cancel/${model.id}" method="GET"> + <button type="link" class="btn btn-default">Cancel query</button> + </form> + </div> + </div> + + <div class="page-header"></div> + <h3>Fragment Profiles</h3> + + <div class="panel-group" id="fragment-accordion"> + <div class="panel panel-default"> + <div class="panel-heading"> + <h4 class="panel-title"> + <a data-toggle="collapse" href="#fragment-overview"> + Overview + </a> + </h4> </div> - <div class="radio-inline"> - <label> - <input type="radio" name="queryType" id="physical" value="PHYSICAL"> - PHYSICAL - </label> + <div id="fragment-overview" class="panel-collapse collapse"> + <div class="panel-body"> + <svg id="fragment-overview-canvas" class="center-block"></svg> + ${model.getFragmentsOverview()} + </div> </div> - <div class="radio-inline"> - <label> - <input type="radio" name="queryType" id="logical" value="LOGICAL"> - LOGICAL - </label> + </div> + <#list model.getFragmentProfiles() as frag> + <div class="panel panel-default"> + <div class="panel-heading"> + <h4 class="panel-title"> + <a data-toggle="collapse" href="#${frag.getId()}"> + ${frag.getDisplayName()} + </a> + </h4> + </div> + <div id="${frag.getId()}" class="panel-collapse collapse"> + <div class="panel-body"> + ${frag.getContent()} + </div> </div> </div> - <button type="submit" class="btn btn-default">Re-run query</button> - </form> - <form action="/profiles/cancel/${model.id}" method="GET"> - <button type="link" class="btn btn-default">Cancel query</button> - </form> - <div class="page-header"> - </div> - <h3>Visualized Plan</h3> - <button id="renderbutton" class="btn btn-default">Generate</button> - <svg id="svg-canvas" style="margin: auto; display: block;"> - <g transform="translate(20, 20)"/> - </svg> - <div class="page-header"> + </#list> </div> - <h3>Physical Plan</h3> - <p><pre>${model.profile.plan}</pre></p> - <div class="page-header"> - </div> - <h3>Profile Summary</h3> - <p>${model.toString()}</p> - <div class="page-header"> + + <div class="page-header"></div> + <h3>Operator Profiles</h3> + + <div class="panel-group" id="operator-accordion"> + <div class="panel panel-default"> + <div class="panel-heading"> + <h4 class="panel-title"> + <a data-toggle="collapse" href="#operator-overview"> + Overview + </a> + </h4> + </div> + <div id="operator-overview" class="panel-collapse collapse"> + <div class="panel-body"> + ${model.getOperatorsOverview()} + </div> + </div> + </div> + + <#list model.getOperatorProfiles() as op> + <div class="panel panel-default"> + <div class="panel-heading"> + <h4 class="panel-title"> + <a data-toggle="collapse" href="#${op.getId()}"> + ${op.getDisplayName()} + </a> + </h4> + </div> + <div id="${op.getId()}" class="panel-collapse collapse"> + <div class="panel-body"> + ${op.getContent()} + </div> + </div> + </div> + </#list> </div> - <div class="span4 collapse-group"> - <a class="btn btn-default" data-toggle="collapse" data-target="#viewdetails">View complete profile</a> + + <div class="page-header"></div> + <h3>Full JSON Profile</h3> + + <div class="span4 collapse-group" id="full-json-profile"> + <a class="btn btn-default" data-toggle="collapse" data-target="#full-json-profile-json">JSON profile</a> <br> <br> - <pre class="collapse" id="viewdetails">${model.profile.toString()}</pre> + <pre class="collapse" id="full-json-profile-json"> + </pre> </div> <div class="page-header"> </div> <br> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6d5ccc81/exec/java-exec/src/main/resources/rest/www/graph.js ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/rest/www/graph.js b/exec/java-exec/src/main/resources/rest/www/graph.js index 0173da5..76f39fa 100644 --- a/exec/java-exec/src/main/resources/rest/www/graph.js +++ b/exec/java-exec/src/main/resources/rest/www/graph.js @@ -1,82 +1,309 @@ /* - Licensed to the Apache Software Foundation (ASF) under one or more contributor - license agreements. See the NOTICE file distributed with this work for additional - information regarding copyright ownership. The ASF licenses this file to - You under the Apache License, Version 2.0 (the "License"); you may not use - this file except in compliance with the License. You may obtain a copy of - the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required - by applicable law or agreed to in writing, software distributed under the - License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS - OF ANY KIND, either express or implied. See the License for the specific - language governing permissions and limitations under the License. */ - + * Licensed to the Apache Software Foundation (ASF) under one or more contributor + * license agreements. See the NOTICE file distributed with this work for additional + * information regarding copyright ownership. The ASF licenses this file to + * You under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of + * the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + * by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + * OF ANY KIND, either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ $(window).load(function () { - document.getElementById("renderbutton").addEventListener("click", function () { - this.remove(); - var queryid = window.location.href.split("/").splice(-1); - $.ajax({ - type: "GET", - dataType: "json", - url: "/profiles/" + queryid + ".json", - success: function (profile) { - var colors = ["red", "green", "blue", "orange", "yellow", "pink", "purple", "gray"]; - - var plan = $.map(profile.plan.trim().split("\n"), function (s) { - return [/^([0-9-]+)( *)([a-zA-Z]*)/.exec(s).slice(1)]; - }); - - // nodes - var g = new dagreD3.Digraph(); - for (var i = 0; i < plan.length; i++) { - g.addNode(plan[i][0], { - label: plan[i][2], - fragment: parseInt(plan[i][0].split("-")[0]) - }); - } - - // edges - var st = [plan[0]]; - for (var i = 1; i < plan.length; i++) { - var top = st.pop(); - while (top[1].length >= plan[i][1].length) - top = st.pop(); - - g.addEdge(null, plan[i][0], top[0]); - - if (plan[i][1].length != top[1].length) - st.push(top); - if (plan[i][1].length >= top[1].length) - st.push(plan[i]); - } - - // rendering - var renderer = new dagreD3.Renderer(); - renderer.zoom(function () {return function (graph, root) {}}); - var oldDrawNodes = renderer.drawNodes(); - renderer.drawNodes(function(graph, root) { - var svgNodes = oldDrawNodes(graph, root); - svgNodes.each(function(u) { - d3.select(this).style("fill", colors[graph.node(u).fragment % colors.length]); - }); - return svgNodes; - }); - - // page placement - var layout = dagreD3.layout() - .nodeSep(20);//.rankDir("LR"); - var layout = renderer.layout(layout).run(g, d3.select("svg g")); - - //var layout = renderer.run(g, d3.select("svg g")); - d3.select("svg") - .attr("width", layout.graph().width + 40) - .attr("height", layout.graph().height + 40); - }, - error: function (x, y, z) { - console.log(x); - console.log(y); - console.log(z); - } - }); - }, false); + // for each record, unroll the array pointed to by "fieldpath" into a new + // record for each element of the array + function unnest (table, fieldpath, dest) { + var faccess = accessor(fieldpath); + return $.map(table, function (record, index) { + var ra = []; + var nested = faccess(record); + for (var i = 0; i < nested.length; i++) { + var newrec = $.extend({}, record); + newrec[dest] = nested[i]; + ra.push(newrec); + } + return ra; + }); + } + + // for each record, project "fieldpath" into "dest". + function extract (table, fieldpath, dest) { + var faccess = accessor(fieldpath); + return $.map(table, function (record, index) { + var newrec = $.extend({}, record); + newrec[dest] = faccess(newrec); + return newrec; + }); + } + + // creates a function that will traverse tree of objects based the '.' + // delimited "path" + function accessor (path) { + path = path.split("."); + return function (obj) { + for (var i = 0; i < path.length; i++) + obj = obj[path[i]]; + return obj; + } + } + + // sample use of unnest/extract + function extractminortimes (profile) { + var t1 = unnest([profile], "fragmentProfile", "ma"); + var t2 = unnest(t1, "ma.minorFragmentProfile", "mi"); + + var timetable = $.map(t2, function (record, index) { + var newrec = { + "name" : record.ma.majorFragmentId + "-" + + record.mi.minorFragmentId, + "category" : record.ma.majorFragmentId, + "start" : (record.mi.startTime - record.start) / 1000.0, + "end" : (record.mi.endTime - record.start) / 1000.0 + }; + return newrec; + }); + + timetable.sort(function (r1, r2) { + if (r1.category == r2.category) { + //return r1.name > r2.name; + return r1.start + r2.end > r1.end + r2.start; + } + else return r1.category > r2.category; + + }); + return timetable; + } + + // write the "fieldpaths" for the table "table" into the "domtable" + function builddomtable (domtable, table, fieldpaths) { + var faccessors = $.map(fieldpaths, function (d, i) { + return accessor(d); + }); + + var domrow = domtable.append("tr"); + for (var i = 0; i < fieldpaths.length; i++) + domrow.append("th").text(fieldpaths[i]); + for (var i = 0; i < table.length; i++) { + domrow = domtable.append("tr"); + for (var j = 0; j < faccessors.length; j++) + domrow.append("td").text(faccessors[j](table[i])); + } + } + + // parse the short physical plan into a dagreeD3 structure + function parseplan (planstring) { + var g = new dagreD3.Digraph(); + var ps = $.map(planstring.trim().split("\n"), function (s) { + return [/^([0-9-]+)( *)([a-zA-Z]*)/.exec(s).slice(1)]; + }); + + // nodes + for (var i = 0; i < ps.length; i++) { + g.addNode(ps[i][0], { + label: ps[i][2], + fragment: parseInt(ps[i][0].split("-")[0]) + }); + } + + // edges + var st = [ps[0]]; + for (var i = 1; i < ps.length; i++) { + var top = st.pop(); + while (top[1].length >= ps[i][1].length) + top = st.pop(); + + g.addEdge(null, ps[i][0], top[0]); + + if (ps[i][1].length != top[1].length) + st.push(top); + if (ps[i][1].length >= top[1].length) + st.push(ps[i]); + } + return g; + } + + // graph a "planstring" into the d3 svg handle "svg" + function buildplangraph (svg, planstring) { + var padding = 20; + var graph = parseplan(planstring); + + var renderer = new dagreD3.Renderer(); + renderer.zoom(function () {return function (graph, root) {}}); + + var oldDrawNodes = renderer.drawNodes(); + renderer.drawNodes(function(graph, root) { + var svgNodes = oldDrawNodes(graph, root); + svgNodes.each(function(u) { + var fc = d3.rgb(globalconfig.majorcolorscale(graph.node(u).fragment)); + d3.select(this).select("rect") + .style("fill", graph.node(u).label.endsWith("Exchange") ? "white" : fc) + .style("stroke", "#000") + .style("stroke-width", "1px") + }); + return svgNodes; + }); + + var oldDrawEdgePaths = renderer.drawEdgePaths(); + renderer.drawEdgePaths(function(graph, root) { + var svgEdgePaths = oldDrawEdgePaths(graph, root); + svgEdgePaths.each(function(u) { + d3.select(this).select("path") + .style("fill", "none") + .style("stroke", "#000") + .style("stroke-width", "1px") + }); + return svgEdgePaths; + }); + + var shiftedgroup = svg.append("g") + .attr("transform", "translate(" + padding + "," + padding + ")"); + var layout = dagreD3.layout().nodeSep(20).rankDir("BT"); + var result = renderer.layout(layout).run(graph, shiftedgroup); + + svg.attr("width", result.graph().width + 2 * padding) + .attr("height", result.graph().height + 2 * padding); + } + + function buildtimingchart (svgdest, timetable) { + var chartprops = { + "w" : 800, + "h" : -1, + "svg" : svgdest, + "bheight" : 2, + "bpad" : 0, + "margin" : 50, + "scaler" : null, + "colorer" : null, + }; + + chartprops.h = timetable.length * (chartprops.bheight + chartprops.bpad * 2) + + chartprops.svg + .attr("width", chartprops.w + 2 * chartprops.margin) + .attr("height", chartprops.h + 2 * chartprops.margin) + .attr("class", "svg"); + + chartprops.scaler = d3.scale.linear() + .domain([d3.min(timetable, accessor("start")), + d3.max(timetable, accessor("end"))]) + .range([0, chartprops.w - chartprops.bpad * 2]); + chartprops.colorer = globalconfig.majorcolorscale; + + // backdrop + chartprops.svg.append("g") + .selectAll("rect") + .data(timetable) + .enter() + .append("rect") + .attr("x", 0) + .attr("y", function(d, i) {return i * (chartprops.bheight + 2 * chartprops.bpad);}) + .attr("width", chartprops.w) + .attr("height", chartprops.bheight + chartprops.bpad * 2) + .attr("stroke", "none") + .attr("fill", function(d) {return d3.rgb(chartprops.colorer(d.category));}) + .attr("opacity", 0.1) + .attr("transform", "translate(" + chartprops.margin + "," + + chartprops.margin + ")"); + + // bars + chartprops.svg.append('g') + .selectAll("rect") + .data(timetable) + .enter() + .append("rect") + //.attr("rx", 3) + //.attr("ry", 3) + .attr("x", function(d) {return chartprops.scaler(d.start) + chartprops.bpad;}) + .attr("y", function(d, i) {return i * (chartprops.bheight + 2 * chartprops.bpad) + chartprops.bpad;}) + .attr("width", function(d) {return (chartprops.scaler(d.end) - chartprops.scaler(d.start));}) + .attr("height", chartprops.bheight) + .attr("stroke", "none") + .attr("fill", function(d) {return d3.rgb(chartprops.colorer(d.category));}) + .attr("transform", "translate(" + chartprops.margin + "," + + chartprops.margin + ")"); + + // grid lines + chartprops.svg.append("g") + .attr("transform", "translate(" + + (chartprops.bpad + chartprops.margin) + "," + + (chartprops.h + chartprops.margin) + ")") + .attr("class", "grid") + .call(d3.svg.axis() + .scale(chartprops.scaler) + .tickSize(-chartprops.h, 0) + .tickFormat("")) + .style("stroke", "#000") + .style("opacity", 0.2); + + // ticks + chartprops.svg.append("g") + .attr("transform", "translate(" + + (chartprops.bpad + chartprops.margin) + "," + + (chartprops.h + chartprops.margin) + ")") + .attr("class", "grid") + .call(d3.svg.axis() + .scale(chartprops.scaler) + .orient('bottom') + .tickSize(0, 0) + .tickFormat(d3.format(".2f"))); + } + + function loadprofile (queryid, callback) { + $.ajax({ + type: "GET", + dataType: "json", + url: "/profiles/" + queryid + ".json", + success: callback, + error: function (x, y, z) { + console.log(x); + console.log(y); + console.log(z); + } + }); + } + + function setupglobalconfig (profile) { + globalconfig.profile = profile; + globalconfig.majorcolorscale = d3.scale.category20() + .domain([0, d3.max(profile.fragmentProfile, accessor("majorFragmentId"))]); + + } + + loadprofile(globalconfig.queryid, function (profile) { + setupglobalconfig(profile); + + var queryvisualdrawn = false; + var timingoverviewdrawn = false; + var jsonprofileshown = false; + + // trigger svg drawing when visible + $('#query-tabs').on('shown.bs.tab', function (e) { + if (queryvisualdrawn || !e.target.href.endsWith("#query-visual")) return; + buildplangraph(d3.select("#query-visual-canvas"), profile.plan); + queryvisualdrawn = true; + }) + $('#fragment-accordion').on('shown.bs.collapse', function (e) { + if (timingoverviewdrawn || e.target.id != "fragment-overview") return; + buildtimingchart(d3.select("#fragment-overview-canvas"), extractminortimes(profile)); + timingoverviewdrawn = true; + }); + + // select default tabs + $('#fragment-overview').collapse('show'); + $('#operator-overview').collapse('show'); + $('#query-tabs a[href="#query-query"]').tab('show'); + + + // add json profile on click + $('#full-json-profile-json').on('shown.bs.collapse', function (e) { + if (jsonprofileshown) return; + $('#full-json-profile-json').html(JSON.stringify(globalconfig.profile, null, 4)); + }); + + //builddomtable(d3.select("#timing-table") + // .append("tbody"), extractminortimes(profile), + // ["name", "start", "end"]); + }); }); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6d5ccc81/exec/java-exec/src/main/resources/rest/www/style.css ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/rest/www/style.css b/exec/java-exec/src/main/resources/rest/www/style.css deleted file mode 100644 index 46ff811..0000000 --- a/exec/java-exec/src/main/resources/rest/www/style.css +++ /dev/null @@ -1,27 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one or more contributor - license agreements. See the NOTICE file distributed with this work for additional - information regarding copyright ownership. The ASF licenses this file to - You under the Apache License, Version 2.0 (the "License"); you may not use - this file except in compliance with the License. You may obtain a copy of - the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required - by applicable law or agreed to in writing, software distributed under the - License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS - OF ANY KIND, either express or implied. See the License for the specific - language governing permissions and limitations under the License. */ - - -.node rect { - stroke: #000; - stroke-width: 1px; -} - -.node text { - fill: #000; -} - -.edgePath path { - stroke: #333; - stroke-width: 1.5px; - fill: none; -}
