DRILL-980: Add profile summary to web ui
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/49a9ff27 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/49a9ff27 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/49a9ff27 Branch: refs/heads/master Commit: 49a9ff27f283cbc1c8749989ff408440a0275e7d Parents: 6801a5d Author: Steven Phillips <[email protected]> Authored: Fri Jun 13 00:17:47 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Mon Jun 16 08:02:36 2014 -0700 ---------------------------------------------------------------------- .../exec/server/rest/ProfileResources.java | 4 +- .../drill/exec/server/rest/ProfileWrapper.java | 148 +++++++++++++++++++ .../drill/exec/work/foreman/FragmentData.java | 2 +- .../work/fragment/AbstractStatusReporter.java | 1 + .../src/main/resources/rest/profile/profile.ftl | 10 +- 5 files changed, 160 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/49a9ff27/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileResources.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileResources.java index 386db46..7d29ea3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileResources.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileResources.java @@ -57,7 +57,9 @@ public class ProfileResources { QueryProfile profile = profiles.get(queryId); if(profile == null) profile = QueryProfile.getDefaultInstance(); - return new Viewable("/rest/profile/profile.ftl", profile); + ProfileWrapper wrapper = new ProfileWrapper(profile); + + return new Viewable("/rest/profile/profile.ftl", wrapper); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/49a9ff27/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java new file mode 100644 index 0000000..a1d4df9 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java @@ -0,0 +1,148 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.server.rest; + +import com.google.common.collect.Lists; +import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType; +import org.apache.drill.exec.proto.UserBitShared.MajorFragmentProfile; +import org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile; +import org.apache.drill.exec.proto.UserBitShared.OperatorProfile; +import org.apache.drill.exec.proto.UserBitShared.QueryProfile; + +import java.text.NumberFormat; +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +public class ProfileWrapper { + + NumberFormat format = NumberFormat.getInstance(Locale.US); + + public QueryProfile profile; + + public ProfileWrapper(QueryProfile profile) { + this.profile = profile; + } + + public QueryProfile getProfile() { + return profile; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("MAJOR FRAGMENTS\nid\tmin\tavg\tmax\t(time in ms)\n\n" + listMajorFragments()); + builder.append("\n"); + for (MajorFragmentProfile majorProfile : profile.getFragmentProfileList()) { + builder.append(String.format("Major Fragment: %d\n%s\n", majorProfile.getMajorFragmentId(), new MajorFragmentWrapper(majorProfile).toString())); + } + return builder.toString(); + } + + public String listMajorFragments() { + StringBuilder builder = new StringBuilder(); + for (MajorFragmentProfile m : profile.getFragmentProfileList()) { + List<Long> totalTimes = Lists.newArrayList(); + for (MinorFragmentProfile minorFragmentProfile : m.getMinorFragmentProfileList()) { + totalTimes.add(minorFragmentProfile.getEndTime() - minorFragmentProfile.getStartTime()); + } + long min = Collections.min(totalTimes); + long max = Collections.max(totalTimes); + long sum = 0; + for (Long l : totalTimes) { + sum += l; + } + long avg = sum / totalTimes.size(); + builder.append(String.format("%d\t%s\t%s\t%s\n", m.getMajorFragmentId(), format.format(min), format.format(avg), format.format(max))); + } + return builder.toString(); + } + + public class MajorFragmentWrapper { + MajorFragmentProfile majorFragmentProfile; + + public MajorFragmentWrapper(MajorFragmentProfile majorFragmentProfile) { + this.majorFragmentProfile = majorFragmentProfile; + } + + @Override + public String toString() { + return String.format("Minor Fragments\nid\ttotal time (ms)\n%s\nOperators\nid\ttype\tmin\tavg\tmax\t(time in ns)\n%s\n", new MinorFragmentsInMajor().toString(), new OperatorsInMajor().toString()); + } + + public class MinorFragmentsInMajor { + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + for (MinorFragmentProfile minorFragmentProfile: majorFragmentProfile.getMinorFragmentProfileList()) { + builder.append(String.format("%d\t%s\n", minorFragmentProfile.getMinorFragmentId(), format.format(minorFragmentProfile.getEndTime() - minorFragmentProfile.getStartTime()))); + } + return builder.toString(); + } + } + + public class OperatorsInMajor { + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + int numOperators = majorFragmentProfile.getMinorFragmentProfile(0).getOperatorProfileCount(); + int numFragments = majorFragmentProfile.getMinorFragmentProfileCount(); + long[][] values = new long[numOperators + 1][numFragments]; + CoreOperatorType[] operatorTypes = new CoreOperatorType[numOperators + 1]; + + for (int i = 0; i < numFragments; i++) { + MinorFragmentProfile minorProfile = majorFragmentProfile.getMinorFragmentProfile(i); + for (int j = 0; j < numOperators; j++) { + OperatorProfile operatorProfile = minorProfile.getOperatorProfile(j); + int operatorId = operatorProfile.getOperatorId(); + values[operatorId][i] = operatorProfile.getProcessNanos() + operatorProfile.getSetupNanos(); + if (i == 0) { + operatorTypes[operatorId] = CoreOperatorType.valueOf(operatorProfile.getOperatorType()); + } + } + } + + for (int j = 0; j < numOperators + 1; j++) { + if (operatorTypes[j] == null) { + continue; + } + long min = Long.MAX_VALUE; + long max = Long.MIN_VALUE; + long sum = 0; + + for (int i = 0; i < numFragments; i++) { + min = Math.min(min, values[j][i]); + max = Math.max(max, values[j][i]); + sum += values[j][i]; + } + + long avg = sum / numFragments; + + builder.append(String.format("%d\t%s\t%s\t%s\t%s\n", j, operatorTypes[j].toString(), format.format(min), format.format(avg), format.format(max))); + } + return builder.toString(); + } + } + } + + + + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/49a9ff27/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentData.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentData.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentData.java index d9dd33e..b9b3de0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentData.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentData.java @@ -31,7 +31,7 @@ public class FragmentData { public FragmentData(FragmentHandle handle, DrillbitEndpoint endpoint, boolean isLocal) { super(); - MinorFragmentProfile f = MinorFragmentProfile.newBuilder().setState(FragmentState.SENDING).build(); + MinorFragmentProfile f = MinorFragmentProfile.newBuilder().setState(FragmentState.SENDING).setMinorFragmentId(handle.getMinorFragmentId()).build(); this.status = FragmentStatus.newBuilder().setHandle(handle).setProfile(f).build(); this.endpoint = endpoint; this.isLocal = isLocal; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/49a9ff27/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/AbstractStatusReporter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/AbstractStatusReporter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/AbstractStatusReporter.java index 30e4b6a..8d9da19 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/AbstractStatusReporter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/AbstractStatusReporter.java @@ -48,6 +48,7 @@ public abstract class AbstractStatusReporter implements StatusReporter{ } status.setHandle(context.getHandle()); b.setMemoryUsed(context.getAllocator().getAllocatedMemory()); + b.setMinorFragmentId(context.getHandle().getMinorFragmentId()); status.setProfile(b); return status; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/49a9ff27/exec/java-exec/src/main/resources/rest/profile/profile.ftl ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/rest/profile/profile.ftl b/exec/java-exec/src/main/resources/rest/profile/profile.ftl index ffd22db..c3149ab 100644 --- a/exec/java-exec/src/main/resources/rest/profile/profile.ftl +++ b/exec/java-exec/src/main/resources/rest/profile/profile.ftl @@ -20,7 +20,7 @@ <h2>Query</h2> <form role="form" action="/query" method="POST"> <div class="form-group"> - <textarea class="form-control" id="query" name="query">${model.query}</textarea> + <textarea class="form-control" id="query" name="query">${model.getProfile().query}</textarea> </div> <div class="form-group"> <div class="radio-inline"> @@ -47,11 +47,15 @@ <div class="page-header"> <h2>Physical Plan</h2> </div> - <p><pre>${model.plan}</pre></p> + <p><pre>${model.profile.plan}</pre></p> <div class="page-header"> - <h2>Complete Profile</h2> + <h2>Profile Summary</h2> </div> <p><pre>${model.toString()}</pre></p> + <div class="page-header"> + <h2>Complete Profile</h2> + </div> + <p><pre>${model.profile.toString()}</pre></p> </#macro> <@page_html/>
