HBASE-19799 Add web UI to rsgroup Signed-off-by: tedyu <yuzhih...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cde1f821 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cde1f821 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cde1f821 Branch: refs/heads/HBASE-19064 Commit: cde1f821ebec14a30214a15aebb02c936f27fd87 Parents: 581fabe Author: Guangxu Cheng <guangxuch...@gmail.com> Authored: Thu Jan 18 15:16:29 2018 +0800 Committer: tedyu <yuzhih...@gmail.com> Committed: Thu Jan 18 12:49:43 2018 -0800 ---------------------------------------------------------------------- .../hadoop/hbase/RSGroupTableAccessor.java | 82 ++++ .../hadoop/hbase/protobuf/ProtobufUtil.java | 14 + hbase-protocol/src/main/protobuf/RSGroup.proto | 34 ++ hbase-rsgroup/src/main/protobuf/RSGroup.proto | 34 -- .../hbase/tmpl/master/MasterStatusTmpl.jamon | 7 +- .../hbase/tmpl/master/RSGroupListTmpl.jamon | 352 +++++++++++++++ .../resources/hbase-webapps/master/rsgroup.jsp | 441 +++++++++++++++++++ 7 files changed, 929 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/cde1f821/hbase-client/src/main/java/org/apache/hadoop/hbase/RSGroupTableAccessor.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RSGroupTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RSGroupTableAccessor.java new file mode 100644 index 0000000..06a7604 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RSGroupTableAccessor.java @@ -0,0 +1,82 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos; +import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Read rs group information from <code>hbase:rsgroup</code>. + */ +@InterfaceAudience.Private +public final class RSGroupTableAccessor { + + //Assigned before user tables + private static final TableName RSGROUP_TABLE_NAME = + TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "rsgroup"); + private static final byte[] META_FAMILY_BYTES = Bytes.toBytes("m"); + private static final byte[] META_QUALIFIER_BYTES = Bytes.toBytes("i"); + + public static List<RSGroupInfo> getAllRSGroupInfo(Connection connection) + throws IOException { + try (Table rsGroupTable = connection.getTable(RSGROUP_TABLE_NAME)) { + List<RSGroupInfo> rsGroupInfos = new ArrayList<>(); + for (Result result : rsGroupTable.getScanner(new Scan())) { + RSGroupInfo rsGroupInfo = getRSGroupInfo(result); + if (rsGroupInfo != null) { + rsGroupInfos.add(rsGroupInfo); + } + } + return rsGroupInfos; + } + } + + private static RSGroupInfo getRSGroupInfo(Result result) throws IOException { + byte[] rsGroupInfo = result.getValue(META_FAMILY_BYTES, META_QUALIFIER_BYTES); + if (rsGroupInfo == null) { + return null; + } + RSGroupProtos.RSGroupInfo proto = + RSGroupProtos.RSGroupInfo.parseFrom(rsGroupInfo); + return ProtobufUtil.toGroupInfo(proto); + } + + public static RSGroupInfo getRSGroupInfo(Connection connection, byte[] rsGroupName) + throws IOException { + try (Table rsGroupTable = connection.getTable(RSGROUP_TABLE_NAME)){ + Result result = rsGroupTable.get(new Get(rsGroupName)); + return getRSGroupInfo(result); + } + } + + private RSGroupTableAccessor() { + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/cde1f821/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index 2f2dc86..fd263b0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -73,6 +73,7 @@ import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.filter.ByteArrayComparable; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.io.TimeRange; +import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse; @@ -92,7 +93,9 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType; import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; +import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; import org.apache.hadoop.hbase.util.Addressing; import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; @@ -1819,4 +1822,15 @@ public final class ProtobufUtil { int port = Addressing.parsePort(str); return ServerName.valueOf(hostname, port, -1L); } + + public static RSGroupInfo toGroupInfo(RSGroupProtos.RSGroupInfo proto) { + RSGroupInfo RSGroupInfo = new RSGroupInfo(proto.getName()); + for(HBaseProtos.ServerName el: proto.getServersList()) { + RSGroupInfo.addServer(Address.fromParts(el.getHostName(), el.getPort())); + } + for(HBaseProtos.TableName pTableName: proto.getTablesList()) { + RSGroupInfo.addTable(ProtobufUtil.toTableName(pTableName)); + } + return RSGroupInfo; + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/cde1f821/hbase-protocol/src/main/protobuf/RSGroup.proto ---------------------------------------------------------------------- diff --git a/hbase-protocol/src/main/protobuf/RSGroup.proto b/hbase-protocol/src/main/protobuf/RSGroup.proto new file mode 100644 index 0000000..7358941 --- /dev/null +++ b/hbase-protocol/src/main/protobuf/RSGroup.proto @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "RSGroupProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; + +message RSGroupInfo { + required string name = 1; + repeated ServerName servers = 4; + repeated TableName tables = 3; +} + http://git-wip-us.apache.org/repos/asf/hbase/blob/cde1f821/hbase-rsgroup/src/main/protobuf/RSGroup.proto ---------------------------------------------------------------------- diff --git a/hbase-rsgroup/src/main/protobuf/RSGroup.proto b/hbase-rsgroup/src/main/protobuf/RSGroup.proto deleted file mode 100644 index 7358941..0000000 --- a/hbase-rsgroup/src/main/protobuf/RSGroup.proto +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package hbase.pb; - -option java_package = "org.apache.hadoop.hbase.protobuf.generated"; -option java_outer_classname = "RSGroupProtos"; -option java_generic_services = true; -option java_generate_equals_and_hash = true; -option optimize_for = SPEED; - -import "HBase.proto"; - -message RSGroupInfo { - required string name = 1; - repeated ServerName servers = 4; - repeated TableName tables = 3; -} - http://git-wip-us.apache.org/repos/asf/hbase/blob/cde1f821/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon index 292a668..d0342db 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon @@ -184,7 +184,12 @@ AssignmentManager assignmentManager = master.getAssignmentManager(); or re-run HBCK in repair mode. </div> </%if> - + <%if master.getMasterCoprocessorHost().findCoprocessor("RSGroupAdminEndpoint") != null %> + <section> + <h2>RSGroup</h2> + <& RSGroupListTmpl; master= master; serverManager= serverManager&> + </section> + </%if> <section> <h2>Region Servers</h2> <& RegionServerListTmpl; master= master; servers = servers &> http://git-wip-us.apache.org/repos/asf/hbase/blob/cde1f821/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon new file mode 100644 index 0000000..9f9831f --- /dev/null +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon @@ -0,0 +1,352 @@ +<%doc> +Copyright The Apache Software Foundation + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + </%doc> + +<%args> +HMaster master; +ServerManager serverManager; +</%args> + +<%import> + java.util.Collections; + java.util.List; + java.util.Map; + java.util.Set; + java.util.stream.Collectors; + org.apache.hadoop.hbase.master.HMaster; + org.apache.hadoop.hbase.ServerLoad; + org.apache.hadoop.hbase.RSGroupTableAccessor; + org.apache.hadoop.hbase.master.ServerManager; + org.apache.hadoop.hbase.net.Address; + org.apache.hadoop.hbase.rsgroup.RSGroupInfo; + org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix; +</%import> +<%java> +List<RSGroupInfo> groups = RSGroupTableAccessor.getAllRSGroupInfo(master.getConnection()); +</%java> + +<%if (groups != null && groups.size() > 0)%> + +<%java> +RSGroupInfo [] rsGroupInfos = groups.toArray(new RSGroupInfo[groups.size()]); +Map<Address, ServerLoad> collectServers = Collections.emptyMap(); +if (master.getServerManager() != null) { + collectServers = + master.getServerManager().getOnlineServers().entrySet().stream() + .collect(Collectors.toMap(p -> p.getKey().getAddress(), Map.Entry::getValue)); +} +</%java> + +<div class="tabbable"> + <ul class="nav nav-pills"> + <li class="active"><a href="#tab_rsgroup_baseStats" data-toggle="tab">Base Stats</a></li> + <li class=""><a href="#tab_rsgroup_memoryStats" data-toggle="tab">Memory</a></li> + <li class=""><a href="#tab_rsgroup_requestStats" data-toggle="tab">Requests</a></li> + <li class=""><a href="#tab_rsgroup_storeStats" data-toggle="tab">Storefiles</a></li> + <li class=""><a href="#tab_rsgroup_compactStats" data-toggle="tab">Compactions</a></li> + </ul> + <div class="tab-content" style="padding-bottom: 9px; border-bottom: 1px solid #ddd;"> + <div class="tab-pane active" id="tab_rsgroup_baseStats"> + <& rsgroup_baseStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &> + </div> + <div class="tab-pane" id="tab_rsgroup_memoryStats"> + <& rsgroup_memoryStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &> + </div> + <div class="tab-pane" id="tab_rsgroup_requestStats"> + <& rsgroup_requestStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &> + </div> + <div class="tab-pane" id="tab_rsgroup_storeStats"> + <& rsgroup_storeStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &> + </div> + <div class="tab-pane" id="tab_rsgroup_compactStats"> + <& rsgroup_compactStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &> + </div> + </div> +</div> + +</%if> + +<%def rsgroup_baseStats> +<%args> + RSGroupInfo [] rsGroupInfos; + Map<Address, ServerLoad> collectServers; +</%args> +<table class="table table-striped"> +<tr> + <th>RSGroup Name</th> + <th>Num. Online Servers</th> + <th>Num. Dead Servers</th> + <th>Num. Tables</th> + <th>Requests Per Second</th> + <th>Num. Regions</th> + <th>Average Load</th> +</tr> +<%java> + int totalOnlineServers = 0; + int totalDeadServers = 0; + int totalTables = 0; + int totalRequests = 0; + int totalRegions = 0; + for (RSGroupInfo rsGroupInfo: rsGroupInfos) { + String rsGroupName = rsGroupInfo.getName(); + int onlineServers = 0; + int deadServers = 0; + int tables = 0; + long requestsPerSecond = 0; + int numRegionsOnline = 0; + Set<Address> servers = rsGroupInfo.getServers(); + for (Address server : servers) { + ServerLoad sl = collectServers.get(server); + if (sl != null) { + requestsPerSecond += sl.getNumberOfRequests(); + numRegionsOnline += sl.getNumberOfRegions(); + //rsgroup total + totalRegions += sl.getNumberOfRegions(); + totalRequests += sl.getNumberOfRequests(); + totalOnlineServers++; + onlineServers++; + } else { + totalDeadServers++; + deadServers++; + } + } + tables = rsGroupInfo.getTables().size(); + totalTables += tables; + double avgLoad = onlineServers == 0 ? 0 : + (double)numRegionsOnline / (double)onlineServers; +</%java> +<tr> + <td><& rsGroupLink; rsGroupName=rsGroupName; &></td> + <td><% onlineServers %></td> + <td><% deadServers %></td> + <td><% tables %></td> + <td><% requestsPerSecond %></td> + <td><% numRegionsOnline %></td> + <td><% avgLoad %></td> +</tr> +<%java> +} +</%java> +<tr><td>Total:<% rsGroupInfos.length %></td> +<td><% totalOnlineServers %></td> +<td><% totalDeadServers %></td> +<td><% totalTables %></td> +<td><% totalRequests %></td> +<td><% totalRegions %></td> +<td><% master.getServerManager().getAverageLoad() %></td> +</tr> +</table> +</%def> + +<%def rsgroup_memoryStats> +<%args> + RSGroupInfo [] rsGroupInfos; + Map<Address, ServerLoad> collectServers; +</%args> +<table class="table table-striped"> +<tr> + <th>RSGroup Name</th> + <th>Used Heap</th> + <th>Max Heap</th> + <th>Memstore Size</th> + +</tr> +<%java> + for (RSGroupInfo rsGroupInfo: rsGroupInfos) { + String rsGroupName = rsGroupInfo.getName(); + long usedHeap = 0; + long maxHeap = 0; + long memstoreSize = 0; + for (Address server : rsGroupInfo.getServers()) { + ServerLoad sl = collectServers.get(server); + if (sl != null) { + usedHeap += sl.getUsedHeapMB(); + maxHeap += sl.getMaxHeapMB(); + memstoreSize += sl.getMemstoreSizeInMB(); + } + } +</%java> +<tr> + <td><& rsGroupLink; rsGroupName=rsGroupName; &></td> + <td><% TraditionalBinaryPrefix.long2String(usedHeap + * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td> + <td><% TraditionalBinaryPrefix.long2String(maxHeap + * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td> + <td><% TraditionalBinaryPrefix.long2String(memstoreSize + * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td> + +</tr> +<%java> +} +</%java> +</table> +</%def> + +<%def rsgroup_requestStats> +<%args> + RSGroupInfo [] rsGroupInfos; + Map<Address, ServerLoad> collectServers; +</%args> +<table class="table table-striped"> +<tr> + <th>RSGroup Name</th> + <th>Request Per Second</th> + <th>Read Request Count</th> + <th>Write Request Count</th> +</tr> +<%java> + for (RSGroupInfo rsGroupInfo: rsGroupInfos) { + String rsGroupName = rsGroupInfo.getName(); + long requestsPerSecond = 0; + long readRequests = 0; + long writeRequests = 0; + for (Address server : rsGroupInfo.getServers()) { + ServerLoad sl = collectServers.get(server); + if (sl != null) { + requestsPerSecond += sl.getNumberOfRequests(); + readRequests += sl.getReadRequestsCount(); + writeRequests += sl.getWriteRequestsCount(); + } + } +</%java> +<tr> +<td><& rsGroupLink; rsGroupName=rsGroupName; &></td> +<td><% requestsPerSecond %></td> +<td><% readRequests %></td> +<td><% writeRequests %></td> +</tr> +<%java> +} +</%java> +</table> +</%def> + + +<%def rsgroup_storeStats> +<%args> + RSGroupInfo [] rsGroupInfos; + Map<Address, ServerLoad> collectServers; +</%args> +<table class="table table-striped"> +<tr> + <th>RSGroup Name</th> + <th>Num. Stores</th> + <th>Num. Storefiles</th> + <th>Storefile Size Uncompressed</th> + <th>Storefile Size</th> + <th>Index Size</th> + <th>Bloom Size</th> +</tr> +<%java> + for (RSGroupInfo rsGroupInfo: rsGroupInfos) { + String rsGroupName = rsGroupInfo.getName(); + int numStores = 0; + long numStorefiles = 0; + long uncompressedStorefileSize = 0; + long storefileSize = 0; + long indexSize = 0; + long bloomSize = 0; + int count = 0; + for (Address server : rsGroupInfo.getServers()) { + ServerLoad sl = collectServers.get(server); + if (sl != null) { + numStores += sl.getStores(); + numStorefiles += sl.getStorefiles(); + uncompressedStorefileSize += sl.getStoreUncompressedSizeMB(); + storefileSize += sl.getStorefileSizeInMB(); + indexSize += sl.getTotalStaticIndexSizeKB(); + bloomSize += sl.getTotalStaticBloomSizeKB(); + count++; + } + } +</%java> +<tr> +<td><& rsGroupLink; rsGroupName=rsGroupName; &></td> +<td><% numStores %></td> +<td><% numStorefiles %></td> +<td><% TraditionalBinaryPrefix.long2String( + uncompressedStorefileSize * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td> +<td><% TraditionalBinaryPrefix.long2String( + storefileSize * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td> +<td><% TraditionalBinaryPrefix.long2String( + indexSize * TraditionalBinaryPrefix.KILO.value, "B", 1) %></td> +<td><% TraditionalBinaryPrefix.long2String( + bloomSize * TraditionalBinaryPrefix.KILO.value, "B", 1) %></td> +</tr> +<%java> +} +</%java> +</table> +</%def> + +<%def rsgroup_compactStats> +<%args> + RSGroupInfo [] rsGroupInfos; + Map<Address, ServerLoad> collectServers; +</%args> +<table class="table table-striped"> +<tr> + <th>RSGroup Name</th> + <th>Num. Compacting KVs</th> + <th>Num. Compacted KVs</th> + <th>Remaining KVs</th> + <th>Compaction Progress</th> +</tr> +<%java> + for (RSGroupInfo rsGroupInfo: rsGroupInfos) { + String rsGroupName = rsGroupInfo.getName(); + int numStores = 0; + long totalCompactingKVs = 0; + long numCompactedKVs = 0; + long remainingKVs = 0; + long compactionProgress = 0; + for (Address server : rsGroupInfo.getServers()) { + ServerLoad sl = collectServers.get(server); + if (sl != null) { + totalCompactingKVs += sl.getTotalCompactingKVs(); + numCompactedKVs += sl.getCurrentCompactedKVs(); + } + } + remainingKVs = totalCompactingKVs - numCompactedKVs; + String percentDone = ""; + if (totalCompactingKVs > 0) { + percentDone = String.format("%.2f", 100 * + ((float) numCompactedKVs / totalCompactingKVs)) + "%"; + } +</%java> +<tr> +<td><& rsGroupLink; rsGroupName=rsGroupName; &></td> +<td><% totalCompactingKVs %></td> +<td><% numCompactedKVs %></td> +<td><% remainingKVs %></td> +<td><% percentDone %></td> +</tr> +<%java> +} +</%java> +</table> +</%def> + + +<%def rsGroupLink> + <%args> + String rsGroupName; + </%args> + <a href=rsgroup.jsp?name=<% rsGroupName %>><% rsGroupName %></a> +</%def> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/cde1f821/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp b/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp new file mode 100644 index 0000000..9f95b76 --- /dev/null +++ b/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp @@ -0,0 +1,441 @@ +<%-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--%> +<%@ page contentType="text/html;charset=UTF-8" + import="java.util.ArrayList" + import="java.util.Collections" + import="java.util.Date" + import="java.util.List" + import="java.util.Map" + import="java.util.regex.Pattern" + import="java.util.stream.Stream" + import="java.util.stream.Collectors" + import="org.apache.hadoop.hbase.HTableDescriptor" + import="org.apache.hadoop.hbase.RSGroupTableAccessor" + import="org.apache.hadoop.hbase.ServerLoad" + import="org.apache.hadoop.hbase.ServerName" + import="org.apache.hadoop.hbase.TableName" + import="org.apache.hadoop.hbase.client.Admin" + import="org.apache.hadoop.hbase.client.RegionInfo" + import="org.apache.hadoop.hbase.client.TableState" + import="org.apache.hadoop.hbase.client.TableDescriptor" + import="org.apache.hadoop.hbase.master.HMaster" + import="org.apache.hadoop.hbase.master.RegionState" + import="org.apache.hadoop.hbase.net.Address" + import="org.apache.hadoop.hbase.rsgroup.RSGroupInfo" + import="org.apache.hadoop.hbase.util.Bytes" + import="org.apache.hadoop.hbase.util.VersionInfo" + import="org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix"%> +<% + HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER); + String rsGroupName = request.getParameter("name"); + List<Address> rsGroupServers = new ArrayList<>(); + List<TableName> rsGroupTables = new ArrayList<>(); + RSGroupInfo rsGroupInfo = null; + if (rsGroupName != null && !rsGroupName.isEmpty()) { + rsGroupInfo = RSGroupTableAccessor.getRSGroupInfo( + master.getConnection(), Bytes.toBytes(rsGroupName)); + if (rsGroupInfo != null) { + rsGroupServers.addAll(rsGroupInfo.getServers()); + rsGroupTables.addAll(rsGroupInfo.getTables()); + } + } + Collections.sort(rsGroupServers); + rsGroupTables.sort((o1, o2) -> { + int compare = Bytes.compareTo(o1.getNamespace(), o2.getNamespace()); + if (compare != 0) + return compare; + compare = Bytes.compareTo(o1.getQualifier(), o2.getQualifier()); + if (compare != 0) + return compare; + return 0; + }); + + Map<Address, ServerLoad> onlineServers = Collections.emptyMap(); + Map<Address, ServerName> serverMaping = Collections.emptyMap(); + if (master.getServerManager() != null) { + onlineServers = master.getServerManager().getOnlineServers().entrySet().stream() + .collect(Collectors.toMap(p -> p.getKey().getAddress(), Map.Entry::getValue)); + serverMaping = + master.getServerManager().getOnlineServers().entrySet().stream() + .collect(Collectors.toMap(p -> p.getKey().getAddress(), Map.Entry::getKey)); + } + pageContext.setAttribute("pageTitle", "RSGroup: " + rsGroupName); +%> +<jsp:include page="header.jsp"> + <jsp:param name="pageTitle" value="${pageTitle}"/> +</jsp:include> +<div class="container-fluid content"> + <% if (rsGroupName == null || rsGroupName.isEmpty() || rsGroupInfo == null) { %> + <div class="row inner_header"> + <div class="page-header"> + <h1>RSGroup: "<%= rsGroupName %>" does not exist</h1> + </div> + </div> + <p>Go <a href="javascript:history.back()">Back</a>, or wait for the redirect. + <% } else { %> + <div class="container-fluid content"> + <div class="row"> + <div class="page-header"> + <h1>RSGroup: <%= rsGroupName %></h1> + </div> + </div> + </div> + <div class="container-fluid content"> + <div class="row"> + <div class="inner_header"> + <h1>Region Servers</h1> + </div> + </div> + <div class="tabbable"> + <% if (rsGroupServers != null && rsGroupServers.size() > 0) { %> + <ul class="nav nav-pills"> + <li class="active"> + <a href="#tab_baseStats" data-toggle="tab">Base Stats</a> + </li> + <li class=""> + <a href="#tab_memoryStats" data-toggle="tab">Memory</a> + </li> + <li class=""> + <a href="#tab_requestStats" data-toggle="tab">Requests</a> + </li> + <li class=""> + <a href="#tab_storeStats" data-toggle="tab">Storefiles</a> + </li> + <li class=""> + <a href="#tab_compactStats" data-toggle="tab">Compactions</a> + </li> + </ul> + + <div class="tab-content" style="padding-bottom: 9px; border-bottom: 1px solid #ddd;"> + <div class="tab-pane active" id="tab_baseStats"> + <table class="table table-striped"> + <tr> + <th>ServerName</th> + <th>Start time</th> + <th>Last contact</th> + <th>Version</th> + <th>Requests Per Second</th> + <th>Num. Regions</th> + </tr> + <% int totalRegions = 0; + int totalRequests = 0; + int inconsistentNodeNum = 0; + String masterVersion = VersionInfo.getVersion(); + for (Address server: rsGroupServers) { + ServerName serverName = serverMaping.get(server); + if (serverName != null) { + ServerLoad sl = onlineServers.get(server); + String version = master.getRegionServerVersion(serverName); + if (!masterVersion.equals(version)) { + inconsistentNodeNum ++; + } + double requestsPerSecond = 0.0; + int numRegionsOnline = 0; + long lastContact = 0; + if (sl != null) { + requestsPerSecond = sl.getRequestsPerSecond(); + numRegionsOnline = sl.getNumberOfRegions(); + totalRegions += sl.getNumberOfRegions(); + totalRequests += sl.getNumberOfRequests(); + lastContact = (System.currentTimeMillis() - sl.getReportTime())/1000; + } + long startcode = serverName.getStartcode(); + int infoPort = master.getRegionServerInfoPort(serverName); + String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status";%> + <tr> + <td><a href="<%= url %>"><%= serverName.getServerName() %></a></td> + <td><%= new Date(startcode) %></td> + <td><%= lastContact %></td> + <td><%= version %></td> + <td><%= String.format("%.0f", requestsPerSecond) %></td> + <td><%= numRegionsOnline %></td> + </tr> + <% } else { %> + <tr> + <td style="color:rgb(192,192,192);"><%= server %></td> + <td style="color:rgb(192,192,192);"><%= "Dead" %></td> + <td></td> + <td></td> + <td></td> + <td></td> + </tr> + <% } %> + <% } %> + <tr><td>Total:<%= rsGroupServers.size() %></td> + <td></td> + <td></td> + <%if (inconsistentNodeNum > 0) { %> + <td style="color:red;"><%= inconsistentNodeNum %> nodes with inconsistent version</td> + <%} else { %> + <td></td> + <%} %> + <td><%= totalRequests %></td> + <td><%= totalRegions %></td> + </tr> + </table> + </div> + <div class="tab-pane" id="tab_memoryStats"> + <table class="table table-striped"> + <tr> + <th>ServerName</th> + <th>Used Heap</th> + <th>Max Heap</th> + <th>Memstore Size</th> + </tr> + <% for (Address server: rsGroupServers) { + ServerName serverName = serverMaping.get(server); + ServerLoad sl = onlineServers.get(server); + if (sl != null && serverName != null) { + int infoPort = master.getRegionServerInfoPort(serverName); + String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; + %> + <tr> + <td><a href="<%= url %>"><%= serverName.getServerName() %></a></td> + <td><%= TraditionalBinaryPrefix.long2String(sl.getUsedHeapMB() + * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td> + <td><%= TraditionalBinaryPrefix.long2String(sl.getMaxHeapMB() + * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td> + <td><%= TraditionalBinaryPrefix.long2String(sl.getMemstoreSizeInMB() + * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td> + </tr> + <% } else { %> + <tr> + <td style="color:rgb(192,192,192);"><%= server %></td> + <td></td> + <td></td> + <td></td> + </tr> + <% } + } %> + </table> + </div> + <div class="tab-pane" id="tab_requestStats"> + <table class="table table-striped"> + <tr> + <th>ServerName</th> + <th>Request Per Second</th> + <th>Read Request Count</th> + <th>Write Request Count</th> + </tr> + <% for (Address server: rsGroupServers) { + ServerName serverName = serverMaping.get(server); + ServerLoad sl = onlineServers.get(server); + if (sl != null && serverName != null) { + int infoPort = master.getRegionServerInfoPort(serverName); + String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; + %> + <tr> + <td><a href="<%= url %>"><%= serverName.getServerName() %></a></td> + <td><%= String.format("%.0f", sl.getRequestsPerSecond()) %></td> + <td><%= sl.getReadRequestsCount() %></td> + <td><%= sl.getWriteRequestsCount() %></td> + </tr> + <% } else { %> + <tr> + <td style="color:rgb(192,192,192);"><%= server %></td> + <td></td> + <td></td> + <td></td> + </tr> + <% } + } %> + </table> + </div> + <div class="tab-pane" id="tab_storeStats"> + <table class="table table-striped"> + <tr> + <th>ServerName</th> + <th>Num. Stores</th> + <th>Num. Storefiles</th> + <th>Storefile Size Uncompressed</th> + <th>Storefile Size</th> + <th>Index Size</th> + <th>Bloom Size</th> + </tr> + <% for (Address server: rsGroupServers) { + ServerName serverName = serverMaping.get(server); + ServerLoad sl = onlineServers.get(server); + if (sl != null && serverName != null) { + int infoPort = master.getRegionServerInfoPort(serverName); + String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; + %> + <tr> + <td><a href="<%= url %>"><%= serverName.getServerName() %></a></td> + <td><%= sl.getStores() %></td> + <td><%= sl.getStorefiles() %></td> + <td><%= TraditionalBinaryPrefix.long2String( + sl.getStoreUncompressedSizeMB() * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td> + <td><%= TraditionalBinaryPrefix.long2String(sl.getStorefileSizeInMB() + * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td> + <td><%= TraditionalBinaryPrefix.long2String(sl.getTotalStaticIndexSizeKB() + * TraditionalBinaryPrefix.KILO.value, "B", 1) %></td> + <td><%= TraditionalBinaryPrefix.long2String(sl.getTotalStaticBloomSizeKB() + * TraditionalBinaryPrefix.KILO.value, "B", 1) %></td> + </tr> + <% } else { %> + <tr> + <td style="color:rgb(192,192,192);"><%= server %></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + </tr> + <% } + } %> + </table> + </div> + <div class="tab-pane" id="tab_compactStats"> + <table class="table table-striped"> + <tr> + <th>ServerName</th> + <th>Num. Compacting KVs</th> + <th>Num. Compacted KVs</th> + <th>Remaining KVs</th> + <th>Compaction Progress</th> + </tr> + <% for (Address server: rsGroupServers) { + ServerName serverName = serverMaping.get(server); + ServerLoad sl = onlineServers.get(server); + if (sl != null && serverName != null) { + String percentDone = ""; + if (sl.getTotalCompactingKVs() > 0) { + percentDone = String.format("%.2f", 100 * + ((float) sl.getCurrentCompactedKVs() / sl.getTotalCompactingKVs())) + "%"; + } + int infoPort = master.getRegionServerInfoPort(serverName); + String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; + %> + <tr> + <td><a href="<%= url %>"><%= serverName.getServerName() %></a></td> + <td><%= sl.getTotalCompactingKVs() %></td> + <td><%= sl.getCurrentCompactedKVs() %></td> + <td><%= sl.getTotalCompactingKVs() - sl.getCurrentCompactedKVs() %></td> + <td><%= percentDone %></td> + </tr> + <% } else { %> + <tr> + <td style="color:rgb(192,192,192);"><%= server %></td> + <td></td> + <td></td> + <td></td> + <td></td> + </tr> + <% } + } %> + </table> + </div> + </div> + <% } else { %> + <p> No Region Servers</p> + <% } %> + </div> + </div> + <br /> + + <div class="container-fluid content"> + <div class="row inner_header"> + <div class="page-header"> + <h1>Tables</h1> + </div> + </div> + + <% if (rsGroupTables != null && rsGroupTables.size() > 0) { + HTableDescriptor[] tables = null; + try (Admin admin = master.getConnection().getAdmin()) { + tables = master.isInitialized() ? admin.listTables((Pattern)null, true) : null; + } + Map<TableName, HTableDescriptor> tableDescriptors + = Stream.of(tables).collect(Collectors.toMap(TableDescriptor::getTableName, p -> p)); + %> + <table class="table table-striped"> + <tr> + <th>Namespace</th> + <th>Table</th> + <th>Stats</th> + <th>Online Regions</th> + <th>Offline Regions</th> + <th>Failed Regions</th> + <th>Split Regions</th> + <th>Other Regions</th> + <th>Description</th> + </tr> + <% for(TableName tableName : rsGroupTables) { + HTableDescriptor htDesc = tableDescriptors.get(tableName); + if(htDesc == null) { + %> + <tr> + <td><%= tableName.getNamespaceAsString() %></td> + <td><%= tableName.getQualifierAsString() %></td> + <td style="color:rgb(0,0,255);"><%= "DELETED" %></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + </tr> + <% } else { %> + <tr> + <td><%= tableName.getNamespaceAsString() %></td> + <td><a href="/table.jsp?name=<%= tableName.getNameAsString() %>"><%= tableName.getQualifierAsString() %></a></td> + <% TableState.State tableState = master.getTableStateManager().getTableState(tableName); + if(TableState.isInStates(tableState, + TableState.State.DISABLED, TableState.State.DISABLING)) { + %> + <td style="color:red;"><%= tableState.name() %></td> + <% } else { %> + <td><%= tableState.name() %></td> + <% } %> + <% Map<RegionState.State, List<RegionInfo>> tableRegions = + master.getAssignmentManager().getRegionStates().getRegionByStateOfTable(tableName); + int openRegionsCount = tableRegions.get(RegionState.State.OPEN).size(); + int offlineRegionsCount = tableRegions.get(RegionState.State.OFFLINE).size(); + int splitRegionsCount = tableRegions.get(RegionState.State.SPLIT).size(); + int failedRegionsCount = tableRegions.get(RegionState.State.FAILED_OPEN).size() + + tableRegions.get(RegionState.State.FAILED_CLOSE).size(); + int otherRegionsCount = 0; + for (List<RegionInfo> list: tableRegions.values()) { + otherRegionsCount += list.size(); + } + // now subtract known states + otherRegionsCount = otherRegionsCount - openRegionsCount + - failedRegionsCount - offlineRegionsCount + - splitRegionsCount; + %> + <td><%= openRegionsCount %></td> + <td><%= offlineRegionsCount %></td> + <td><%= failedRegionsCount %></td> + <td><%= splitRegionsCount %></td> + <td><%= otherRegionsCount %></td> + <td><%= htDesc.toStringCustomizedValues() %></td> + </tr> + <% } + } %> + <p> <%= rsGroupTables.size() %> table(s) in set.</p> + </table> + <% } else { %> + <p> No Tables</p> + <% } %> + </div> +<% } %> +</div> +<jsp:include page="footer.jsp" />
