This is an automated email from the ASF dual-hosted git repository. meiyi pushed a commit to branch branch-2.4 in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.4 by this push: new 35d44c6 HBASE-25636 Expose HBCK report as metrics (#3031) 35d44c6 is described below commit 35d44c6ce6b8973a2e5f75bd78f92b60a03572e1 Author: meiyi <myime...@gmail.com> AuthorDate: Thu Mar 11 15:12:30 2021 +0800 HBASE-25636 Expose HBCK report as metrics (#3031) Signed-off-by: zhangduo <zhang...@apache.org> --- .../master/MetricsAssignmentManagerSource.java | 70 ++++++++++++++++++++++ .../master/MetricsAssignmentManagerSourceImpl.java | 58 ++++++++++++++++++ .../org/apache/hadoop/hbase/master/HbckChore.java | 10 ++++ .../hbase/master/MetricsAssignmentManager.java | 28 +++++++++ .../hbase/master/janitor/CatalogJanitor.java | 12 ++++ 5 files changed, 178 insertions(+) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSource.java index 822e407..2794542 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSource.java @@ -60,6 +60,27 @@ public interface MetricsAssignmentManagerSource extends BaseSource { String RIT_DURATION_DESC = "Total durations in milliseconds for all Regions in Transition (Histogram)."; + // HBCK report metrics + String ORPHAN_REGIONS_ON_RS = "orphanRegionsOnRS"; + String ORPHAN_REGIONS_ON_FS = "orphanRegionsOnFS"; + String INCONSISTENT_REGIONS = "inconsistentRegions"; + + String ORPHAN_REGIONS_ON_RS_DESC = "Current number of Orphan Regions on RS (Gauge)."; + String ORPHAN_REGIONS_ON_FS_DESC = "Current number of Orphan Regions on FS (Gauge)."; + String INCONSISTENT_REGIONS_DESC = "Current number of Inconsistent Regions (Gauge)."; + + // CatalogJanitor Consistency report metrics + String HOLES = "holes"; + String OVERLAPS = "overlaps"; + String UNKNOWN_SERVER_REGIONS = "unknownServerRegions"; + String EMPTY_REGION_INFO_REGIONS = "emptyRegionInfoRegions"; + + String HOLES_DESC = "Current number of Holes (Gauge)."; + String OVERLAPS_DESC = "Current number of Overlaps (Gauge)."; + String UNKNOWN_SERVER_REGIONS_DESC = "Current number of Unknown Server Regions (Gauge)."; + String EMPTY_REGION_INFO_REGIONS_DESC = + "Current number of Regions with Empty Region Info (Gauge)."; + String ASSIGN_METRIC_PREFIX = "assign"; String UNASSIGN_METRIC_PREFIX = "unassign"; String MOVE_METRIC_PREFIX = "move"; @@ -99,6 +120,55 @@ public interface MetricsAssignmentManagerSource extends BaseSource { void updateUnknownServerOpenRegions(int unknownRegions); /** + * Set the number of orphan regions on RS. + * + * @param orphanRegionsOnRs count of the orphan regions on RS in HBCK chore report. + */ + void setOrphanRegionsOnRs(int orphanRegionsOnRs); + + /** + * Set the number of orphan regions on FS. + * + * @param orphanRegionsOnFs count of the orphan regions on FS in HBCK chore report. + */ + void setOrphanRegionsOnFs(int orphanRegionsOnFs); + + /** + * Set the number of inconsistent regions. + * + * @param inconsistentRegions count of the inconsistent regions in HBCK chore report. + */ + void setInconsistentRegions(int inconsistentRegions); + + /** + * Set the number of holes. + * + * @param holes count of the holes in CatalogJanitor Consistency report. + */ + void setHoles(int holes); + + /** + * Set the number of overlaps. + * + * @param overlaps count of the overlaps in CatalogJanitor Consistency report. + */ + void setOverlaps(int overlaps); + + /** + * Set the number of unknown server regions. + * @param unknownServerRegions count of the unknown server regions in CatalogJanitor Consistency + * report. + */ + void setUnknownServerRegions(int unknownServerRegions); + + /** + * Set the number of regions with empty region info. + * @param emptyRegionInfoRegions count of the regions with empty region info in CatalogJanitor + * Consistency report. + */ + void setEmptyRegionInfoRegions(int emptyRegionInfoRegions); + + /** * TODO: Remove. This may not be needed now as assign and unassign counts are tracked separately * Increment the count of operations (assign/unassign). */ diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java index 59b3647..c78464d 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java @@ -39,6 +39,15 @@ public class MetricsAssignmentManagerSourceImpl private MutableGaugeLong deadServerOpenRegions; private MutableGaugeLong unknownServerOpenRegions; + private MutableGaugeLong orphanRegionsOnRsGauge; + private MutableGaugeLong orphanRegionsOnFsGauge; + private MutableGaugeLong inconsistentRegionsGauge; + + private MutableGaugeLong holesGauge; + private MutableGaugeLong overlapsGauge; + private MutableGaugeLong unknownServerRegionsGauge; + private MutableGaugeLong emptyRegionInfoRegionsGauge; + private MutableFastCounter operationCounter; private OperationMetrics assignMetrics; @@ -70,6 +79,20 @@ public class MetricsAssignmentManagerSourceImpl deadServerOpenRegions = metricsRegistry.newGauge(DEAD_SERVER_OPEN_REGIONS, "", 0); unknownServerOpenRegions = metricsRegistry.newGauge(UNKNOWN_SERVER_OPEN_REGIONS, "", 0); + orphanRegionsOnRsGauge = + metricsRegistry.newGauge(ORPHAN_REGIONS_ON_RS, ORPHAN_REGIONS_ON_RS_DESC, 0L); + orphanRegionsOnFsGauge = + metricsRegistry.newGauge(ORPHAN_REGIONS_ON_FS, ORPHAN_REGIONS_ON_FS_DESC, 0L); + inconsistentRegionsGauge = + metricsRegistry.newGauge(INCONSISTENT_REGIONS, INCONSISTENT_REGIONS_DESC, 0L); + + holesGauge = metricsRegistry.newGauge(HOLES, HOLES_DESC, 0L); + overlapsGauge = metricsRegistry.newGauge(OVERLAPS, OVERLAPS_DESC, 0L); + unknownServerRegionsGauge = + metricsRegistry.newGauge(UNKNOWN_SERVER_REGIONS, UNKNOWN_SERVER_REGIONS_DESC, 0L); + emptyRegionInfoRegionsGauge = + metricsRegistry.newGauge(EMPTY_REGION_INFO_REGIONS, EMPTY_REGION_INFO_REGIONS_DESC, 0L); + /** * NOTE: Please refer to HBASE-9774 and HBASE-14282. Based on these two issues, HBase is * moving away from using Hadoop's metric2 to having independent HBase specific Metrics. Use @@ -121,6 +144,41 @@ public class MetricsAssignmentManagerSourceImpl } @Override + public void setOrphanRegionsOnRs(int orphanRegionsOnRs) { + orphanRegionsOnRsGauge.set(orphanRegionsOnRs); + } + + @Override + public void setOrphanRegionsOnFs(int orphanRegionsOnFs) { + orphanRegionsOnFsGauge.set(orphanRegionsOnFs); + } + + @Override + public void setInconsistentRegions(int inconsistentRegions) { + inconsistentRegionsGauge.set(inconsistentRegions); + } + + @Override + public void setHoles(int holes) { + holesGauge.set(holes); + } + + @Override + public void setOverlaps(int overlaps) { + overlapsGauge.set(overlaps); + } + + @Override + public void setUnknownServerRegions(int unknownServerRegions) { + unknownServerRegionsGauge.set(unknownServerRegions); + } + + @Override + public void setEmptyRegionInfoRegions(int emptyRegionInfoRegions) { + emptyRegionInfoRegionsGauge.set(emptyRegionInfoRegions); + } + + @Override public OperationMetrics getAssignMetrics() { return assignMetrics; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HbckChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HbckChore.java index 63fb5dd..050c03a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HbckChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HbckChore.java @@ -144,6 +144,7 @@ public class HbckChore extends ScheduledChore { LOG.warn("Unexpected", t); } running = false; + updateAssignmentManagerMetrics(); } // This function does the sanity checks of making sure the chore is not run when it is @@ -310,6 +311,15 @@ public class HbckChore extends ScheduledChore { tableDirs.size(), numRegions, orphanRegionsOnFS.size()); } + private void updateAssignmentManagerMetrics() { + master.getAssignmentManager().getAssignmentManagerMetrics() + .updateOrphanRegionsOnRs(getOrphanRegionsOnRS().size()); + master.getAssignmentManager().getAssignmentManagerMetrics() + .updateOrphanRegionsOnFs(getOrphanRegionsOnFS().size()); + master.getAssignmentManager().getAssignmentManagerMetrics() + .updateInconsistentRegions(getInconsistentRegions().size()); + } + /** * When running, the HBCK report may be changed later. */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManager.java index 38aeef2..4c89263 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManager.java @@ -104,6 +104,34 @@ public class MetricsAssignmentManager { assignmentManagerSource.updateUnknownServerOpenRegions(unknownRegions); } + public void updateOrphanRegionsOnRs(int orphanRegionsOnRs) { + assignmentManagerSource.setOrphanRegionsOnRs(orphanRegionsOnRs); + } + + public void updateOrphanRegionsOnFs(int orphanRegionsOnFs) { + assignmentManagerSource.setOrphanRegionsOnFs(orphanRegionsOnFs); + } + + public void updateInconsistentRegions(int inconsistentRegions) { + assignmentManagerSource.setInconsistentRegions(inconsistentRegions); + } + + public void updateHoles(int holes) { + assignmentManagerSource.setHoles(holes); + } + + public void updateOverlaps(int overlaps) { + assignmentManagerSource.setOverlaps(overlaps); + } + + public void updateUnknownServerRegions(int unknownServerRegions) { + assignmentManagerSource.setUnknownServerRegions(unknownServerRegions); + } + + public void updateEmptyRegionInfoRegions(int emptyRegionInfoRegions) { + assignmentManagerSource.setEmptyRegionInfoRegions(emptyRegionInfoRegions); + } + /** * @return Set of common metrics for assign procedure */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java index f0189ad..c23ccf0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java @@ -165,6 +165,7 @@ public class CatalogJanitor extends ScheduledChore { if (!this.lastReport.isEmpty()) { LOG.warn(this.lastReport.toString()); } + updateAssignmentManagerMetrics(); if (isRIT(this.services.getAssignmentManager())) { LOG.warn("Playing-it-safe skipping merge/split gc'ing of regions from hbase:meta while " + @@ -404,6 +405,17 @@ public class CatalogJanitor extends ScheduledChore { return this.services.getTableDescriptors().get(tableName); } + private void updateAssignmentManagerMetrics() { + services.getAssignmentManager().getAssignmentManagerMetrics() + .updateHoles(lastReport.getHoles().size()); + services.getAssignmentManager().getAssignmentManagerMetrics() + .updateOverlaps(lastReport.getOverlaps().size()); + services.getAssignmentManager().getAssignmentManagerMetrics() + .updateUnknownServerRegions(lastReport.getUnknownServers().size()); + services.getAssignmentManager().getAssignmentManagerMetrics() + .updateEmptyRegionInfoRegions(lastReport.getEmptyRegionInfo().size()); + } + private static void checkLog4jProperties() { String filename = "log4j.properties"; try (final InputStream inStream =