This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 01ca956ecf7 HBASE-30102 Add metric to account for region data
classified as cold by the Time Based Priority logic (#8128)
01ca956ecf7 is described below
commit 01ca956ecf71143085c5fd4a2a118f0e9c1d6ee0
Author: Wellington Ramos Chevreuil <[email protected]>
AuthorDate: Thu Apr 30 14:26:06 2026 +0100
HBASE-30102 Add metric to account for region data classified as cold by the
Time Based Priority logic (#8128)
Signed-off-by: Peter Somogyi <[email protected]>
Signed-off-by: Tak Lon (Stephen) Wu <[email protected]>
---
.../org/apache/hadoop/hbase/RegionMetrics.java | 6 +
.../apache/hadoop/hbase/RegionMetricsBuilder.java | 26 +++-
.../regionserver/MetricsRegionServerSource.java | 5 +
.../regionserver/MetricsRegionSourceImpl.java | 4 +
.../hbase/regionserver/MetricsRegionWrapper.java | 5 +
.../regionserver/TestMetricsRegionSourceImpl.java | 5 +
.../src/main/protobuf/server/ClusterStatus.proto | 8 +-
.../hadoop/hbase/io/hfile/BlockCacheUtil.java | 2 +-
.../apache/hadoop/hbase/io/hfile/HFileInfo.java | 5 +
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 34 +++--
.../hbase/io/hfile/bucket/BucketProtoUtils.java | 13 +-
.../hbase/regionserver/DataTieringManager.java | 67 +++++++++-
.../hadoop/hbase/regionserver/HRegionServer.java | 30 +++--
.../apache/hadoop/hbase/regionserver/HStore.java | 4 +
.../regionserver/MetricsRegionWrapperImpl.java | 12 ++
.../regionserver/regionListStoreStats.jsp | 2 +
.../hbase/master/TestRegionsRecoveryChore.java | 6 +
.../regionserver/MetricsRegionWrapperStub.java | 5 +
.../hbase/regionserver/TestDataTieringManager.java | 147 +++++++++++++++++++++
19 files changed, 350 insertions(+), 36 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
index b029d028856..a4551c8a340 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
@@ -144,4 +144,10 @@ public interface RegionMetrics {
/** Returns current prefetch ratio of this region on this server */
float getCurrentRegionCachedRatio();
+
+ /**
+ * Returns the ratio of tiered cold data size to total region (HFiles) size
on this server, in the
+ * range [0,1].
+ */
+ float getCurrentRegionColdDataRatio();
}
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
index d3361693079..2486cd89652 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
@@ -81,7 +81,8 @@ public final class RegionMetricsBuilder {
.setUncompressedStoreFileSize(
new Size(regionLoadPB.getStoreUncompressedSizeMB(),
Size.Unit.MEGABYTE))
.setRegionSizeMB(new Size(regionLoadPB.getRegionSizeMB(),
Size.Unit.MEGABYTE))
-
.setCurrentRegionCachedRatio(regionLoadPB.getCurrentRegionCachedRatio()).build();
+ .setCurrentRegionCachedRatio(regionLoadPB.getCurrentRegionCachedRatio())
+
.setCurrentRegionColdDataRatio(regionLoadPB.getCurrentRegionColdDataRatio()).build();
}
private static List<ClusterStatusProtos.StoreSequenceId>
@@ -122,7 +123,8 @@ public final class RegionMetricsBuilder {
.setStoreUncompressedSizeMB(
(int)
regionMetrics.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE))
.setRegionSizeMB((int)
regionMetrics.getRegionSizeMB().get(Size.Unit.MEGABYTE))
-
.setCurrentRegionCachedRatio(regionMetrics.getCurrentRegionCachedRatio()).build();
+ .setCurrentRegionCachedRatio(regionMetrics.getCurrentRegionCachedRatio())
+
.setCurrentRegionColdDataRatio(regionMetrics.getCurrentRegionColdDataRatio()).build();
}
public static RegionMetricsBuilder newBuilder(byte[] name) {
@@ -158,6 +160,7 @@ public final class RegionMetricsBuilder {
private CompactionState compactionState;
private Size regionSizeMB = Size.ZERO;
private float currentRegionCachedRatio;
+ private float currentRegionColdDataRatio;
private RegionMetricsBuilder(byte[] name) {
this.name = name;
@@ -303,6 +306,11 @@ public final class RegionMetricsBuilder {
return this;
}
+ public RegionMetricsBuilder setCurrentRegionColdDataRatio(float value) {
+ this.currentRegionColdDataRatio = value;
+ return this;
+ }
+
public RegionMetrics build() {
return new RegionMetricsImpl(name, storeCount, storeFileCount,
storeRefCount,
maxCompactedStoreFileRefCount, compactingCellCount, compactedCellCount,
storeFileSize,
@@ -310,7 +318,8 @@ public final class RegionMetricsBuilder {
uncompressedStoreFileSize, writeRequestCount, readRequestCount,
cpRequestCount,
filteredReadRequestCount, completedSequenceId, storeSequenceIds,
dataLocality,
lastMajorCompactionTimestamp, dataLocalityForSsd, blocksLocalWeight,
blocksLocalWithSsdWeight,
- blocksTotalWeight, compactionState, regionSizeMB,
currentRegionCachedRatio);
+ blocksTotalWeight, compactionState, regionSizeMB,
currentRegionCachedRatio,
+ currentRegionColdDataRatio);
}
private static class RegionMetricsImpl implements RegionMetrics {
@@ -343,6 +352,7 @@ public final class RegionMetricsBuilder {
private final CompactionState compactionState;
private final Size regionSizeMB;
private final float currentRegionCachedRatio;
+ private final float currentRegionColdDataRatio;
RegionMetricsImpl(byte[] name, int storeCount, int storeFileCount, int
storeRefCount,
int maxCompactedStoreFileRefCount, final long compactingCellCount, long
compactedCellCount,
@@ -352,7 +362,8 @@ public final class RegionMetricsBuilder {
long filteredReadRequestCount, long completedSequenceId, Map<byte[],
Long> storeSequenceIds,
float dataLocality, long lastMajorCompactionTimestamp, float
dataLocalityForSsd,
long blocksLocalWeight, long blocksLocalWithSsdWeight, long
blocksTotalWeight,
- CompactionState compactionState, Size regionSizeMB, float
currentRegionCachedRatio) {
+ CompactionState compactionState, Size regionSizeMB, float
currentRegionCachedRatio,
+ float currentRegionColdDataRatio) {
this.name = Preconditions.checkNotNull(name);
this.storeCount = storeCount;
this.storeFileCount = storeFileCount;
@@ -382,6 +393,7 @@ public final class RegionMetricsBuilder {
this.compactionState = compactionState;
this.regionSizeMB = regionSizeMB;
this.currentRegionCachedRatio = currentRegionCachedRatio;
+ this.currentRegionColdDataRatio = currentRegionColdDataRatio;
}
@Override
@@ -529,6 +541,11 @@ public final class RegionMetricsBuilder {
return currentRegionCachedRatio;
}
+ @Override
+ public float getCurrentRegionColdDataRatio() {
+ return currentRegionColdDataRatio;
+ }
+
@Override
public String toString() {
StringBuilder sb =
@@ -571,6 +588,7 @@ public final class RegionMetricsBuilder {
Strings.appendKeyValue(sb, "compactionState", compactionState);
Strings.appendKeyValue(sb, "regionSizeMB", regionSizeMB);
Strings.appendKeyValue(sb, "currentRegionCachedRatio",
currentRegionCachedRatio);
+ Strings.appendKeyValue(sb, "currentRegionColdDataRatio",
currentRegionColdDataRatio);
return sb.toString();
}
}
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index ffcbb1fa5d2..1087ca3c60a 100644
---
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -654,6 +654,11 @@ public interface MetricsRegionServerSource extends
BaseSource, JvmPauseMonitorSo
String CURRENT_REGION_CACHE_RATIO = "currentRegionCacheRatio";
String CURRENT_REGION_CACHE_RATIO_DESC = "The percentage of caching
completed for this region.";
+ String CURRENT_REGION_COLD_DATA_RATIO = "currentRegionColdDataRatio";
+
+ String CURRENT_REGION_COLD_DATA_RATIO_DESC = "The percentage of data in this
region that "
+ + "is marked as cold by the configured time based priority logic.";
+
String EXCLUDE_DATA_NODES_COUNT = "excludedDataNodesCount";
String EXCLUDE_DATA_NODES_COUNT_DESC =
"Count of slow/connect error DataNodes excluded during WAL write
operation";
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
index f9cafa12494..be87513b1db 100644
---
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
@@ -245,6 +245,10 @@ public class MetricsRegionSourceImpl implements
MetricsRegionSource {
Interns.info(regionNamePrefix +
MetricsRegionServerSource.CURRENT_REGION_CACHE_RATIO,
MetricsRegionServerSource.CURRENT_REGION_CACHE_RATIO_DESC),
this.regionWrapper.getCurrentRegionCacheRatio());
+ mrb.addGauge(
+ Interns.info(regionNamePrefix +
MetricsRegionServerSource.CURRENT_REGION_COLD_DATA_RATIO,
+ MetricsRegionServerSource.CURRENT_REGION_COLD_DATA_RATIO_DESC),
+ this.regionWrapper.getCurrentRegionColdDataRatio());
mrb.addCounter(
Interns.info(regionNamePrefix +
MetricsRegionSource.COMPACTIONS_COMPLETED_COUNT,
MetricsRegionSource.COMPACTIONS_COMPLETED_DESC),
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
index 66cbd47a66a..9b407ccb118 100644
---
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
@@ -80,6 +80,11 @@ public interface MetricsRegionWrapper {
*/
float getCurrentRegionCacheRatio();
+ /**
+ * Gets the current cold data % ratio for this region.
+ */
+ float getCurrentRegionColdDataRatio();
+
/**
* Get the total number of read requests that have been issued against this
region
*/
diff --git
a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
index 5d2f1f9e051..c0489843945 100644
---
a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
+++
b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
@@ -130,6 +130,11 @@ public class TestMetricsRegionSourceImpl {
return 0;
}
+ @Override
+ public float getCurrentRegionColdDataRatio() {
+ return 0;
+ }
+
@Override
public long getReadRequestCount() {
return 0;
diff --git a/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto
b/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto
index 58fd3c8d2a5..afff971687d 100644
--- a/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto
@@ -183,6 +183,12 @@ message RegionLoad {
/** Current region cache ratio on this server */
optional float current_region_cached_ratio = 29;
+
+ /**
+ * Ratio of tiered cold data size to total region size (HFiles) on this
server,
+ * in the range [0,1]. See DataTieringManager region cold data tracking.
+ */
+ optional float current_region_cold_data_ratio = 30;
}
message UserLoad {
@@ -316,7 +322,7 @@ message ServerLoad {
* The metrics for write requests on this region server
*/
optional uint64 write_requests_count = 14;
-
+
/**
* The active monitored tasks
*/
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java
index e39cb21a422..f48da92f515 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java
@@ -295,7 +295,7 @@ public class BlockCacheUtil {
public static HFileContext cloneContext(HFileContext context) {
HFileContext newContext = new
HFileContextBuilder().withBlockSize(context.getBlocksize())
.withBytesPerCheckSum(0).withChecksumType(ChecksumType.NULL) // no
checksums in cached data
- .withCompression(context.getCompression())
+
.withCompression(context.getCompression()).withHFileName(context.getHFileName())
.withDataBlockEncoding(context.getDataBlockEncoding())
.withHBaseCheckSum(context.isUseHBaseChecksum()).withCompressTags(context.isCompressTags())
.withIncludesMvcc(context.isIncludesMvcc()).withIncludesTags(context.isIncludesTags())
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java
index b3da98f1343..b258caa3133 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java
@@ -77,6 +77,8 @@ public class HFileInfo implements SortedMap<byte[], byte[]> {
static final byte[] KEY_OF_BIGGEST_CELL = Bytes.toBytes(RESERVED_PREFIX +
"KEY_OF_BIGGEST_CELL");
static final byte[] LEN_OF_BIGGEST_CELL = Bytes.toBytes(RESERVED_PREFIX +
"LEN_OF_BIGGEST_CELL");
public static final byte[] MAX_TAGS_LEN = Bytes.toBytes(RESERVED_PREFIX +
"MAX_TAGS_LEN");
+ public static final byte[] FILE_SIZE = Bytes.toBytes(RESERVED_PREFIX +
"FILE_SIZE");
+ public static final byte[] FILE_PATH = Bytes.toBytes(RESERVED_PREFIX +
"FILE_PATH");
private final SortedMap<byte[], byte[]> map = new
TreeMap<>(Bytes.BYTES_COMPARATOR);
/**
@@ -397,6 +399,9 @@ public class HFileInfo implements SortedMap<byte[], byte[]>
{
}
// close the block reader
context.getInputStreamWrapper().unbuffer();
+
+ put(FILE_SIZE, Bytes.toBytes(context.getFileSize()));
+ put(FILE_PATH, Bytes.toBytes(context.getFilePath().toString()));
} catch (Throwable t) {
IOUtils.closeQuietly(context.getInputStreamWrapper(),
e -> LOG.warn("failed to close input stream wrapper", e));
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index 4839494ca62..bf05285bea3 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -784,10 +784,9 @@ public class BucketCache implements BlockCache, HeapSize {
String regionName = key.getRegionName();
regionCachedSize.merge(regionName, cachedSize,
(previousSize, newBlockSize) -> previousSize + newBlockSize);
- LOG.trace("Updating region cached size for region: {}", regionName);
// If all the blocks for a region are evicted from the cache,
// remove the entry for that region from regionCachedSize map.
- if (regionCachedSize.get(regionName) <= 0) {
+ if (regionCachedSize.getOrDefault(regionName, 0L) <= 0) {
regionCachedSize.remove(regionName);
}
}
@@ -1645,13 +1644,7 @@ public class BucketCache implements BlockCache, HeapSize
{
dumpPrefetchList();
}
regionCachedSize.clear();
- fullyCachedFiles.forEach((hFileName, hFileSize) -> {
- // Get the region name for each file
- String regionEncodedName = hFileSize.getFirst();
- long cachedFileSize = hFileSize.getSecond();
- regionCachedSize.merge(regionEncodedName, cachedFileSize,
- (oldpf, fileSize) -> oldpf + fileSize);
- });
+ backingMap.forEach((k, v) -> updateRegionCachedSize(k, v.getLength()));
}
private void dumpPrefetchList() {
@@ -1723,7 +1716,10 @@ public class BucketCache implements BlockCache, HeapSize
{
java.util.Map<java.lang.Integer, java.lang.String> deserializer) throws
IOException {
Pair<ConcurrentHashMap<BlockCacheKey, BucketEntry>,
NavigableSet<BlockCacheKey>> pair2 =
BucketProtoUtils.fromPB(deserializer, chunk, this::createRecycler);
- backingMap.putAll(pair2.getFirst());
+ pair2.getFirst().forEach((k, v) -> {
+ backingMap.put(k, v);
+ updateRegionCachedSize(k, v.getLength());
+ });
blocksByHFile.addAll(pair2.getSecond());
}
@@ -1774,6 +1770,7 @@ public class BucketCache implements BlockCache, HeapSize {
backingMap.clear();
blocksByHFile.clear();
+ regionCachedSize.clear();
// Read the backing map entries in batches.
int numChunks = 0;
@@ -1787,7 +1784,6 @@ public class BucketCache implements BlockCache, HeapSize {
verifyFileIntegrity(cacheEntry);
verifyCapacityAndClasses(cacheEntry.getCacheCapacity(),
cacheEntry.getIoClass(),
cacheEntry.getMapClass());
- updateRegionSizeMapWhileRetrievingFromFile();
}
/**
@@ -1949,6 +1945,10 @@ public class BucketCache implements BlockCache, HeapSize
{
// split references, we might be evicting just half of the blocks
LOG.debug("found {} blocks for file {}, starting offset: {}, end offset:
{}", keySet.size(),
hfileName, initOffset, endOffset);
+ return evictBlockSet(keySet);
+ }
+
+ private int evictBlockSet(Set<BlockCacheKey> keySet) {
int numEvicted = 0;
for (BlockCacheKey key : keySet) {
if (evictBlock(key)) {
@@ -2490,7 +2490,17 @@ public class BucketCache implements BlockCache, HeapSize
{
String fileName = hFileInfo.getHFileContext().getHFileName();
DataTieringManager dataTieringManager = DataTieringManager.getInstance();
if (dataTieringManager != null && !dataTieringManager.isHotData(hFileInfo,
conf)) {
- LOG.debug("Data tiering is enabled for file: '{}' and it is not hot
data", fileName);
+ LOG.debug("Custom tiering is enabled for file: '{}' and it is not hot
data", fileName);
+ // If custom tiering has been just enabled for a file that was cached,
we now need
+ // to evict it.
+ Set<BlockCacheKey> keySet =
+ getAllCacheKeysForFile(hFileInfo.getHFileContext().getHFileName(), 0,
Long.MAX_VALUE);
+ int evictedBlocks = evictBlockSet(keySet);
+ if (evictedBlocks > 0) {
+ LOG.debug(
+ "Evicted {} blocks for file {} as it is now considered cold by
DataTieringManager",
+ evictedBlocks, fileName);
+ }
return Optional.of(false);
}
// if we don't have the file in fullyCachedFiles, we should cache it
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java
index b87e0e0dd62..95808a7a855 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java
@@ -100,9 +100,16 @@ final class BucketProtoUtils {
}
private static BucketCacheProtos.BlockCacheKey toPB(BlockCacheKey key) {
- return
BucketCacheProtos.BlockCacheKey.newBuilder().setHfilename(key.getHfileName())
- .setOffset(key.getOffset()).setPrimaryReplicaBlock(key.isPrimary())
- .setBlockType(toPB(key.getBlockType())).build();
+ BucketCacheProtos.BlockCacheKey.Builder builder =
BucketCacheProtos.BlockCacheKey.newBuilder()
+ .setHfilename(key.getHfileName()).setOffset(key.getOffset())
+
.setPrimaryReplicaBlock(key.isPrimary()).setBlockType(toPB(key.getBlockType()));
+ if (key.getCfName() != null) {
+ builder.setFamilyName(key.getCfName());
+ }
+ if (key.getRegionName() != null) {
+ builder.setRegionName(key.getRegionName());
+ }
+ return builder.build();
}
private static BucketCacheProtos.BlockType toPB(BlockType blockType) {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java
index 2c92d9238dc..53766189a56 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java
@@ -17,17 +17,25 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import static org.apache.hadoop.hbase.io.hfile.HFileInfo.FILE_PATH;
+import static org.apache.hadoop.hbase.io.hfile.HFileInfo.FILE_SIZE;
+
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
import org.apache.hadoop.hbase.io.hfile.HFileInfo;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,6 +64,11 @@ public class DataTieringManager {
private static DataTieringManager instance;
private final Map<String, HRegion> onlineRegions;
+ // Accounts for the total size of cold data in each region, together with a
list of cold files in
+ // that region.
+ private final Map<String, Pair<List<String>, Long>> regionColdDataSize =
+ new ConcurrentHashMap<>();
+
private DataTieringManager(Map<String, HRegion> onlineRegions) {
this.onlineRegions = onlineRegions;
}
@@ -203,7 +216,34 @@ public class DataTieringManager {
if (isWithinGracePeriod(maxTimestamp, configuration)) {
return true;
}
- return hotDataValidator(maxTimestamp,
getDataTieringHotDataAge(configuration));
+ LOG.debug("Max TS: {} for file {}. Cutoff Age TS: {}", maxTimestamp,
+ hFileInfo.getHFileContext().getHFileName(),
getDataTieringHotDataAge(configuration));
+ boolean isHot = hotDataValidator(maxTimestamp,
getDataTieringHotDataAge(configuration));
+ if (!isHot) {
+ Path path = new Path(Bytes.toString(hFileInfo.get(FILE_PATH)));
+ String regionName = path.getParent().getParent().getName();
+ regionColdDataSize.compute(regionName, (k, v) -> {
+ if (v == null) {
+ List<String> files = new ArrayList<>();
+ files.add(hFileInfo.getHFileContext().getHFileName());
+ LOG.debug("computing file {} with size {} as cold data for region
{}",
+ hFileInfo.getHFileContext().getHFileName(),
Bytes.toLong(hFileInfo.get(FILE_SIZE)),
+ regionName);
+ return new Pair<>(files, Bytes.toLong(hFileInfo.get(FILE_SIZE)));
+ } else {
+ if
(!v.getFirst().contains(hFileInfo.getHFileContext().getHFileName())) {
+ v.getFirst().add(hFileInfo.getHFileContext().getHFileName());
+ v.setSecond(v.getSecond() +
Bytes.toLong(hFileInfo.get(FILE_SIZE)));
+ LOG.debug(
+ "adding file {} with size {} as cold data for region {}. Total
cold data size for the region is {}",
+ hFileInfo.getHFileContext().getHFileName(),
Bytes.toLong(hFileInfo.get(FILE_SIZE)),
+ regionName, v.getSecond());
+ }
+ return v;
+ }
+ });
+ }
+ return isHot;
}
// DataTieringType.NONE or other types are considered hot by default
return true;
@@ -347,4 +387,29 @@ public class DataTieringManager {
public static void resetForTestingOnly() {
instance = null;
}
+
+ public Map<String, Pair<List<String>, Long>> getRegionColdDataSize() {
+ return regionColdDataSize;
+ }
+
+ /**
+ * Updates regionColdData size for the region containing the passed
compactedFiles.
+ */
+ public void updateRegionColdDataSize(String encodedRegionName,
+ Collection<HStoreFile> compactedFiles, Collection<HStoreFile> newFiles) {
+ regionColdDataSize.computeIfPresent(encodedRegionName, (k, v) -> {
+ for (HStoreFile file : compactedFiles) {
+ if (v.getFirst().contains(file.getPath().getName())) {
+ v.getFirst().remove(file.getPath().getName());
+ v.setSecond(v.getSecond() -
Bytes.toLong(file.getMetadataValue(FILE_SIZE)));
+ }
+ }
+ for (HStoreFile file : newFiles) {
+ // call isHotData to account for the new file size in
regionColdDataSize, if the new file is
+ // considered cold data as per data-tiering logic.
+ isHotData(file.getFileInfo().getHFileInfo(),
file.getFileInfo().getConf());
+ }
+ return v;
+ });
+ }
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index c16a5dd17be..672607fe5bc 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -65,7 +65,6 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.function.Consumer;
import java.util.stream.Collectors;
import javax.management.MalformedObjectNameException;
import javax.servlet.http.HttpServlet;
@@ -110,9 +109,7 @@ import org.apache.hadoop.hbase.executor.ExecutorType;
import org.apache.hadoop.hbase.http.InfoServer;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.io.hfile.BlockCacheFactory;
-import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache;
import org.apache.hadoop.hbase.io.hfile.HFile;
-import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
import org.apache.hadoop.hbase.io.util.MemorySizeUtil;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
import org.apache.hadoop.hbase.ipc.DecommissionedHostRejectedException;
@@ -1245,7 +1242,6 @@ public class HRegionServer extends
HBaseServerBase<RSRpcServices>
});
});
});
-
serverLoad.setReportStartTime(reportStartTime);
serverLoad.setReportEndTime(reportEndTime);
if (this.infoServer != null) {
@@ -1545,15 +1541,6 @@ public class HRegionServer extends
HBaseServerBase<RSRpcServices>
}
}
- private void computeIfPersistentBucketCache(Consumer<BucketCache>
computation) {
- if (blockCache instanceof CombinedBlockCache) {
- BlockCache l2 = ((CombinedBlockCache) blockCache).getSecondLevelCache();
- if (l2 instanceof BucketCache && ((BucketCache) l2).isCachePersistent())
{
- computation.accept((BucketCache) l2);
- }
- }
- }
-
/**
* @param r Region to get RegionLoad for.
* @param regionLoadBldr the RegionLoad.Builder, can be null
@@ -1619,6 +1606,16 @@ public class HRegionServer extends
HBaseServerBase<RSRpcServices>
}
});
});
+ final MutableFloat currentRegionColdDataRatio = new MutableFloat(0.0f);
+ if (DataTieringManager.getInstance() != null) {
+
DataTieringManager.getInstance().getRegionColdDataSize().computeIfPresent(regionEncodedName,
+ (k, v) -> {
+ int coldSizeMB = roundSize(v.getSecond(), unitMB);
+ currentRegionColdDataRatio
+ .setValue(regionSizeMB == 0 ? 0.0f : (float) coldSizeMB /
regionSizeMB);
+ return v;
+ });
+ }
HDFSBlocksDistribution hdfsBd = r.getHDFSBlocksDistribution();
float dataLocality =
hdfsBd.getBlockLocalityIndex(serverName.getHostname());
@@ -1650,7 +1647,8 @@ public class HRegionServer extends
HBaseServerBase<RSRpcServices>
.setBlocksLocalWithSsdWeight(blocksLocalWithSsdWeight).setBlocksTotalWeight(blocksTotalWeight)
.setCompactionState(ProtobufUtil.createCompactionStateForRegionLoad(r.getCompactionState()))
.setLastMajorCompactionTs(r.getOldestHfileTs(true)).setRegionSizeMB(regionSizeMB)
- .setCurrentRegionCachedRatio(currentRegionCachedRatio.floatValue());
+ .setCurrentRegionCachedRatio(currentRegionCachedRatio.floatValue())
+ .setCurrentRegionColdDataRatio(currentRegionColdDataRatio.floatValue());
r.setCompleteSequenceId(regionLoadBldr);
return regionLoadBldr.build();
}
@@ -3150,6 +3148,10 @@ public class HRegionServer extends
HBaseServerBase<RSRpcServices>
@Override
public boolean removeRegion(final HRegion r, ServerName destination) {
HRegion toReturn =
this.onlineRegions.remove(r.getRegionInfo().getEncodedName());
+ if (DataTieringManager.getInstance() != null) {
+ DataTieringManager.getInstance().getRegionColdDataSize()
+ .remove(r.getRegionInfo().getEncodedName());
+ }
metricsRegionServerImpl.requestsCountCache.remove(r.getRegionInfo().getEncodedName());
if (destination != null) {
long closeSeqNum = r.getMaxFlushedSeqId();
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index 918dbd850ae..25518a89f49 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -1280,6 +1280,10 @@ public class HStore
}, () -> {
synchronized (filesCompacting) {
filesCompacting.removeAll(compactedFiles);
+ if (DataTieringManager.getInstance() != null) {
+ DataTieringManager.getInstance().updateRegionColdDataSize(
+ region.getRegionInfo().getEncodedName(), compactedFiles, result);
+ }
}
});
// These may be null when the RS is shutting down. The space quota Chores
will fix the Region
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
index 81ed1849fca..dfcbf3d1031 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.metrics2.MetricsExecutor;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
@@ -68,6 +69,8 @@ public class MetricsRegionWrapperImpl implements
MetricsRegionWrapper, Closeable
private float currentRegionCacheRatio;
private final String tableDescriptorHash;
+ private float currentRegionColdDataRatio;
+
public MetricsRegionWrapperImpl(HRegion region) {
this.region = region;
this.tableDescriptorHash = determineTableDescriptorHash();
@@ -142,6 +145,10 @@ public class MetricsRegionWrapperImpl implements
MetricsRegionWrapper, Closeable
return currentRegionCacheRatio;
}
+ public float getCurrentRegionColdDataRatio() {
+ return currentRegionColdDataRatio;
+ }
+
@Override
public long getStoreRefCount() {
return storeRefCount;
@@ -349,6 +356,11 @@ public class MetricsRegionWrapperImpl implements
MetricsRegionWrapper, Closeable
region.getRegionInfo().getEncodedName(),
regionCachedAmount.getValue(),
tempStoreFileSize);
currentRegionCacheRatio = regionCachedAmount.floatValue() /
tempStoreFileSize;
+ if (DataTieringManager.getInstance() != null) {
+ currentRegionColdDataRatio =
DataTieringManager.getInstance().getRegionColdDataSize()
+ .getOrDefault(region.getRegionInfo().getEncodedName(), new
Pair<>(null, 0L)).getSecond()
+ / (float) tempStoreFileSize;
+ }
}
numStoreFiles = tempNumStoreFiles;
storeRefCount = tempStoreRefCount;
diff --git
a/hbase-server/src/main/resources/hbase-webapps/regionserver/regionListStoreStats.jsp
b/hbase-server/src/main/resources/hbase-webapps/regionserver/regionListStoreStats.jsp
index b663b74536e..45934d55234 100644
---
a/hbase-server/src/main/resources/hbase-webapps/regionserver/regionListStoreStats.jsp
+++
b/hbase-server/src/main/resources/hbase-webapps/regionserver/regionListStoreStats.jsp
@@ -45,6 +45,7 @@
<th>Data Locality</th>
<th>Len Of Biggest Cell</th>
<th>% Cached</th>
+ <th>% Cold Data</th>
</tr>
</thead>
@@ -111,6 +112,7 @@
<td><%= load.getDataLocality() %></td>
<td><%= String.format("%,1d", lenOfBiggestCellInRegion) %></td>
<td><%= StringUtils.formatPercent(load.getCurrentRegionCachedRatio(),
2) %></td>
+ <td><%=
StringUtils.formatPercent(load.getCurrentRegionColdDataRatio(), 2) %></td>
<% } %>
</tr>
<% } %>
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
index 06b66d118bf..eeba839ac11 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
@@ -399,6 +399,7 @@ public class TestRegionsRecoveryChore {
public Map<String, Integer> getRegionCachedInfo() {
return new HashMap<>();
}
+
};
return serverMetrics;
}
@@ -550,6 +551,11 @@ public class TestRegionsRecoveryChore {
public float getCurrentRegionCachedRatio() {
return 0.0f;
}
+
+ @Override
+ public float getCurrentRegionColdDataRatio() {
+ return 0.0f;
+ }
};
return regionMetrics;
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
index 9c9a5e9dc9c..ed3621748fa 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
@@ -97,6 +97,11 @@ public class MetricsRegionWrapperStub implements
MetricsRegionWrapper {
return 0;
}
+ @Override
+ public float getCurrentRegionColdDataRatio() {
+ return 0;
+ }
+
@Override
public long getReadRequestCount() {
return 105;
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java
index 26269acd178..7ef7ce0884b 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java
@@ -28,6 +28,7 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -61,6 +62,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.CacheTestUtils;
import org.apache.hadoop.hbase.io.hfile.HFileBlock;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
+import org.apache.hadoop.hbase.io.hfile.HFileInfo;
import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
import
org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
@@ -349,6 +351,151 @@ public class TestDataTieringManager {
assert
(coldDataFiles.containsKey(hStoreFiles.get(3).getFileInfo().getActiveFileName()));
}
+ /**
+ * {@link DataTieringManager#isHotData(HFileInfo,
org.apache.hadoop.conf.Configuration)} should
+ * record cold files in {@link DataTieringManager#getRegionColdDataSize()}
(path and aggregate
+ * size).
+ */
+ @Test
+ public void testRegionColdDataSizeRecordColdHFile() throws IOException {
+ initializeTestEnvironment();
+ dataTieringManager.getRegionColdDataSize().clear();
+
+ HStoreFile coldFile = hStoreFiles.get(3);
+ String region = coldFile.getPath().getParent().getParent().getName();
+ assertFalse("fixture file should be cold for TIME_RANGE tiering",
dataTieringManager
+ .isHotData(coldFile.getFileInfo().getHFileInfo(),
coldFile.getFileInfo().getConf()));
+
+ Map<String, Pair<List<String>, Long>> coldByRegion =
dataTieringManager.getRegionColdDataSize();
+ assertTrue(coldByRegion.containsKey(region));
+ Pair<List<String>, Long> entry = coldByRegion.get(region);
+ long expected =
Bytes.toLong(coldFile.getFileInfo().getHFileInfo().get(HFileInfo.FILE_SIZE));
+ assertEquals(expected, (long) entry.getSecond());
+ assertEquals(1, entry.getFirst().size());
+ assertTrue(entry.getFirst().contains(coldFile.getPath().getName()));
+ }
+
+ @Test
+ public void testRegionColdDataSizeSkipsHotHFile() throws IOException {
+ initializeTestEnvironment();
+ dataTieringManager.getRegionColdDataSize().clear();
+
+ HStoreFile hotFile = hStoreFiles.get(0);
+
assertTrue(dataTieringManager.isHotData(hotFile.getFileInfo().getHFileInfo(),
+ hotFile.getFileInfo().getConf()));
+ assertTrue(dataTieringManager.getRegionColdDataSize().isEmpty());
+ }
+
+ @Test
+ public void testRegionColdDataSizeSkipsNoTieringHFile() throws IOException {
+ initializeTestEnvironment();
+ dataTieringManager.getRegionColdDataSize().clear();
+
+ HStoreFile file = hStoreFiles.get(1);
+ assertTrue(dataTieringManager.isHotData(file.getFileInfo().getHFileInfo(),
+ file.getFileInfo().getConf()));
+ String encoded = file.getPath().getParent().getParent().getName();
+
assertFalse(dataTieringManager.getRegionColdDataSize().containsKey(encoded));
+ }
+
+ @Test
+ public void testRegionColdDataSizeForSameHFile() throws IOException {
+ initializeTestEnvironment();
+ dataTieringManager.getRegionColdDataSize().clear();
+
+ HStoreFile coldFile = hStoreFiles.get(3);
+ long expected =
Bytes.toLong(coldFile.getFileInfo().getHFileInfo().get(HFileInfo.FILE_SIZE));
+ dataTieringManager.isHotData(coldFile.getFileInfo().getHFileInfo(),
+ coldFile.getFileInfo().getConf());
+ dataTieringManager.isHotData(coldFile.getFileInfo().getHFileInfo(),
+ coldFile.getFileInfo().getConf());
+
+ String region = coldFile.getPath().getParent().getParent().getName();
+ Pair<List<String>, Long> entry =
dataTieringManager.getRegionColdDataSize().get(region);
+ assertNotNull(entry);
+ assertEquals(expected, (long) entry.getSecond());
+ assertEquals(1, entry.getFirst().size());
+ }
+
+ @Test
+ public void testUpdateRegionColdDataSizeNoopWhenRegionNotTracked() throws
IOException {
+ initializeTestEnvironment();
+ dataTieringManager.getRegionColdDataSize().clear();
+ HStoreFile coldFile = hStoreFiles.get(3);
+ dataTieringManager.updateRegionColdDataSize("not-a-real-encoded-region",
+ Collections.singletonList(coldFile), Collections.emptyList());
+ assertTrue(dataTieringManager.getRegionColdDataSize().isEmpty());
+ }
+
+ @Test
+ public void testUpdateRegionColdDataSizeRemovesCompactedColdAddsNewHot()
throws IOException {
+ initializeTestEnvironment();
+ dataTieringManager.getRegionColdDataSize().clear();
+
+ HStoreFile coldFile = hStoreFiles.get(3);
+ String regionName = coldFile.getPath().getParent().getParent().getName();
+ dataTieringManager.isHotData(coldFile.getFileInfo().getHFileInfo(),
+ coldFile.getFileInfo().getConf());
+
+ HRegion region = testOnlineRegions.get(regionName);
+ assertNotNull(region);
+ HStore hStore = region.getStore(Bytes.toBytes("cf2"));
+ HStoreFile newFile =
createHStoreFile(hStore.getStoreContext().getFamilyStoreDirectoryPath(),
+ hStore.getReadOnlyConfiguration(), System.currentTimeMillis(),
region.getRegionFileSystem());
+ newFile.initReader();
+ hStore.refreshStoreFiles();
+
+ dataTieringManager.updateRegionColdDataSize(regionName,
Collections.singletonList(coldFile),
+ Collections.singletonList(newFile));
+
+ Pair<List<String>, Long> after =
dataTieringManager.getRegionColdDataSize().get(regionName);
+ assertNotNull(after);
+ assertTrue("Cold compacted file should be removed from tracking",
+ after.getFirst().isEmpty() ||
!after.getFirst().contains(coldFile.getPath().getName()));
+ assertEquals(0L, (long) after.getSecond());
+ }
+
+ /**
+ * Like {@link #testUpdateRegionColdDataSizeRemovesCompactedColdAddsNewHot},
but the replacement
+ * store file is still cold under TIME_RANGE rules so {@link
DataTieringManager} should keep the
+ * region entry and record the new file's size.
+ */
+ @Test
+ public void testUpdateRegionColdDataSizeRemovesCompactedColdAddsNewCold()
throws IOException {
+ initializeTestEnvironment();
+ dataTieringManager.getRegionColdDataSize().clear();
+
+ HStoreFile coldFile = hStoreFiles.get(3);
+ String regionName = coldFile.getPath().getParent().getParent().getName();
+ dataTieringManager.isHotData(coldFile.getFileInfo().getHFileInfo(),
+ coldFile.getFileInfo().getConf());
+
+ HRegion region = testOnlineRegions.get(regionName);
+ assertNotNull(region);
+ HStore hStore = region.getStore(Bytes.toBytes("cf2"));
+ // Region2 hot-age is 2.5 * DAY; use 4 * DAY so the new file stays cold.
+ long coldTimestamp = System.currentTimeMillis() - 4 * DAY;
+ HStoreFile newFile =
createHStoreFile(hStore.getStoreContext().getFamilyStoreDirectoryPath(),
+ hStore.getReadOnlyConfiguration(), coldTimestamp,
region.getRegionFileSystem());
+ newFile.initReader();
+ hStore.refreshStoreFiles();
+
+ assertFalse("new store file must be cold for this scenario",
dataTieringManager
+ .isHotData(newFile.getFileInfo().getHFileInfo(),
newFile.getFileInfo().getConf()));
+
+ dataTieringManager.updateRegionColdDataSize(regionName,
Collections.singletonList(coldFile),
+ Collections.singletonList(newFile));
+
+ Pair<List<String>, Long> after =
dataTieringManager.getRegionColdDataSize().get(regionName);
+ assertNotNull(after);
+ assertFalse("compacted cold file should no longer be tracked",
+ after.getFirst().contains(coldFile.getPath().getName()));
+ assertEquals(1, after.getFirst().size());
+ assertTrue(after.getFirst().contains(newFile.getPath().getName()));
+ long expectedNew =
Bytes.toLong(newFile.getFileInfo().getHFileInfo().get(HFileInfo.FILE_SIZE));
+ assertEquals(expectedNew, (long) after.getSecond());
+ }
+
/*
* Verify that two cold blocks(both) are evicted when bucket reaches its
capacity. The hot file
* remains in the cache.
