kgeisz commented on code in PR #7451:
URL: https://github.com/apache/hbase/pull/7451#discussion_r2519624527
##########
hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java:
##########
@@ -200,4 +200,16 @@ public static TableName getTableName(Path archivePath) {
if (p == null) return null;
return TableName.valueOf(p.getName(), tbl);
}
+
+ public static boolean isHFileArchived(Path path) {
+ Path currentDir = path;
+ for (int i = 0; i < 6; i++) {
Review Comment:
Why does this always iterate 6 times? My assumption is it's because that's
the max depth of an archived file. Can that ever be more than 6?
##########
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestPrefetchWithBucketCache.java:
##########
@@ -374,6 +377,132 @@ public void testPrefetchMetricProgress() throws Exception
{
});
}
+ @Test
+ public void testPrefetchMetricProgressForLinks() throws Exception {
+ conf.setLong(BUCKET_CACHE_SIZE_KEY, 200);
+ blockCache = BlockCacheFactory.createBlockCache(conf);
+ cacheConf = new CacheConfig(conf, blockCache);
+ final RegionInfo hri =
+
RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
+ // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/
+ Configuration testConf = new Configuration(this.conf);
+ Path testDir = TEST_UTIL.getDataTestDir(name.getMethodName());
+ CommonFSUtils.setRootDir(testConf, testDir);
+ Path tableDir = CommonFSUtils.getTableDir(testDir, hri.getTable());
+ RegionInfo region =
RegionInfoBuilder.newBuilder(TableName.valueOf(tableDir.getName())).build();
+ Path regionDir = new Path(tableDir, region.getEncodedName());
+ Path cfDir = new Path(regionDir, "cf");
+ HRegionFileSystem regionFS =
+ HRegionFileSystem.createRegionOnFileSystem(testConf, fs, tableDir,
region);
+ Path storeFile = writeStoreFile(100, cfDir);
+ // Prefetches the file blocks
+ LOG.debug("First read should prefetch the blocks.");
+ readStoreFile(storeFile);
+ BucketCache bc =
BucketCache.getBucketCacheFromCacheConfig(cacheConf).get();
+ // Our file should have 6 DATA blocks. We should wait for all of them to
be cached
+ Waiter.waitFor(testConf, 300, () -> bc.getBackingMap().size() == 6);
+ long cachedSize =
bc.getRegionCachedInfo().get().get(region.getEncodedName());
+
+ final RegionInfo dstHri =
+
RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
+ HRegionFileSystem dstRegionFs =
HRegionFileSystem.createRegionOnFileSystem(testConf, fs,
+ CommonFSUtils.getTableDir(testDir, dstHri.getTable()), dstHri);
+
+ Path dstPath = new Path(regionFS.getTableDir(), new
Path(dstHri.getRegionNameAsString(), "cf"));
+
+ Path linkFilePath =
+ new Path(dstPath, HFileLink.createHFileLinkName(region,
storeFile.getName()));
+
+ StoreFileTracker sft = StoreFileTrackerFactory.create(testConf, false,
+ StoreContext.getBuilder().withFamilyStoreDirectoryPath(dstPath)
+ .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.of("cf"))
+ .withRegionFileSystem(dstRegionFs).build());
+ sft.createHFileLink(hri.getTable(), hri.getEncodedName(),
storeFile.getName(), true);
+ StoreFileInfo sfi = sft.getStoreFileInfo(linkFilePath, true);
+
+ HStoreFile hsf = new HStoreFile(sfi, BloomType.NONE, cacheConf);
+ assertTrue(sfi.isLink());
+ hsf.initReader();
+ HFile.Reader reader = hsf.getReader().getHFileReader();
+ while (!reader.prefetchComplete()) {
+ // Sleep for a bit
+ Thread.sleep(1000);
+ }
+ // HFileLink use the path of the target file to create a reader, so it
should resolve to the
+ // already cached blocks and not insert new blocks in the cache.
+ Waiter.waitFor(testConf, 300, () -> bc.getBackingMap().size() == 6);
+
+ assertEquals(cachedSize, (long)
bc.getRegionCachedInfo().get().get(region.getEncodedName()));
+ }
+
+ @Test
+ public void testPrefetchMetricProgressForLinksToArchived() throws Exception {
+ conf.setLong(BUCKET_CACHE_SIZE_KEY, 200);
+ blockCache = BlockCacheFactory.createBlockCache(conf);
+ cacheConf = new CacheConfig(conf, blockCache);
+
+ // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/
+ Configuration testConf = new Configuration(this.conf);
+ Path testDir = TEST_UTIL.getDataTestDir(name.getMethodName());
+ CommonFSUtils.setRootDir(testConf, testDir);
+
+ final RegionInfo hri =
+
RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
+ Path tableDir = CommonFSUtils.getTableDir(testDir, hri.getTable());
+ RegionInfo region =
RegionInfoBuilder.newBuilder(TableName.valueOf(tableDir.getName())).build();
+ Path regionDir = new Path(tableDir, region.getEncodedName());
+ Path cfDir = new Path(regionDir, "cf");
+
+ Path storeFile = writeStoreFile(100, cfDir);
+ // Prefetches the file blocks
+ LOG.debug("First read should prefetch the blocks.");
+ readStoreFile(storeFile);
+ BucketCache bc =
BucketCache.getBucketCacheFromCacheConfig(cacheConf).get();
+ // Our file should have 6 DATA blocks. We should wait for all of them to
be cached
+ Waiter.waitFor(testConf, 300, () -> bc.getBackingMap().size() == 6);
+ long cachedSize =
bc.getRegionCachedInfo().get().get(region.getEncodedName());
+
+ // create another file, but in the archive dir, hence it won't be cached
+ Path archiveRoot = new Path(testDir, "archive");
+ Path archiveTableDir = CommonFSUtils.getTableDir(archiveRoot,
hri.getTable());
+ Path archiveRegionDir = new Path(archiveTableDir, region.getEncodedName());
+ Path archiveCfDir = new Path(archiveRegionDir, "cf");
+ Path archivedFile = writeStoreFile(100, archiveCfDir);
+
+ final RegionInfo testRegion =
+
RegionInfoBuilder.newBuilder(TableName.valueOf(tableDir.getName())).build();
+ final HRegionFileSystem testRegionFs =
HRegionFileSystem.createRegionOnFileSystem(testConf, fs,
+ CommonFSUtils.getTableDir(testDir, testRegion.getTable()), testRegion);
+ // Just create a link to the archived file
+ Path dstPath = new Path(tableDir, new Path(testRegion.getEncodedName(),
"cf"));
+
+ Path linkFilePath =
+ new Path(dstPath, HFileLink.createHFileLinkName(region,
archivedFile.getName()));
+
+ StoreFileTracker sft = StoreFileTrackerFactory.create(testConf, false,
+ StoreContext.getBuilder().withFamilyStoreDirectoryPath(dstPath)
+ .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.of("cf"))
+ .withRegionFileSystem(testRegionFs).build());
+ sft.createHFileLink(hri.getTable(), hri.getEncodedName(),
storeFile.getName(), true);
+ StoreFileInfo sfi = sft.getStoreFileInfo(linkFilePath, true);
+
+ HStoreFile hsf = new HStoreFile(sfi, BloomType.NONE, cacheConf);
+ assertTrue(sfi.isLink());
+ hsf.initReader();
+ HFile.Reader reader = hsf.getReader().getHFileReader();
+ while (!reader.prefetchComplete()) {
+ // Sleep for a bit
+ Thread.sleep(1000);
+ }
+ // HFileLink use the path of the target file to create a reader, but the
target file is in the
+ // archive, so it wasn't cached previously and should ba cached when we
open the link.
Review Comment:
nit
```suggestion
// archive, so it wasn't cached previously and should be cached when we
open the link.
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
