This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 63c0838bb22 HBASE-27826 Add FSFT implementations for Virtual links and
enable them as part of SplitProcedure (#6936)
63c0838bb22 is described below
commit 63c0838bb22554e0f0e8ea936fc24992d4754833
Author: gvprathyusha6 <[email protected]>
AuthorDate: Tue Apr 28 03:58:49 2026 +0530
HBASE-27826 Add FSFT implementations for Virtual links and enable them as
part of SplitProcedure (#6936)
Signed-off-by: Andrew Purtell <[email protected]>
Reviewed-by: sanjeet006py <[email protected]>
---
.../protobuf/server/region/StoreFileTracker.proto | 2 +
.../java/org/apache/hadoop/hbase/io/HFileLink.java | 2 +-
.../java/org/apache/hadoop/hbase/io/Reference.java | 2 +-
.../assignment/MergeTableRegionsProcedure.java | 8 +-
.../assignment/SplitTableRegionProcedure.java | 130 +++++++++++----------
.../hadoop/hbase/regionserver/HMobStore.java | 1 +
.../hbase/regionserver/HRegionFileSystem.java | 54 ++++++---
.../apache/hadoop/hbase/regionserver/HStore.java | 12 +-
.../hadoop/hbase/regionserver/StoreFileInfo.java | 52 ++++++++-
.../hadoop/hbase/regionserver/StoreUtils.java | 11 ++
.../FileBasedStoreFileTracker.java | 86 +++++++++++++-
.../storefiletracker/StoreFileListFile.java | 2 +-
.../storefiletracker/StoreFileTracker.java | 20 +++-
.../storefiletracker/StoreFileTrackerBase.java | 42 +++++--
.../storefiletracker/StoreFileTrackerFactory.java | 11 +-
.../hbase/snapshot/RestoreSnapshotHelper.java | 108 +++++++++--------
...shotFromClientAfterSplittingRegionTestBase.java | 1 -
.../apache/hadoop/hbase/io/hfile/TestPrefetch.java | 2 +-
.../hfile/bucket/TestPrefetchWithBucketCache.java | 2 +-
.../hbase/master/cleaner/TestHFileLinkCleaner.java | 18 ++-
.../hbase/mob/TestMobCompactionWithDefaults.java | 83 +++++++++----
.../regionserver/TestDirectStoreSplitsMerges.java | 14 +--
.../regionserver/TestHRegionReplayEvents.java | 19 ++-
.../hadoop/hbase/regionserver/TestHStoreFile.java | 10 +-
.../regionserver/TestMergesSplitsAddToTracker.java | 8 +-
.../TestSplitTransactionOnCluster.java | 9 +-
.../hbase/regionserver/TestStoreFileInfo.java | 8 +-
.../storefiletracker/TestStoreFileListFile.java | 49 ++++++++
28 files changed, 555 insertions(+), 211 deletions(-)
diff --git
a/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto
b/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto
index 001cb3ea233..985c693b9c6 100644
---
a/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto
+++
b/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto
@@ -25,9 +25,11 @@ option java_generic_services = true;
option java_generate_equals_and_hash = true;
option optimize_for = SPEED;
+import "server/io/FS.proto";
message StoreFileEntry {
required string name = 1;
required uint64 size = 2;
+ optional Reference reference = 3;
}
message StoreFileList {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
index bd5fac1c3c4..b31ebfe578d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
@@ -81,7 +81,7 @@ public class HFileLink extends FileLink {
* The pattern should be used for hfile and reference links that can be
found in
* /hbase/table/region/family/
*/
- private static final Pattern REF_OR_HFILE_LINK_PATTERN =
+ public static final Pattern REF_OR_HFILE_LINK_PATTERN =
Pattern.compile(String.format("^(?:(%s)(?:=))?(%s)=(%s)-(.+)$",
TableName.VALID_NAMESPACE_REGEX,
TableName.VALID_TABLE_QUALIFIER_REGEX,
RegionInfoBuilder.ENCODED_REGION_NAME_REGEX));
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java
index 5388a1105c3..ab305d02f0f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java
@@ -56,7 +56,7 @@ public class Reference {
* For split HStoreFiles, it specifies if the file covers the lower half or
the upper half of the
* key range
*/
- static enum Range {
+ public static enum Range {
/** HStoreFile contains upper half of key range */
top,
/** HStoreFile contains lower half of key range */
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
index c370fed9d9c..8cc447c5224 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
@@ -605,7 +605,7 @@ public class MergeTableRegionsProcedure
final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(),
regionsToMerge[0].getTable());
final FileSystem fs = mfs.getFileSystem();
- List<Path> mergedFiles = new ArrayList<>();
+ List<StoreFileInfo> mergedFiles = new ArrayList<StoreFileInfo>();
HRegionFileSystem mergeRegionFs = HRegionFileSystem
.createRegionOnFileSystem(env.getMasterConfiguration(), fs, tableDir,
mergedRegion);
@@ -622,11 +622,11 @@ public class MergeTableRegionsProcedure
.setState(State.MERGING_NEW);
}
- private List<Path> mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem
regionFs,
+ private List<StoreFileInfo> mergeStoreFiles(MasterProcedureEnv env,
HRegionFileSystem regionFs,
HRegionFileSystem mergeRegionFs, RegionInfo mergedRegion) throws
IOException {
final TableDescriptor htd =
env.getMasterServices().getTableDescriptors().get(mergedRegion.getTable());
- List<Path> mergedFiles = new ArrayList<>();
+ List<StoreFileInfo> mergedFiles = new ArrayList<StoreFileInfo>();
for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
String family = hcd.getNameAsString();
StoreFileTracker tracker =
@@ -643,7 +643,7 @@ public class MergeTableRegionsProcedure
// is running in a regionserver's Store context, or we might not be
able
// to read the hfiles.
storeFileInfo.setConf(storeConfiguration);
- Path refFile =
mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family,
+ StoreFileInfo refFile =
mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family,
new HStoreFile(storeFileInfo, hcd.getBloomFilterType(),
CacheConfig.DISABLED), tracker);
mergedFiles.add(refFile);
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
index 3d3d3d18de2..07d4698a42a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
@@ -71,7 +71,6 @@ import
org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFac
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.wal.WALSplitUtil;
@@ -660,20 +659,57 @@ public class SplitTableRegionProcedure
HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
env.getMasterConfiguration(), fs, tabledir, getParentRegion(), false);
regionFs.createSplitsDir(daughterOneRI, daughterTwoRI);
+ Pair<List<StoreFileInfo>, List<StoreFileInfo>> expectedReferences =
+ splitStoreFiles(env, regionFs);
+ final ExecutorService threadPool = Executors.newFixedThreadPool(2,
+ new
ThreadFactoryBuilder().setNameFormat("RegionCommitter-pool-%d").setDaemon(true)
+
.setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER).build());
+ Future<Path> futureOne = threadPool.submit(new Callable<Path>() {
+ @Override
+ public Path call() throws IOException {
+ return regionFs.commitDaughterRegion(daughterOneRI,
expectedReferences.getFirst(), env);
+ }
+ });
+ Future<Path> futureTwo = threadPool.submit(new Callable<Path>() {
+ @Override
+ public Path call() throws IOException {
+ return regionFs.commitDaughterRegion(daughterTwoRI,
expectedReferences.getSecond(), env);
+ }
+ });
+ handleThreadPoolShutdown(threadPool, env.getMasterConfiguration());
- Pair<List<Path>, List<Path>> expectedReferences = splitStoreFiles(env,
regionFs);
-
- assertSplitResultFilesCount(fs, expectedReferences.getFirst().size(),
- regionFs.getSplitsDir(daughterOneRI));
- regionFs.commitDaughterRegion(daughterOneRI,
expectedReferences.getFirst(), env);
- assertSplitResultFilesCount(fs, expectedReferences.getFirst().size(),
- new Path(tabledir, daughterOneRI.getEncodedName()));
+ try {
+ futureOne.get();
+ futureTwo.get();
+ } catch (InterruptedException e) {
+ throw (InterruptedIOException) new InterruptedIOException().initCause(e);
+ } catch (ExecutionException e) {
+ throw new IOException("Daughter region commit failed", e);
+ }
+ }
- assertSplitResultFilesCount(fs, expectedReferences.getSecond().size(),
- regionFs.getSplitsDir(daughterTwoRI));
- regionFs.commitDaughterRegion(daughterTwoRI,
expectedReferences.getSecond(), env);
- assertSplitResultFilesCount(fs, expectedReferences.getSecond().size(),
- new Path(tabledir, daughterTwoRI.getEncodedName()));
+ private void handleThreadPoolShutdown(ExecutorService threadPool,
Configuration conf)
+ throws IOException {
+ threadPool.shutdown();
+ // Wait for all the tasks to finish.
+ // When splits ran on the RegionServer, how-long-to-wait-configuration was
named
+ // fileSplitTimeout. If set, use its value.
+ long fileSplitTimeout = conf.getLong("hbase.master.fileSplitTimeout",
+ conf.getLong("hbase.regionserver.fileSplitTimeout", 600000));
+ try {
+ boolean stillRunning = !threadPool.awaitTermination(fileSplitTimeout,
TimeUnit.MILLISECONDS);
+ if (stillRunning) {
+ threadPool.shutdownNow();
+ // wait for the thread to shutdown completely.
+ while (!threadPool.isTerminated()) {
+ Thread.sleep(50);
+ }
+ throw new IOException(
+ "Took too long to split the files and create the references,
aborting split");
+ }
+ } catch (InterruptedException e) {
+ throw (InterruptedIOException) new InterruptedIOException().initCause(e);
+ }
}
private void deleteDaughterRegions(final MasterProcedureEnv env) throws
IOException {
@@ -689,8 +725,8 @@ public class SplitTableRegionProcedure
* Create Split directory
* @param env MasterProcedureEnv
*/
- private Pair<List<Path>, List<Path>> splitStoreFiles(final
MasterProcedureEnv env,
- final HRegionFileSystem regionFs) throws IOException {
+ private Pair<List<StoreFileInfo>, List<StoreFileInfo>> splitStoreFiles(
+ final MasterProcedureEnv env, final HRegionFileSystem regionFs) throws
IOException {
final Configuration conf = env.getMasterConfiguration();
TableDescriptor htd =
env.getMasterServices().getTableDescriptors().get(getTableName());
// The following code sets up a thread pool executor with as many slots as
@@ -745,7 +781,8 @@ public class SplitTableRegionProcedure
final ExecutorService threadPool = Executors.newFixedThreadPool(maxThreads,
new
ThreadFactoryBuilder().setNameFormat("StoreFileSplitter-pool-%d").setDaemon(true)
.setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER).build());
- final List<Future<Pair<Path, Path>>> futures = new
ArrayList<Future<Pair<Path, Path>>>(nbFiles);
+ final List<Future<Pair<StoreFileInfo, StoreFileInfo>>> futures =
+ new ArrayList<Future<Pair<StoreFileInfo, StoreFileInfo>>>(nbFiles);
// Split each store file.
for (Map.Entry<String, Collection<StoreFileInfo>> e : files.entrySet()) {
@@ -769,35 +806,13 @@ public class SplitTableRegionProcedure
}
}
}
- // Shutdown the pool
- threadPool.shutdown();
-
- // Wait for all the tasks to finish.
- // When splits ran on the RegionServer, how-long-to-wait-configuration was
named
- // hbase.regionserver.fileSplitTimeout. If set, use its value.
- long fileSplitTimeout = conf.getLong("hbase.master.fileSplitTimeout",
- conf.getLong("hbase.regionserver.fileSplitTimeout", 600000));
- try {
- boolean stillRunning = !threadPool.awaitTermination(fileSplitTimeout,
TimeUnit.MILLISECONDS);
- if (stillRunning) {
- threadPool.shutdownNow();
- // wait for the thread to shutdown completely.
- while (!threadPool.isTerminated()) {
- Thread.sleep(50);
- }
- throw new IOException(
- "Took too long to split the" + " files and create the references,
aborting split");
- }
- } catch (InterruptedException e) {
- throw (InterruptedIOException) new InterruptedIOException().initCause(e);
- }
-
- List<Path> daughterA = new ArrayList<>();
- List<Path> daughterB = new ArrayList<>();
+ handleThreadPoolShutdown(threadPool, conf);
+ List<StoreFileInfo> daughterA = new ArrayList<>();
+ List<StoreFileInfo> daughterB = new ArrayList<>();
// Look for any exception
- for (Future<Pair<Path, Path>> future : futures) {
+ for (Future<Pair<StoreFileInfo, StoreFileInfo>> future : futures) {
try {
- Pair<Path, Path> p = future.get();
+ Pair<StoreFileInfo, StoreFileInfo> p = future.get();
if (p.getFirst() != null) {
daughterA.add(p.getFirst());
}
@@ -819,19 +834,8 @@ public class SplitTableRegionProcedure
return new Pair<>(daughterA, daughterB);
}
- private void assertSplitResultFilesCount(final FileSystem fs,
- final int expectedSplitResultFileCount, Path dir) throws IOException {
- if (expectedSplitResultFileCount != 0) {
- int resultFileCount = FSUtils.getRegionReferenceAndLinkFileCount(fs,
dir);
- if (expectedSplitResultFileCount != resultFileCount) {
- throw new IOException("Failing split. Didn't have expected reference
and HFileLink files"
- + ", expected=" + expectedSplitResultFileCount + ", actual=" +
resultFileCount);
- }
- }
- }
-
- private Pair<Path, Path> splitStoreFile(HRegionFileSystem regionFs,
TableDescriptor htd,
- ColumnFamilyDescriptor hcd, HStoreFile sf) throws IOException {
+ private Pair<StoreFileInfo, StoreFileInfo> splitStoreFile(HRegionFileSystem
regionFs,
+ TableDescriptor htd, ColumnFamilyDescriptor hcd, HStoreFile sf) throws
IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("pid=" + getProcId() + " splitting started for store file: " +
sf.getPath()
+ " for region: " + getParentRegion().getShortNameToLog());
@@ -847,22 +851,22 @@ public class SplitTableRegionProcedure
StoreFileTrackerFactory.create(regionFs.getFileSystem().getConf(), htd,
hcd,
HRegionFileSystem.create(regionFs.getFileSystem().getConf(),
regionFs.getFileSystem(),
regionFs.getTableDir(), daughterTwoRI));
- final Path path_first = regionFs.splitStoreFile(this.daughterOneRI,
familyName, sf, splitRow,
- false, splitPolicy, daughterOneSft);
- final Path path_second = regionFs.splitStoreFile(this.daughterTwoRI,
familyName, sf, splitRow,
- true, splitPolicy, daughterTwoSft);
+ final StoreFileInfo sfiFirst = regionFs.splitStoreFile(this.daughterOneRI,
familyName, sf,
+ splitRow, false, splitPolicy, daughterOneSft);
+ final StoreFileInfo sfiSecond =
regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf,
+ splitRow, true, splitPolicy, daughterTwoSft);
if (LOG.isDebugEnabled()) {
LOG.debug("pid=" + getProcId() + " splitting complete for store file: "
+ sf.getPath()
+ " for region: " + getParentRegion().getShortNameToLog());
}
- return new Pair<Path, Path>(path_first, path_second);
+ return new Pair<StoreFileInfo, StoreFileInfo>(sfiFirst, sfiSecond);
}
/**
* Utility class used to do the file splitting / reference writing in
parallel instead of
* sequentially.
*/
- private class StoreFileSplitter implements Callable<Pair<Path, Path>> {
+ private class StoreFileSplitter implements Callable<Pair<StoreFileInfo,
StoreFileInfo>> {
private final HRegionFileSystem regionFs;
private final ColumnFamilyDescriptor hcd;
private final HStoreFile sf;
@@ -883,7 +887,7 @@ public class SplitTableRegionProcedure
}
@Override
- public Pair<Path, Path> call() throws IOException {
+ public Pair<StoreFileInfo, StoreFileInfo> call() throws IOException {
return splitStoreFile(regionFs, htd, hcd, sf);
}
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
index f828fe33cea..daed80ee8db 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
@@ -102,6 +102,7 @@ public class HMobStore extends HStore {
// table, we need to find the original mob files by this table name. For
details please see
// cloning snapshot for mob files.
private final byte[] refCellTags;
+ private StoreFileTracker mobStoreSFT = null;
public HMobStore(final HRegion region, final ColumnFamilyDescriptor family,
final Configuration confParam, boolean warmup) throws IOException {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
index a1bf09cba5d..ac7a5faa274 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
@@ -496,7 +496,7 @@ public class HRegionFileSystem {
* in the filesystem.
* @param regionInfo daughter {@link
org.apache.hadoop.hbase.client.RegionInfo}
*/
- public Path commitDaughterRegion(final RegionInfo regionInfo, List<Path>
allRegionFiles,
+ public Path commitDaughterRegion(final RegionInfo regionInfo,
List<StoreFileInfo> allRegionFiles,
MasterProcedureEnv env) throws IOException {
Path regionDir = this.getSplitsDir(regionInfo);
if (fs.exists(regionDir)) {
@@ -511,21 +511,43 @@ public class HRegionFileSystem {
return regionDir;
}
- private void insertRegionFilesIntoStoreTracker(List<Path> allFiles,
MasterProcedureEnv env,
- HRegionFileSystem regionFs) throws IOException {
+ private void insertRegionFilesIntoStoreTracker(List<StoreFileInfo> allFiles,
+ MasterProcedureEnv env, HRegionFileSystem regionFs) throws IOException {
TableDescriptor tblDesc =
env.getMasterServices().getTableDescriptors().get(regionInfo.getTable());
// we need to map trackers per store
Map<String, StoreFileTracker> trackerMap = new HashMap<>();
// we need to map store files per store
Map<String, List<StoreFileInfo>> fileInfoMap = new HashMap<>();
- for (Path file : allFiles) {
+ for (StoreFileInfo sfi : allFiles) {
+ Path file = sfi.getPath();
String familyName = file.getParent().getName();
trackerMap.computeIfAbsent(familyName, t ->
StoreFileTrackerFactory.create(conf, tblDesc,
tblDesc.getColumnFamily(Bytes.toBytes(familyName)), regionFs));
fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>());
List<StoreFileInfo> infos = fileInfoMap.get(familyName);
- infos.add(trackerMap.get(familyName).getStoreFileInfo(file, true));
+ infos.add(sfi);
+ }
+ for (Map.Entry<String, StoreFileTracker> entry : trackerMap.entrySet()) {
+ entry.getValue().add(fileInfoMap.get(entry.getKey()));
+ }
+ }
+
+ private void insertRegionfilePathsIntoStoreTracker(List<StoreFileInfo>
allFiles,
+ MasterProcedureEnv env, HRegionFileSystem regionFs) throws IOException {
+ TableDescriptor tblDesc =
+ env.getMasterServices().getTableDescriptors().get(regionInfo.getTable());
+ // we need to map trackers per store
+ Map<String, StoreFileTracker> trackerMap = new HashMap<>();
+ // we need to map store files per store
+ Map<String, List<StoreFileInfo>> fileInfoMap = new HashMap<>();
+ for (StoreFileInfo file : allFiles) {
+ String familyName = file.getPath().getParent().getName();
+ trackerMap.computeIfAbsent(familyName, t ->
StoreFileTrackerFactory.create(conf, tblDesc,
+ tblDesc.getColumnFamily(familyName.getBytes()), regionFs));
+ fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>());
+ List<StoreFileInfo> infos = fileInfoMap.get(familyName);
+ infos.add(file);
}
for (Map.Entry<String, StoreFileTracker> entry : trackerMap.entrySet()) {
entry.getValue().add(fileInfoMap.get(entry.getKey()));
@@ -568,8 +590,9 @@ public class HRegionFileSystem {
* have a reference to a Region.
* @return Path to created reference.
*/
- public Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f,
byte[] splitRow,
- boolean top, RegionSplitPolicy splitPolicy, StoreFileTracker tracker)
throws IOException {
+ public StoreFileInfo splitStoreFile(RegionInfo hri, String familyName,
HStoreFile f,
+ byte[] splitRow, boolean top, RegionSplitPolicy splitPolicy,
StoreFileTracker tracker)
+ throws IOException {
Path splitDir = new Path(getSplitsDir(hri), familyName);
// Add the referred-to regions name as a dot separated suffix.
// See REF_NAME_REGEX regex above. The referred-to regions name is
@@ -581,7 +604,7 @@ public class HRegionFileSystem {
Path p = new Path(splitDir, f.getPath().getName() + "." +
parentRegionName);
if (fs.exists(p)) {
LOG.warn("Found an already existing split file for {}. Assuming this is
a recovery.", p);
- return p;
+ return tracker.getStoreFileInfo(fs.getFileStatus(p), p, true);
}
boolean createLinkFile = false;
if (splitPolicy == null ||
!splitPolicy.skipStoreFileRangeCheck(familyName)) {
@@ -639,12 +662,12 @@ public class HRegionFileSystem {
hfileName = m.group(4);
}
// must create back reference here
- tracker.createHFileLink(linkedTable, linkedRegion, hfileName, true);
+ HFileLink hFileLink = tracker.createHFileLink(linkedTable,
linkedRegion, hfileName, true);
Path path =
new Path(splitDir, HFileLink.createHFileLinkName(linkedTable,
linkedRegion, hfileName));
LOG.info("Created linkFile:" + path.toString() + " for child: " +
hri.getEncodedName()
+ ", parent: " + regionInfoForFs.getEncodedName());
- return path;
+ return new StoreFileInfo(conf, fs, path, hFileLink);
} catch (IOException e) {
// if create HFileLink file failed, then just skip the error and
create Reference file
LOG.error("Create link file for " + hfileName + " for child " +
hri.getEncodedName()
@@ -655,7 +678,7 @@ public class HRegionFileSystem {
Reference r =
top ? Reference.createTopReference(splitRow) :
Reference.createBottomReference(splitRow);
tracker.createReference(r, p);
- return p;
+ return new StoreFileInfo(conf, fs, p, r);
}
//
===========================================================================
@@ -696,7 +719,7 @@ public class HRegionFileSystem {
* @return Path to created reference.
* @throws IOException if the merge write fails.
*/
- public Path mergeStoreFile(RegionInfo mergingRegion, String familyName,
HStoreFile f,
+ public StoreFileInfo mergeStoreFile(RegionInfo mergingRegion, String
familyName, HStoreFile f,
StoreFileTracker tracker) throws IOException {
Path referenceDir = new Path(getMergesDir(regionInfoForFs), familyName);
// A whole reference to the store file.
@@ -710,13 +733,14 @@ public class HRegionFileSystem {
// suffix and into the new region location (under same family).
Path p = new Path(referenceDir, f.getPath().getName() + "." +
mergingRegionName);
tracker.createReference(r, p);
- return p;
+ StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, p, r);
+ return storeFileInfo;
}
/**
* Commit a merged region, making it ready for use.
*/
- public void commitMergedRegion(List<Path> allMergedFiles, MasterProcedureEnv
env)
+ public void commitMergedRegion(List<StoreFileInfo> allMergedFiles,
MasterProcedureEnv env)
throws IOException {
Path regionDir = getMergesDir(regionInfoForFs);
if (regionDir != null && fs.exists(regionDir)) {
@@ -724,7 +748,7 @@ public class HRegionFileSystem {
Path regionInfoFile = new Path(regionDir, REGION_INFO_FILE);
byte[] regionInfoContent = getRegionInfoFileContent(regionInfo);
writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
- insertRegionFilesIntoStoreTracker(allMergedFiles, env, this);
+ insertRegionfilePathsIntoStoreTracker(allMergedFiles, env, this);
}
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index fde89d122e2..918dbd850ae 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -74,6 +74,7 @@ import org.apache.hadoop.hbase.conf.ConfigKey;
import org.apache.hadoop.hbase.conf.ConfigurationManager;
import org.apache.hadoop.hbase.conf.PropagatingConfigurationObserver;
import org.apache.hadoop.hbase.coprocessor.ReadOnlyConfiguration;
+import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
@@ -1393,6 +1394,11 @@ public class HStore
for (HStoreFile sf : this.getStorefiles()) {
if (inputFiles.contains(sf.getPath().getName())) {
inputStoreFiles.add(sf);
+ } else if (
+ !isPrimaryReplicaStore() && sf.getFileInfo().isLink()
+ &&
inputFiles.contains(HFileLink.getReferencedHFileName(sf.getPath().getName()))
+ ) {
+ inputStoreFiles.add(sf);
}
}
@@ -1404,7 +1410,8 @@ public class HStore
compactionOutputs.remove(sf.getPath().getName());
}
for (String compactionOutput : compactionOutputs) {
- StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false,
storeContext);
+ StoreFileTracker sft =
+ StoreFileTrackerFactory.create(conf, isPrimaryReplicaStore(),
storeContext);
StoreFileInfo storeFileInfo =
getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(),
compactionOutput, sft);
HStoreFile storeFile =
storeEngine.createStoreFileAndReader(storeFileInfo);
@@ -2049,7 +2056,8 @@ public class HStore
List<HStoreFile> storeFiles = new ArrayList<>(fileNames.size());
for (String file : fileNames) {
// open the file as a store file (hfile link, etc)
- StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false,
storeContext);
+ StoreFileTracker sft =
+ StoreFileTrackerFactory.create(conf, isPrimaryReplicaStore(),
storeContext);
StoreFileInfo storeFileInfo =
getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), file,
sft);
HStoreFile storeFile =
storeEngine.createStoreFileAndReader(storeFileInfo);
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
index 1184f39da66..8fdd3f40461 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
@@ -173,6 +173,17 @@ public class StoreFileInfo implements Configurable {
this(conf, fs, fileStatus, null, link);
}
+ /**
+ * Create a Store File Info from an HFileLink
+ * @param conf The {@link Configuration} to use
+ * @param fs The current file system to use
+ * @param fileStatus The {@link FileStatus} of the file
+ */
+ public StoreFileInfo(final Configuration conf, final FileSystem fs, final
Path initiaPath,
+ final HFileLink link) {
+ this(conf, fs, initiaPath, null, link);
+ }
+
/**
* Create a Store File Info from an HFileLink
* @param conf The {@link Configuration} to use
@@ -185,6 +196,18 @@ public class StoreFileInfo implements Configurable {
this(conf, fs, fileStatus, reference, null);
}
+ /**
+ * Create a Store File Info from an HFileLink
+ * @param conf The {@link Configuration} to use
+ * @param fs The current file system to use
+ * @param fileStatus The {@link FileStatus} of the file
+ * @param reference The reference instance
+ */
+ public StoreFileInfo(final Configuration conf, final FileSystem fs, final
Path initialPath,
+ final Reference reference) {
+ this(conf, fs, initialPath, reference, null);
+ }
+
/**
* Create a Store File Info from an HFileLink and a Reference
* @param conf The {@link Configuration} to use
@@ -206,6 +229,26 @@ public class StoreFileInfo implements Configurable {
this.conf.getBoolean(STORE_FILE_READER_NO_READAHEAD,
DEFAULT_STORE_FILE_READER_NO_READAHEAD);
}
+ /**
+ * Create a Store File Info from an HFileLink and a Reference
+ * @param conf The {@link Configuration} to use
+ * @param fs The current file system to use
+ * @param fileStatus The {@link FileStatus} of the file
+ * @param reference The reference instance
+ * @param link The link instance
+ */
+ public StoreFileInfo(final Configuration conf, final FileSystem fs, final
Path path,
+ final Reference reference, final HFileLink link) {
+ this.fs = fs;
+ this.conf = conf;
+ this.primaryReplica = false;
+ this.initialPath = path;
+ this.reference = reference;
+ this.link = link;
+ this.noReadahead =
+ this.conf.getBoolean(STORE_FILE_READER_NO_READAHEAD,
DEFAULT_STORE_FILE_READER_NO_READAHEAD);
+ }
+
/**
* Create a Store File Info from an HFileLink and a Reference
* @param conf The {@link Configuration} to use
@@ -611,12 +654,17 @@ public class StoreFileInfo implements Configurable {
* @return <tt>true</tt> if the file could be a valid store file,
<tt>false</tt> otherwise
*/
public static boolean validateStoreFileName(final String fileName) {
- if (HFileLink.isHFileLink(fileName) || isReference(fileName)) {
+ if (HFileLink.isHFileLink(fileName) || isReference(fileName) ||
isMobFileLink(fileName)) {
return true;
}
return !fileName.contains("-");
}
+ public static boolean isMobFileLink(String fileName) {
+ Matcher m = HFileLink.REF_OR_HFILE_LINK_PATTERN.matcher(fileName);
+ return m.matches() && !isReference(fileName);
+ }
+
/**
* Return if the specified file is a valid store file or not.
* @param fileStatus The {@link FileStatus} of the file
@@ -632,7 +680,7 @@ public class StoreFileInfo implements Configurable {
// Check for empty hfile. Should never be the case but can happen
// after data loss in hdfs for whatever reason (upgrade, etc.): HBASE-646
// NOTE: that the HFileLink is just a name, so it's an empty file.
- if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0) {
+ if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0 &&
!isMobFileLink(p.getName())) {
LOG.warn("Skipping {} because it is empty. HBASE-646 DATA LOSS?", p);
return false;
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java
index c9ee019e9af..778a419bb30 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
@@ -35,6 +36,7 @@ import org.apache.hadoop.hbase.ExtendedCell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.util.ChecksumType;
import org.apache.yetus.audience.InterfaceAudience;
@@ -183,6 +185,15 @@ public final class StoreUtils {
return
storefiles.stream().map(HStoreFile::getFileInfo).collect(Collectors.toList());
}
+ public static List<HStoreFile> toHStoreFile(List<StoreFileInfo>
storeFileInfoList,
+ BloomType bloomType, CacheConfig cacheConf) throws IOException {
+ List<HStoreFile> hStoreFiles = new ArrayList<HStoreFile>();
+ for (StoreFileInfo storeFileInfo : storeFileInfoList) {
+ hStoreFiles.add(new HStoreFile(storeFileInfo, bloomType, cacheConf));
+ }
+ return hStoreFiles;
+ }
+
public static long getTotalUncompressedBytes(List<HStoreFile> files) {
return files.stream()
.mapToLong(file -> getStorefileFieldSize(file,
StoreFileReader::getTotalUncompressedBytes))
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java
index b000d837d59..fae97b9dd93 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.regionserver.storefiletracker;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
@@ -29,13 +30,19 @@ import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.io.Reference;
import org.apache.hadoop.hbase.regionserver.StoreContext;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
+
import
org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileEntry;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileList;
@@ -107,8 +114,16 @@ class FileBasedStoreFileTracker extends
StoreFileTrackerBase {
}
private StoreFileEntry toStoreFileEntry(StoreFileInfo info) {
- return
StoreFileEntry.newBuilder().setName(info.getPath().getName()).setSize(info.getSize())
- .build();
+
org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileEntry.Builder
entryBuilder =
+
StoreFileEntry.newBuilder().setName(info.getPath().getName()).setSize(info.getSize());
+ if (info.isReference()) {
+ org.apache.hadoop.hbase.shaded.protobuf.generated.FSProtos.Reference
reference =
+
org.apache.hadoop.hbase.shaded.protobuf.generated.FSProtos.Reference.newBuilder()
+ .setSplitkey(ByteString.copyFrom(info.getReference().getSplitKey()))
+ .setRange(info.getReference().convert().getRange()).build();
+ entryBuilder.setReference(reference);
+ }
+ return entryBuilder.build();
}
@Override
@@ -119,7 +134,8 @@ class FileBasedStoreFileTracker extends
StoreFileTrackerBase {
builder.addStoreFile(toStoreFileEntry(info));
}
for (StoreFileInfo info : newFiles) {
- builder.addStoreFile(toStoreFileEntry(info));
+ if (!storefiles.containsKey(info.getPath().getName()))
+ builder.addStoreFile(toStoreFileEntry(info));
}
backedFile.update(builder);
if (LOG.isTraceEnabled()) {
@@ -176,4 +192,68 @@ class FileBasedStoreFileTracker extends
StoreFileTrackerBase {
}
}
}
+
+ @Override
+ public Reference readReference(Path p) throws IOException {
+ String fileName = p.getName();
+ StoreFileList list = backedFile.load(true);
+ for (StoreFileEntry entry : list.getStoreFileList()) {
+ if (entry.getName().equals(fileName)) {
+ if (entry.hasReference()) {
+ return Reference.convert(entry.getReference());
+ } else {
+ LOG.debug(
+ "Fallback to reading reference from FS as it is not part of
StoreFileEntry. This is when FSFT is reading older version of
StoreFileListFile");
+ return super.readReference(p);
+ }
+ }
+ }
+ throw new FileNotFoundException("Reference does not exist for path : " +
p);
+ }
+
+ @Override
+ public boolean hasReferences() throws IOException {
+ StoreFileList list = backedFile.load(true);
+ for (StoreFileEntry entry : list.getStoreFileList()) {
+ if (entry.hasReference() || HFileLink.isHFileLink(entry.getName())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public HFileLink createHFileLink(TableName linkedTable, String linkedRegion,
String hfileName,
+ boolean createBackRef) throws IOException {
+ FileSystem fs = ctx.getRegionFileSystem().getFileSystem();
+ HFileLink hfileLink = HFileLink.build(conf, linkedTable, linkedRegion,
+ ctx.getFamily().getNameAsString(), hfileName);
+ Path backRefPath = null;
+ if (createBackRef) {
+ Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
linkedTable, linkedRegion,
+ ctx.getFamily().getNameAsString());
+ Path backRefssDir = HFileLink.getBackReferencesDir(archiveStoreDir,
hfileName);
+ fs.mkdirs(backRefssDir);
+ // Create the reference for the link
+ String refName =
HFileLink.createBackReferenceName(ctx.getTableName().toString(),
+ ctx.getRegionInfo().getEncodedName());
+ backRefPath = new Path(backRefssDir, refName);
+ fs.createNewFile(backRefPath);
+ }
+ return hfileLink;
+ }
+
+ @Override
+ public Reference createReference(Reference reference, Path path) throws
IOException {
+ return reference;
+ }
+
+ @Override
+ public Reference createAndCommitReference(Reference reference, Path path)
throws IOException {
+ StoreFileInfo storeFileInfo =
+ new StoreFileInfo(ctx.getRegionFileSystem().getFileSystem().getConf(),
+ ctx.getRegionFileSystem().getFileSystem(), path, reference);
+ add(Collections.singleton(storeFileInfo));
+ return reference;
+ }
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java
index e107688b3d7..fffd4ab499e 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java
@@ -74,7 +74,7 @@ class StoreFileListFile {
private static final Logger LOG =
LoggerFactory.getLogger(StoreFileListFile.class);
// the current version for StoreFileList
- static final long VERSION = 1;
+ static final long VERSION = 2;
static final String TRACK_FILE_DIR = ".filelist";
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java
index 7e6980e854e..c56a40997c1 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.io.Reference;
import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
@@ -103,6 +104,8 @@ public interface StoreFileTracker {
Reference createReference(Reference reference, Path path) throws IOException;
+ Reference createAndCommitReference(Reference reference, Path path) throws
IOException;
+
/**
* Reads the reference file from the given path.
* @param path the {@link Path} to the reference file in the file system.
@@ -132,7 +135,20 @@ public interface StoreFileTracker {
* @return the file link name.
* @throws IOException on file or parent directory creation failure.
*/
- String createHFileLink(final TableName linkedTable, final String
linkedRegion,
+ HFileLink createHFileLink(final TableName linkedTable, final String
linkedRegion,
+ final String hfileName, final boolean createBackRef) throws IOException;
+
+ /**
+ * Create a new HFileLink and add to SFT
+ * <p>
+ * It also adds a back-reference to the hfile back-reference directory to
simplify the
+ * reference-count and the cleaning process.
+ * @param hfileLinkName - HFileLink name (it contains hfile-region-table)
+ * @param createBackRef - Whether back reference should be created. Defaults
to true.
+ * @return the file link name.
+ * @throws IOException on file or parent directory creation failure.
+ */
+ HFileLink createAndCommitHFileLink(final TableName linkedTable, final String
linkedRegion,
final String hfileName, final boolean createBackRef) throws IOException;
/**
@@ -145,7 +161,7 @@ public interface StoreFileTracker {
* @return the file link name.
* @throws IOException on file or parent directory creation failure.
*/
- String createFromHFileLink(final String hfileName, final boolean
createBackRef)
+ HFileLink createFromHFileLink(final String hfileName, final boolean
createBackRef)
throws IOException;
/**
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java
index 87eca7b93c9..843ec651fb9 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java
@@ -23,6 +23,7 @@ import java.io.BufferedInputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.regex.Matcher;
@@ -219,6 +220,11 @@ abstract class StoreFileTrackerBase implements
StoreFileTracker {
return reference;
}
+ @Override
+ public Reference createAndCommitReference(Reference reference, Path path)
throws IOException {
+ return createReference(reference, path);
+ }
+
/**
* Returns true if the specified family has reference files
* @param familyName Column Family Name
@@ -324,7 +330,18 @@ abstract class StoreFileTrackerBase implements
StoreFileTracker {
isPrimaryReplica);
}
- public String createHFileLink(final TableName linkedTable, final String
linkedRegion,
+ public HFileLink createAndCommitHFileLink(final TableName linkedTable, final
String linkedRegion,
+ final String hfileName, final boolean createBackRef) throws IOException {
+ HFileLink hFileLink = createHFileLink(linkedTable, linkedRegion,
hfileName, createBackRef);
+ Path path = new Path(ctx.getFamilyStoreDirectoryPath(),
+ HFileLink.createHFileLinkName(linkedTable, linkedRegion, hfileName));
+ StoreFileInfo storeFileInfo =
+ new StoreFileInfo(conf, this.ctx.getRegionFileSystem().getFileSystem(),
path, hFileLink);
+ add(Arrays.asList(storeFileInfo));
+ return hFileLink;
+ }
+
+ public HFileLink createHFileLink(final TableName linkedTable, final String
linkedRegion,
final String hfileName, final boolean createBackRef) throws IOException {
String name = HFileLink.createHFileLinkName(linkedTable, linkedRegion,
hfileName);
String refName =
HFileLink.createBackReferenceName(ctx.getTableName().toString(),
@@ -349,7 +366,8 @@ abstract class StoreFileTrackerBase implements
StoreFileTracker {
try {
// Create the link
if (fs.createNewFile(new Path(ctx.getFamilyStoreDirectoryPath(), name)))
{
- return name;
+ return new HFileLink(new Path(ctx.getFamilyStoreDirectoryPath(),
name), backRefPath, null,
+ archiveStoreDir);
}
} catch (IOException e) {
LOG.error("couldn't create the link=" + name + " for " +
ctx.getFamilyStoreDirectoryPath(),
@@ -365,14 +383,22 @@ abstract class StoreFileTrackerBase implements
StoreFileTracker {
}
- public String createFromHFileLink(final String hfileLinkName, final boolean
createBackRef)
+ public HFileLink createFromHFileLink(final String hfileLinkName, final
boolean createBackRef)
throws IOException {
- Matcher m = HFileLink.LINK_NAME_PATTERN.matcher(hfileLinkName);
- if (!m.matches()) {
- throw new IllegalArgumentException(hfileLinkName + " is not a valid
HFileLink name!");
+ Matcher hfileLinkMatcher =
HFileLink.LINK_NAME_PATTERN.matcher(hfileLinkName);
+ if (hfileLinkMatcher.matches()) {
+ return createHFileLink(
+ TableName.valueOf(hfileLinkMatcher.group(1),
hfileLinkMatcher.group(2)),
+ hfileLinkMatcher.group(3), hfileLinkMatcher.group(4), createBackRef);
+ }
+ if (StoreFileInfo.isMobFileLink(hfileLinkName)) {
+ Matcher mobLinkMatcher =
HFileLink.REF_OR_HFILE_LINK_PATTERN.matcher(hfileLinkName);
+ if (mobLinkMatcher.matches()) {
+ return createHFileLink(TableName.valueOf(mobLinkMatcher.group(1),
mobLinkMatcher.group(2)),
+ mobLinkMatcher.group(3), mobLinkMatcher.group(4), createBackRef);
+ }
}
- return createHFileLink(TableName.valueOf(m.group(1), m.group(2)),
m.group(3), m.group(4),
- createBackRef);
+ throw new IllegalArgumentException(hfileLinkName + " is not a valid
HFileLink name!");
}
@Override
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java
index 828f1974fca..a2410238bb3 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.procedure2.util.StringUtils;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.StoreContext;
@@ -118,7 +119,15 @@ public final class StoreFileTrackerFactory {
public static StoreFileTracker create(Configuration conf, boolean
isPrimaryReplica,
StoreContext ctx) {
- Class<? extends StoreFileTracker> tracker = getTrackerClass(conf);
+ Class<? extends StoreFileTracker> tracker;
+ if (
+ ctx != null && ctx.getRegionInfo().getEncodedName()
+ .equals(MobUtils.getMobRegionInfo(ctx.getTableName()).getEncodedName())
+ ) {
+ tracker = Trackers.DEFAULT.clazz;
+ } else {
+ tracker = getTrackerClass(conf);
+ }
LOG.debug("instantiating StoreFileTracker impl {}", tracker.getName());
return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx);
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
index 3f01432472d..f0f1ba3899a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
@@ -32,6 +32,7 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ThreadPoolExecutor;
+import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -39,6 +40,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
@@ -465,7 +467,8 @@ public class RestoreSnapshotHelper {
*/
private void restoreRegion(final RegionInfo regionInfo,
final SnapshotRegionManifest regionManifest) throws IOException {
- restoreRegion(regionInfo, regionManifest, new Path(tableDir,
regionInfo.getEncodedName()));
+ restoreRegion(regionInfo, regionManifest, new Path(tableDir,
regionInfo.getEncodedName()),
+ tableDir);
}
/**
@@ -478,7 +481,8 @@ public class RestoreSnapshotHelper {
return;
}
restoreRegion(regionInfo, regionManifest,
- MobUtils.getMobRegionPath(conf, tableDesc.getTableName()));
+ MobUtils.getMobRegionPath(conf, tableDesc.getTableName()),
+ MobUtils.getMobTableDir(conf, tableDesc.getTableName()));
}
/**
@@ -486,26 +490,27 @@ public class RestoreSnapshotHelper {
* snapshot.
*/
private void restoreRegion(final RegionInfo regionInfo,
- final SnapshotRegionManifest regionManifest, Path regionDir) throws
IOException {
+ final SnapshotRegionManifest regionManifest, Path regionDir, Path
tableDir) throws IOException {
Map<String, List<SnapshotRegionManifest.StoreFile>> snapshotFiles =
getRegionHFileReferences(regionManifest);
String tableName = tableDesc.getTableName().getNameAsString();
final String snapshotName = snapshotDesc.getName();
- Path regionPath = new Path(tableDir, regionInfo.getEncodedName());
- HRegionFileSystem regionFS = (fs.exists(regionPath))
+ HRegionFileSystem regionFS = (fs.exists(regionDir))
? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir,
regionInfo, false)
: HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir,
regionInfo);
// Restore families present in the table
for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) {
byte[] family = Bytes.toBytes(familyDir.getName());
-
+ ColumnFamilyDescriptor familyDescriptor =
ColumnFamilyDescriptorBuilder.of(family);
StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true,
-
StoreContext.getBuilder().withColumnFamilyDescriptor(tableDesc.getColumnFamily(family))
+ StoreContext.getBuilder().withColumnFamilyDescriptor(familyDescriptor)
.withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build());
- Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
+ List<StoreFileInfo> storeFileInfos = tracker.load();
+ List<String> familyFiles = storeFileInfos.stream()
+ .map(storeFileInfo ->
storeFileInfo.getPath().getName()).collect(Collectors.toList());
List<SnapshotRegionManifest.StoreFile> snapshotFamilyFiles =
snapshotFiles.remove(familyDir.getName());
List<StoreFileInfo> filesToTrack = new ArrayList<>();
@@ -526,11 +531,13 @@ public class RestoreSnapshotHelper {
// Remove hfiles not present in the snapshot
for (String hfileName : familyFiles) {
- Path hfile = new Path(familyDir, hfileName);
- if (!fs.getFileStatus(hfile).isDirectory()) {
- LOG.trace("Removing HFile=" + hfileName + " not present in
snapshot=" + snapshotName
- + " from region=" + regionInfo.getEncodedName() + " table=" +
tableName);
- HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir,
family, hfile);
+ for (StoreFileInfo storeFileInfo : storeFileInfos) {
+ if (hfileName.equals(storeFileInfo.getPath().getName())) {
+ tracker.removeStoreFiles(
+
StoreUtils.toHStoreFile(Collections.singletonList(storeFileInfo), null, null));
+ LOG.trace("Removing HFile=" + hfileName + " not present in
snapshot=" + snapshotName
+ + " from region=" + regionInfo.getEncodedName() + " table=" +
tableName);
+ }
}
}
@@ -538,15 +545,17 @@ public class RestoreSnapshotHelper {
for (SnapshotRegionManifest.StoreFile storeFile : hfilesToAdd) {
LOG.debug("Restoring missing HFileLink " + storeFile.getName() + "
of snapshot="
+ snapshotName + " to region=" + regionInfo.getEncodedName() + "
table=" + tableName);
- String fileName =
+ StoreFileInfo storeFileInfo =
restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs,
tracker);
// mark the reference file to be added to tracker
- filesToTrack.add(tracker.getStoreFileInfo(new Path(familyDir,
fileName), true));
+ filesToTrack.add(storeFileInfo);
}
} else {
// Family doesn't exists in the snapshot
LOG.trace("Removing family=" + Bytes.toString(family) + " in
snapshot=" + snapshotName
+ " from region=" + regionInfo.getEncodedName() + " table=" +
tableName);
+ LOG.debug("Removing family=" + Bytes.toString(family) + " in
snapshot=" + snapshotName
+ + " from region=" + regionInfo.getEncodedName() + " table=" +
tableName);
HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo,
familyDir, family);
fs.delete(familyDir, true);
}
@@ -571,29 +580,14 @@ public class RestoreSnapshotHelper {
for (SnapshotRegionManifest.StoreFile storeFile :
familyEntry.getValue()) {
LOG.trace("Adding HFileLink (Not present in the table) " +
storeFile.getName()
+ " of snapshot " + snapshotName + " to table=" + tableName);
- String fileName =
+ StoreFileInfo storeFileInfo =
restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs,
tracker);
- files.add(tracker.getStoreFileInfo(new Path(familyDir, fileName),
true));
+ files.add(storeFileInfo);
}
tracker.set(files);
}
}
- private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws
IOException {
- FileStatus[] hfiles = CommonFSUtils.listStatus(fs, familyDir);
- if (hfiles == null) {
- return Collections.emptySet();
- }
-
- Set<String> familyFiles = new HashSet<>(hfiles.length);
- for (int i = 0; i < hfiles.length; ++i) {
- String hfileName = hfiles[i].getPath().getName();
- familyFiles.add(hfileName);
- }
-
- return familyFiles;
- }
-
/**
* Clone specified regions. For each region create a new region and create a
HFileLink for each
* hfile.
@@ -661,8 +655,7 @@ public class RestoreSnapshotHelper {
for (SnapshotRegionManifest.FamilyFiles familyFiles :
manifest.getFamilyFilesList()) {
Path familyDir = new Path(regionDir,
familyFiles.getFamilyName().toStringUtf8());
List<StoreFileInfo> clonedFiles = new ArrayList<>();
- Path regionPath = new Path(tableDir, newRegionInfo.getEncodedName());
- HRegionFileSystem regionFS = (fs.exists(regionPath))
+ HRegionFileSystem regionFS = (fs.exists(regionDir))
? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir,
newRegionInfo, false)
: HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir,
newRegionInfo);
@@ -671,11 +664,14 @@ public class RestoreSnapshotHelper {
StoreFileTracker tracker =
StoreFileTrackerFactory
.create(sftConf, true,
- StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir)
+ StoreContext.getBuilder()
+ .withFamilyStoreDirectoryPath(
+ new Path(regionDir,
familyFiles.getFamilyName().toStringUtf8()))
.withRegionFileSystem(regionFS)
.withColumnFamilyDescriptor(
ColumnFamilyDescriptorBuilder.of(familyFiles.getFamilyName().toByteArray()))
.build());
+ tracker.load();
for (SnapshotRegionManifest.StoreFile storeFile :
familyFiles.getStoreFilesList()) {
LOG.info("Adding HFileLink " + storeFile.getName() + " from cloned
region " + "in snapshot "
+ snapshotName + " to table=" + tableName);
@@ -686,17 +682,16 @@ public class RestoreSnapshotHelper {
if (fs.exists(mobPath)) {
fs.delete(mobPath, true);
}
- restoreStoreFile(familyDir, snapshotRegionInfo, storeFile,
createBackRefs, tracker);
+ StoreFileInfo storeFileInfo =
+ restoreStoreFile(familyDir, snapshotRegionInfo, storeFile,
createBackRefs, tracker);
+ clonedFiles.add(storeFileInfo);
} else {
- String file =
+ StoreFileInfo storeFileInfo =
restoreStoreFile(familyDir, snapshotRegionInfo, storeFile,
createBackRefs, tracker);
- clonedFiles.add(tracker.getStoreFileInfo(new Path(familyDir, file),
true));
+ clonedFiles.add(storeFileInfo);
}
}
- // we don't need to track files under mobdir
- if (!MobUtils.isMobRegionInfo(newRegionInfo)) {
- tracker.set(clonedFiles);
- }
+ tracker.add(clonedFiles);
}
}
@@ -727,17 +722,23 @@ public class RestoreSnapshotHelper {
* @param createBackRef - Whether back reference should be created. Defaults
to true.
* @param storeFile store file name (can be a Reference, HFileLink or
simple HFile)
*/
- private String restoreStoreFile(final Path familyDir, final RegionInfo
regionInfo,
+ private StoreFileInfo restoreStoreFile(final Path familyDir, final
RegionInfo regionInfo,
final SnapshotRegionManifest.StoreFile storeFile, final boolean
createBackRef,
final StoreFileTracker tracker) throws IOException {
String hfileName = storeFile.getName();
- if (HFileLink.isHFileLink(hfileName)) {
- return tracker.createFromHFileLink(hfileName, createBackRef);
+ StoreFileInfo info = null;
+ if (HFileLink.isHFileLink(hfileName) ||
StoreFileInfo.isMobFileLink(hfileName)) {
+ HFileLink hfileLink = tracker.createFromHFileLink(hfileName,
createBackRef);
+ info = new StoreFileInfo(conf, fs, new Path(familyDir, hfileName),
hfileLink);
+ return info;
} else if (StoreFileInfo.isReference(hfileName)) {
return restoreReferenceFile(familyDir, regionInfo, storeFile, tracker);
} else {
- return tracker.createHFileLink(regionInfo.getTable(),
regionInfo.getEncodedName(), hfileName,
- createBackRef);
+ HFileLink hfileLink =
tracker.createAndCommitHFileLink(regionInfo.getTable(),
+ regionInfo.getEncodedName(), hfileName, createBackRef);
+ return new StoreFileInfo(conf, fs, new Path(familyDir, HFileLink
+ .createHFileLinkName(regionInfo.getTable(),
regionInfo.getEncodedName(), hfileName)),
+ hfileLink);
}
}
@@ -764,10 +765,11 @@ public class RestoreSnapshotHelper {
* @param regionInfo destination region info for the table
* @param storeFile reference file name
*/
- private String restoreReferenceFile(final Path familyDir, final RegionInfo
regionInfo,
+ private StoreFileInfo restoreReferenceFile(final Path familyDir, final
RegionInfo regionInfo,
final SnapshotRegionManifest.StoreFile storeFile, final StoreFileTracker
tracker)
throws IOException {
String hfileName = storeFile.getName();
+ StoreFileInfo storeFileInfo = null;
// Extract the referred information (hfile name and parent region)
Path refPath =
@@ -800,11 +802,15 @@ public class RestoreSnapshotHelper {
// Create the new reference
if (storeFile.hasReference()) {
Reference reference = Reference.convert(storeFile.getReference());
- tracker.createReference(reference, outPath);
+ tracker.createAndCommitReference(reference, outPath);
+ storeFileInfo = new StoreFileInfo(conf, fs, outPath, reference);
} else {
InputStream in;
if (linkPath != null) {
- in = HFileLink.buildFromHFileLinkPattern(conf, linkPath).open(fs);
+ HFileLink hfileLink = HFileLink.buildFromHFileLinkPattern(conf,
linkPath);
+ storeFileInfo = new StoreFileInfo(conf, fs, outPath, hfileLink);
+ tracker.add(Collections.singletonList(storeFileInfo));
+ in = hfileLink.open(fs);
} else {
linkPath = new Path(new Path(
HRegion.getRegionDir(snapshotManifest.getSnapshotDir(),
regionInfo.getEncodedName()),
@@ -832,7 +838,7 @@ public class RestoreSnapshotHelper {
daughters.setSecond(regionName);
}
}
- return outPath.getName();
+ return storeFileInfo;
}
/**
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/CloneSnapshotFromClientAfterSplittingRegionTestBase.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/CloneSnapshotFromClientAfterSplittingRegionTestBase.java
index 6d74233dd77..5027da6762f 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/CloneSnapshotFromClientAfterSplittingRegionTestBase.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/CloneSnapshotFromClientAfterSplittingRegionTestBase.java
@@ -68,7 +68,6 @@ public class
CloneSnapshotFromClientAfterSplittingRegionTestBase
// Take a snapshot
admin.snapshot(snapshotName2, tableName);
-
// Clone the snapshot to another table
admin.cloneSnapshot(snapshotName2, clonedTableName);
SnapshotTestingUtils.waitForTableToBeOnline(TEST_UTIL, clonedTableName);
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
index 73b0f527981..37b3de11bd4 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
@@ -429,7 +429,7 @@ public class TestPrefetch {
.withRegionFileSystem(regionFS).build());
HStoreFile file = new HStoreFile(fs, storeFile, conf, cacheConf,
BloomType.NONE, true, sft);
Path ref = regionFS.splitStoreFile(region, "cf", file,
fileWithSplitPoint.getSecond(), false,
- new ConstantSizeRegionSplitPolicy(), sft);
+ new ConstantSizeRegionSplitPolicy(), sft).getPath();
conf.setBoolean(HBASE_REGION_SERVER_ENABLE_COMPACTION, compactionEnabled);
HStoreFile refHsf = new HStoreFile(this.fs, ref, conf, cacheConf,
BloomType.NONE, true, sft);
refHsf.initReader();
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestPrefetchWithBucketCache.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestPrefetchWithBucketCache.java
index 8e341979a59..0513808938e 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestPrefetchWithBucketCache.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestPrefetchWithBucketCache.java
@@ -200,7 +200,7 @@ public class TestPrefetchWithBucketCache {
byte[] splitPoint = RandomKeyValueUtil.randomOrderedKey(rand, 50);
HStoreFile file = new HStoreFile(fs, storeFile, conf, cacheConf,
BloomType.NONE, true, sft);
Path ref = regionFS.splitStoreFile(region, "cf", file, splitPoint, false,
- new ConstantSizeRegionSplitPolicy(), sft);
+ new ConstantSizeRegionSplitPolicy(), sft).getPath();
HStoreFile refHsf = new HStoreFile(this.fs, ref, conf, cacheConf,
BloomType.NONE, true, sft);
// starts reader for the ref. The ref should resolve to the original file
blocks
// and not duplicate blocks in the cache.
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java
index 9c6c6f95f86..ffd12ba3423 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java
@@ -22,6 +22,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
+import java.util.Collections;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -33,8 +34,11 @@ import
org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.hbase.regionserver.StoreContext;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
import
org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -77,6 +81,8 @@ public class TestHFileLinkCleaner {
private Path linkBackRef;
private FileStatus[] backRefs;
private HFileCleaner cleaner;
+ private StoreFileTracker sft;
+ private HFileLink hfileLink;
private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
private static DirScanPool POOL;
private static final long TTL = 1000;
@@ -119,7 +125,7 @@ public class TestHFileLinkCleaner {
HRegionFileSystem regionFS = HRegionFileSystem.create(conf, fs,
CommonFSUtils.getTableDir(rootDir, tableLinkName), hriLink);
- StoreFileTracker sft = StoreFileTrackerFactory.create(conf, true,
+ sft = StoreFileTrackerFactory.create(conf, true,
StoreContext.getBuilder()
.withFamilyStoreDirectoryPath(new Path(regionFS.getRegionDir(),
familyName))
.withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.of(familyName))
@@ -136,8 +142,9 @@ public class TestHFileLinkCleaner {
// Create link to hfile
familyLinkPath = getFamilyDirPath(rootDir, tableLinkName,
hriLink.getEncodedName(), familyName);
fs.mkdirs(familyLinkPath);
- hfileLinkName =
+ hfileLink =
sft.createHFileLink(hri.getTable(), hri.getEncodedName(), hfileName,
createBackReference);
+ hfileLinkName = hfileName;
linkBackRefDir = HFileLink.getBackReferencesDir(archiveStoreDir,
hfileName);
assertTrue(fs.exists(linkBackRefDir));
backRefs = fs.listStatus(linkBackRefDir);
@@ -185,7 +192,12 @@ public class TestHFileLinkCleaner {
assertTrue(fs.exists(hfilePath));
// simulate after removing the reference in data directory, the Link
backref can be removed
- fs.delete(new Path(familyLinkPath, hfileLinkName), false);
+ Path linkPath = new Path(familyLinkPath,
+ HFileLink.createHFileLinkName(hri.getTable(), hri.getEncodedName(),
hfileName));
+ HStoreFile storeFile =
+ new HStoreFile(new StoreFileInfo(conf, fs, linkPath, hfileLink),
BloomType.NONE, null);
+ sft.removeStoreFiles(Collections.singletonList(storeFile));
+
cleaner.chore();
assertFalse(fs.exists(linkBackRef), "Link should be deleted");
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java
index 787dc5b46dc..44eaca9928f 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java
@@ -43,9 +43,13 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HStore;
+import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
import
org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.RegionSplitter;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -169,7 +173,13 @@ public class TestMobCompactionWithDefaults {
LOG.info("MOB compaction " + description() + " started");
loadAndFlushThreeTimes(rows, table, famStr);
mobCompact(tableDescriptor, familyDescriptor);
- assertEquals(numRegions * 4, getNumberOfMobFiles(table, famStr),
+ long filesAfterCompaction =
+ getNumberOfFilesInMobFamilyDir(tableDescriptor, familyDescriptor,
famStr);
+ LOG.info("Files after compaction: {}", filesAfterCompaction);
+
+ assertEquals(numRegions * 4,
+ getNumberOfFilesInMobFamilyDir(tableDescriptor,
+ tableDescriptor.getColumnFamily(famStr.getBytes()), famStr),
"Should have 4 MOB files per region due to 3xflush + compaction.");
cleanupAndVerifyCounts(table, famStr, 3 * rows);
LOG.info("MOB compaction " + description() + " finished OK");
@@ -183,15 +193,32 @@ public class TestMobCompactionWithDefaults {
LOG.debug("Taking snapshot and cloning table {}", table);
admin.snapshot(TestMobUtils.getTableName(testMethodName), table);
admin.cloneSnapshot(TestMobUtils.getTableName(testMethodName), clone);
- assertEquals(3 * numRegions, getNumberOfMobFiles(clone, famStr),
+ assertEquals(3 * numRegions,
+ getNumberOfStoreFiles(admin.getDescriptor(clone),
+ admin.getDescriptor(clone).getColumnFamily(famStr.getBytes()), famStr),
"Should have 3 hlinks per region in MOB area from snapshot clone");
- mobCompact(admin.getDescriptor(clone), familyDescriptor);
- assertEquals(4 * numRegions, getNumberOfMobFiles(clone, famStr),
+ mobCompact(admin.getDescriptor(clone),
+ admin.getDescriptor(clone).getColumnFamily(famStr.getBytes()));
+ assertEquals(4 * numRegions,
+ getNumberOfFilesInMobFamilyDir(admin.getDescriptor(clone),
+ admin.getDescriptor(clone).getColumnFamily(famStr.getBytes()), famStr),
"Should have 3 hlinks + 1 MOB file per region due to clone + compact");
cleanupAndVerifyCounts(clone, famStr, 3 * rows);
LOG.info("MOB compaction of cloned snapshot, " + description() + "
finished OK");
}
+ protected long getNumberOfFilesInMobFamilyDir(TableDescriptor descriptor,
+ ColumnFamilyDescriptor familyDesc, String family) throws IOException {
+ FileSystem fs = FileSystem.get(conf);
+ Path dir = MobUtils.getMobFamilyPath(conf, descriptor.getTableName(),
family);
+ FileStatus[] stat = fs.listStatus(dir);
+ for (FileStatus st : stat) {
+ LOG.debug("MOB Directory content: {}", st.getPath());
+ }
+ LOG.debug("MOB Directory content total files: {}", stat.length);
+ return stat.length;
+ }
+
@TestTemplate
public void testMobFileCompactionAfterSnapshotCloneAndFlush()
throws InterruptedException, IOException {
@@ -201,11 +228,17 @@ public class TestMobCompactionWithDefaults {
LOG.debug("Taking snapshot and cloning table {}", table);
admin.snapshot(TestMobUtils.getTableName(testMethodName), table);
admin.cloneSnapshot(TestMobUtils.getTableName(testMethodName), clone);
- assertEquals(3 * numRegions, getNumberOfMobFiles(clone, famStr),
+ assertEquals(3 * numRegions,
+ getNumberOfStoreFiles(admin.getDescriptor(clone),
+ admin.getDescriptor(clone).getColumnFamily(famStr.getBytes()), famStr),
"Should have 3 hlinks per region in MOB area from snapshot clone");
loadAndFlushThreeTimes(rows, clone, famStr);
- mobCompact(admin.getDescriptor(clone), familyDescriptor);
- assertEquals(7 * numRegions, getNumberOfMobFiles(clone, famStr),
+ mobCompact(admin.getDescriptor(clone),
+ admin.getDescriptor(clone).getColumnFamily(famStr.getBytes()));
+ CommonFSUtils.logFileSystemState(FileSystem.get(conf),
HTU.getDefaultRootDirPath(), LOG);
+ assertEquals(7 * numRegions,
+ getNumberOfFilesInMobFamilyDir(admin.getDescriptor(clone),
+ admin.getDescriptor(clone).getColumnFamily(famStr.getBytes()), famStr),
"Should have 7 MOB file per region due to clone + 3xflush + compact");
cleanupAndVerifyCounts(clone, famStr, 6 * rows);
LOG.info("MOB compaction of cloned snapshot w flush, " + description() + "
finished OK");
@@ -213,12 +246,15 @@ public class TestMobCompactionWithDefaults {
protected void loadAndFlushThreeTimes(int rows, TableName table, String
family)
throws IOException {
- final long start = getNumberOfMobFiles(table, family);
+ final long start =
getNumberOfFilesInMobFamilyDir(admin.getDescriptor(table),
+ admin.getDescriptor(table).getColumnFamily(family.getBytes()), family);
// Load and flush data 3 times
loadData(table, rows);
loadData(table, rows);
loadData(table, rows);
- assertEquals(start + numRegions * 3, getNumberOfMobFiles(table, family),
+ assertEquals(start + numRegions * 3,
+ getNumberOfFilesInMobFamilyDir(admin.getDescriptor(table),
+ admin.getDescriptor(table).getColumnFamily(family.getBytes()), family),
"Should have 3 more mob files per region from flushing.");
}
@@ -285,7 +321,9 @@ public class TestMobCompactionWithDefaults {
HTU.getMiniHBaseCluster().getRegionServer(sn).getRSMobFileCleanerChore().chore();
}
- assertEquals(numRegions, getNumberOfMobFiles(table, family),
+ assertEquals(numRegions,
+ getNumberOfFilesInMobFamilyDir(admin.getDescriptor(table),
+ admin.getDescriptor(table).getColumnFamily(family.getBytes()), family),
"After cleaning, we should have 1 MOB file per region based on size.");
LOG.debug("checking count of rows");
@@ -294,18 +332,6 @@ public class TestMobCompactionWithDefaults {
}
- protected long getNumberOfMobFiles(TableName tableName, String family)
throws IOException {
- FileSystem fs = FileSystem.get(conf);
- Path dir = MobUtils.getMobFamilyPath(conf, tableName, family);
- FileStatus[] stat = fs.listStatus(dir);
- for (FileStatus st : stat) {
- LOG.debug("MOB Directory content: {}", st.getPath());
- }
- LOG.debug("MOB Directory content total files: {}", stat.length);
-
- return stat.length;
- }
-
protected long scanTable(TableName tableName) {
try (final Table table = HTU.getConnection().getTable(tableName);
final ResultScanner scanner = table.getScanner(fam)) {
@@ -326,4 +352,17 @@ public class TestMobCompactionWithDefaults {
}
return 0;
}
+
+ protected long getNumberOfStoreFiles(TableDescriptor descriptor,
+ ColumnFamilyDescriptor familyDesc, String family) throws IOException {
+ List<HRegion> regions =
HTU.getHBaseCluster().getRegions(descriptor.getTableName());
+ long totalFiles = 0;
+ for (HRegion region : regions) {
+ HStore store = region.getStore(familyDesc.getName());
+ // This counts regular region files (with MOB references) via SFT
+ StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false,
store.getStoreContext());
+ totalFiles += sft.load().size();
+ }
+ return totalFiles;
+ }
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java
index cc9455641cb..2d7d4b92a2e 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java
@@ -91,7 +91,7 @@ public class TestDirectStoreSplitsMerges {
StoreFileTrackerFactory.create(TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(),
true, region.getStores().get(0).getStoreContext());
Path result = regionFS.splitStoreFile(daughterA,
Bytes.toString(FAMILY_NAME), file,
- Bytes.toBytes("002"), false, region.getSplitPolicy(), sft);
+ Bytes.toBytes("002"), false, region.getSplitPolicy(), sft).getPath();
// asserts the reference file naming is correct
validateResultingFile(region.getRegionInfo().getEncodedName(), result);
// Additionally check if split region dir was created directly under table
dir, not on .tmp
@@ -170,13 +170,13 @@ public class TestDirectStoreSplitsMerges {
Path splitDirA = regionFS.getSplitsDir(daughterA);
Path splitDirB = regionFS.getSplitsDir(daughterB);
HStoreFile file = (HStoreFile)
region.getStore(FAMILY_NAME).getStorefiles().toArray()[0];
- List<Path> filesA = new ArrayList<>();
+ List<StoreFileInfo> filesA = new ArrayList<>();
StoreFileTracker sft =
StoreFileTrackerFactory.create(TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(),
true, region.getStores().get(0).getStoreContext());
filesA.add(regionFS.splitStoreFile(daughterA, Bytes.toString(FAMILY_NAME),
file,
Bytes.toBytes("002"), false, region.getSplitPolicy(), sft));
- List<Path> filesB = new ArrayList<>();
+ List<StoreFileInfo> filesB = new ArrayList<>();
filesB.add(regionFS.splitStoreFile(daughterB, Bytes.toString(FAMILY_NAME),
file,
Bytes.toBytes("002"), true, region.getSplitPolicy(), sft));
MasterProcedureEnv env =
@@ -217,7 +217,7 @@ public class TestDirectStoreSplitsMerges {
true, first.getStore(FAMILY_NAME).getStoreContext()));
// merge file from second region
file = (HStoreFile)
second.getStore(FAMILY_NAME).getStorefiles().toArray()[0];
- List<Path> mergedFiles = new ArrayList<>();
+ List<StoreFileInfo> mergedFiles = new ArrayList<>();
mergedFiles.add(mergeFileFromRegion(mergeRegionFs, second, file,
StoreFileTrackerFactory
.create(configuration, true,
second.getStore(FAMILY_NAME).getStoreContext())));
MasterProcedureEnv env =
@@ -241,11 +241,11 @@ public class TestDirectStoreSplitsMerges {
}
}
- private Path mergeFileFromRegion(HRegionFileSystem regionFS, HRegion
regionToMerge,
+ private StoreFileInfo mergeFileFromRegion(HRegionFileSystem regionFS,
HRegion regionToMerge,
HStoreFile file, StoreFileTracker sft) throws IOException {
- Path mergedFile = regionFS.mergeStoreFile(regionToMerge.getRegionInfo(),
+ StoreFileInfo mergedFile =
regionFS.mergeStoreFile(regionToMerge.getRegionInfo(),
Bytes.toString(FAMILY_NAME), file, sft);
- validateResultingFile(regionToMerge.getRegionInfo().getEncodedName(),
mergedFile);
+ validateResultingFile(regionToMerge.getRegionInfo().getEncodedName(),
mergedFile.getPath());
return mergedFile;
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
index 3647a4e47ad..7b6262a1610 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
@@ -66,6 +66,9 @@ import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.regionserver.HRegion.FlushResultImpl;
import org.apache.hadoop.hbase.regionserver.HRegion.PrepareFlushResult;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequester;
+import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
+import
org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
import
org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -193,6 +196,10 @@ public class TestHRegionReplayEvents {
when(rss.getServerName()).thenReturn(ServerName.valueOf("foo", 1, 1));
when(rss.getConfiguration()).thenReturn(CONF);
when(rss.getRegionServerAccounting()).thenReturn(new
RegionServerAccounting(CONF));
+ when(rss.getRegionServerSpaceQuotaManager()).thenReturn(null); // or mock
it properly
+ when(rss.getFlushRequester()).thenReturn(mock(FlushRequester.class));
+
when(rss.getCompactionRequestor()).thenReturn(mock(CompactionRequester.class));
+ when(rss.getMetrics()).thenReturn(mock(MetricsRegionServer.class));
String string =
org.apache.hadoop.hbase.executor.EventType.RS_COMPACTED_FILES_DISCHARGER.toString();
ExecutorService es = new ExecutorService(string);
@@ -421,10 +428,12 @@ public class TestHRegionReplayEvents {
// assert that the compaction is applied
for (HStore store : secondaryRegion.getStores()) {
+ StoreFileTracker sft =
+ StoreFileTrackerFactory.create(CONF, false,
store.getStoreContext());
if (store.getColumnFamilyName().equals("cf1")) {
assertEquals(1, store.getStorefilesCount());
} else {
- assertEquals(expectedStoreFileCount, store.getStorefilesCount());
+ assertEquals(expectedStoreFileCount, sft.load().size());
}
}
} else {
@@ -1538,17 +1547,17 @@ public class TestHRegionReplayEvents {
// replay the bulk load event
secondaryRegion.replayWALBulkLoadEventMarker(bulkloadEvent);
- List<String> storeFileName = new ArrayList<>();
+ List<String> storeFileNames = new ArrayList<>();
for (StoreDescriptor storeDesc : bulkloadEvent.getStoresList()) {
- storeFileName.addAll(storeDesc.getStoreFileList());
+ storeFileNames.addAll(storeDesc.getStoreFileList());
}
// assert that the bulk loaded files are picked
for (HStore s : secondaryRegion.getStores()) {
for (HStoreFile sf : s.getStorefiles()) {
- storeFileName.remove(sf.getPath().getName());
+ storeFileNames.remove(sf.getPath().getName());
}
}
- assertTrue("Found some store file isn't loaded:" + storeFileName,
storeFileName.isEmpty());
+ assertTrue("Found some store file isn't loaded:" + storeFileNames,
storeFileNames.isEmpty());
LOG.info("-- Verifying edits from secondary");
for (byte[] family : families) {
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
index 69f549dd6b9..1580d1e690e 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
@@ -1171,12 +1171,12 @@ public class TestHStoreFile {
private Path splitStoreFile(final HRegionFileSystem regionFs, final
RegionInfo hri,
final String family, final HStoreFile sf, final byte[] splitKey, boolean
isTopRef,
StoreFileTracker sft) throws IOException {
- Path path = regionFs.splitStoreFile(hri, family, sf, splitKey, isTopRef,
null, sft);
- if (null == path) {
+ StoreFileInfo sfi = regionFs.splitStoreFile(hri, family, sf, splitKey,
isTopRef, null, sft);
+ if (null == sfi) {
return null;
}
- List<Path> splitFiles = new ArrayList<>();
- splitFiles.add(path);
+ List<StoreFileInfo> splitFiles = new ArrayList<>();
+ splitFiles.add(sfi);
MasterProcedureEnv mockEnv = mock(MasterProcedureEnv.class);
MasterServices mockServices = mock(MasterServices.class);
when(mockEnv.getMasterServices()).thenReturn(mockServices);
@@ -1187,7 +1187,7 @@ public class TestHStoreFile {
.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
when(mockTblDescs.get(any())).thenReturn(mockTblDesc);
Path regionDir = regionFs.commitDaughterRegion(hri, splitFiles, mockEnv);
- return new Path(new Path(regionDir, family), path.getName());
+ return new Path(new Path(regionDir, family), sfi.getPath().getName());
}
private StoreFileWriter writeStoreFile(Configuration conf, CacheConfig
cacheConf, Path path,
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java
index 47790dd20da..59f16f56dca 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java
@@ -122,7 +122,7 @@ public class TestMergesSplitsAddToTracker {
.setEndKey(region.getRegionInfo().getEndKey()).setSplit(false)
.setRegionId(region.getRegionInfo().getRegionId()).build();
HStoreFile file = (HStoreFile)
region.getStore(FAMILY_NAME).getStorefiles().toArray()[0];
- List<Path> splitFilesA = new ArrayList<>();
+ List<StoreFileInfo> splitFilesA = new ArrayList<>();
HRegionFileSystem regionFs = region.getRegionFileSystem();
StoreFileTracker sft =
StoreFileTrackerFactory.create(region.getBaseConf(), true,
StoreContext.getBuilder()
@@ -130,7 +130,7 @@ public class TestMergesSplitsAddToTracker {
.withRegionFileSystem(regionFs).build());
splitFilesA.add(regionFS.splitStoreFile(daughterA,
Bytes.toString(FAMILY_NAME), file,
Bytes.toBytes("002"), false, region.getSplitPolicy(), sft));
- List<Path> splitFilesB = new ArrayList<>();
+ List<StoreFileInfo> splitFilesB = new ArrayList<>();
splitFilesB.add(regionFS.splitStoreFile(daughterB,
Bytes.toString(FAMILY_NAME), file,
Bytes.toBytes("002"), true, region.getSplitPolicy(), sft));
MasterProcedureEnv env =
@@ -164,7 +164,7 @@ public class TestMergesSplitsAddToTracker {
TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(),
regionFS.getFileSystem(),
regionFS.getTableDir(), mergeResult);
- List<Path> mergedFiles = new ArrayList<>();
+ List<StoreFileInfo> mergedFiles = new ArrayList<>();
// merge file from first region
mergedFiles.add(mergeFileFromRegion(first, mergeFS));
// merge file from second region
@@ -256,7 +256,7 @@ public class TestMergesSplitsAddToTracker {
}
}
- private Path mergeFileFromRegion(HRegion regionToMerge, HRegionFileSystem
mergeFS)
+ private StoreFileInfo mergeFileFromRegion(HRegion regionToMerge,
HRegionFileSystem mergeFS)
throws IOException {
HStoreFile file = (HStoreFile)
regionToMerge.getStore(FAMILY_NAME).getStorefiles().toArray()[0];
HRegionFileSystem regionFs = regionToMerge.getRegionFileSystem();
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 5c81bd8cd0d..32f88ea8dc6 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -957,11 +957,12 @@ public class TestSplitTransactionOnCluster {
HRegionFileSystem hfs = region.getRegionFileSystem();
StoreFileTracker sft =
StoreFileTrackerFactory.create(TESTING_UTIL.getConfiguration(), true,
store.getStoreContext());
- Path referencePath = hfs.splitStoreFile(region.getRegionInfo(), "f",
- storefiles.iterator().next(), Bytes.toBytes("row1"), false,
region.getSplitPolicy(), sft);
- assertNull(referencePath);
- referencePath = hfs.splitStoreFile(region.getRegionInfo(), "i_f",
+ StoreFileInfo referenceStoreFileInfo =
hfs.splitStoreFile(region.getRegionInfo(), "f",
storefiles.iterator().next(), Bytes.toBytes("row1"), false,
region.getSplitPolicy(), sft);
+ assertNull(referenceStoreFileInfo);
+ Path referencePath =
+ hfs.splitStoreFile(region.getRegionInfo(), "i_f",
storefiles.iterator().next(),
+ Bytes.toBytes("row1"), false, region.getSplitPolicy(),
sft).getPath();
assertNotNull(referencePath);
} finally {
TESTING_UTIL.deleteTable(tableName);
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java
index 29040ad58be..530a39f7306 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java
@@ -93,10 +93,10 @@ public class TestStoreFileInfo {
HFileLink link2 = new HFileLink(new Path(origin, "f1"), new Path(tmp,
"f1"),
new Path(mob, "f1"), new Path(archive, "f1"));
- StoreFileInfo info1 =
- new StoreFileInfo(TEST_UTIL.getConfiguration(),
TEST_UTIL.getTestFileSystem(), null, link1);
- StoreFileInfo info2 =
- new StoreFileInfo(TEST_UTIL.getConfiguration(),
TEST_UTIL.getTestFileSystem(), null, link2);
+ StoreFileInfo info1 = new StoreFileInfo(TEST_UTIL.getConfiguration(),
+ TEST_UTIL.getTestFileSystem(), new Path(archive, "f1"), link1);
+ StoreFileInfo info2 = new StoreFileInfo(TEST_UTIL.getConfiguration(),
+ TEST_UTIL.getTestFileSystem(), new Path(archive, "f1"), link2);
assertEquals(info1, info2);
assertEquals(info1.hashCode(), info2.hashCode());
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileListFile.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileListFile.java
index 82278a9e350..64865caac1e 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileListFile.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileListFile.java
@@ -21,7 +21,10 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
@@ -32,8 +35,10 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseCommonTestingUtil;
+import org.apache.hadoop.hbase.io.Reference;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.StoreContext;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -256,4 +261,48 @@ public class TestStoreFileListFile {
StoreFileList list = storeFileListFile.load(true);
assertEquals(1, list.getStoreFileCount());
}
+
+ @Test
+ public void testReadReferenceFallbackForOldStoreFileEntry() throws
IOException {
+ FileSystem fs = FileSystem.get(UTIL.getConfiguration());
+ HRegionFileSystem hfs = mock(HRegionFileSystem.class);
+ when(hfs.getFileSystem()).thenReturn(fs);
+ StoreContext ctx =
StoreContext.getBuilder().withFamilyStoreDirectoryPath(testDir)
+ .withRegionFileSystem(hfs).build();
+ FileBasedStoreFileTracker tracker =
+ new FileBasedStoreFileTracker(UTIL.getConfiguration(), true, ctx);
+
+ String referenceFileName = "abcdef0123456789.parentRegion";
+ assertTrue(StoreFileInfo.isReference(referenceFileName));
+ Reference expected =
Reference.createTopReference(Bytes.toBytes("split-row"));
+ Path referenceFile = new Path(testDir, referenceFileName);
+ expected.write(fs, referenceFile);
+
+ // Simulate older StoreFileListFile where StoreFileEntry does not contain
reference field.
+ StoreFileListFile oldFormatStoreFileListFile = new StoreFileListFile(ctx);
+ oldFormatStoreFileListFile.update(StoreFileList.newBuilder()
+
.addStoreFile(StoreFileEntry.newBuilder().setName(referenceFileName).setSize(1)));
+
+ org.apache.logging.log4j.core.Appender mockAppender =
+ mock(org.apache.logging.log4j.core.Appender.class);
+ when(mockAppender.getName()).thenReturn("mockAppender");
+ when(mockAppender.isStarted()).thenReturn(true);
+ org.apache.logging.log4j.core.Logger logger =
+ (org.apache.logging.log4j.core.Logger)
org.apache.logging.log4j.LogManager
+ .getLogger(FileBasedStoreFileTracker.class);
+ org.apache.logging.log4j.Level oldLevel = logger.getLevel();
+ logger.setLevel(org.apache.logging.log4j.Level.DEBUG);
+ logger.addAppender(mockAppender);
+ try {
+ Reference actual = tracker.readReference(referenceFile);
+ assertEquals(expected, actual);
+ verify(mockAppender, atLeastOnce()).append(
+ argThat(event -> event.getMessage() != null &&
event.getMessage().getFormattedMessage()
+ .contains("Fallback to reading reference from FS as it is not part
of StoreFileEntry. "
+ + "This is when FSFT is reading older version of
StoreFileListFile")));
+ } finally {
+ logger.removeAppender(mockAppender);
+ logger.setLevel(oldLevel);
+ }
+ }
}
