This is an automated email from the ASF dual-hosted git repository. anmolnar pushed a commit to branch HBASE-28957 in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 822b197394d69b4cdd7861b41b1359770c1d2de8 Author: Andor Molnar <[email protected]> AuthorDate: Mon Mar 23 18:49:20 2026 -0500 HBASE-28957. Resolve build errors --- .idea/vcs.xml | 39 ---------------------- .../hadoop/hbase/backup/impl/BackupCommands.java | 6 ++-- .../backup/impl/DefaultPitrRestoreHandler.java | 4 ++- .../backup/impl/IncrementalTableBackupClient.java | 15 ++++----- .../hadoop/hbase/backup/util/BackupUtils.java | 16 +++++++++ .../apache/hadoop/hbase/backup/TestBackupBase.java | 2 +- ...estBackupDeleteWithContinuousBackupAndPITR.java | 3 +- .../hadoop/hbase/backup/TestBackupDescribe.java | 5 +-- .../hadoop/hbase/backup/TestBackupUtils.java | 16 +++++---- .../hbase/backup/impl/TestBackupAdminImpl.java | 14 ++++---- .../hbase/backup/impl/TestBackupCommands.java | 3 +- 11 files changed, 53 insertions(+), 70 deletions(-) diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index b2aa0751c81..00000000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,39 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<project version="4"> - <component name="VcsDirectoryMappings"> - <mapping directory="" vcs="Git" /> - </component> - <component name="IssueNavigationConfiguration"> - <option name="links"> - <list> - <IssueNavigationLink> - <option name="issueRegexp" value="HBASE\-\d+" /> - <option name="linkRegexp" value="https://issues.apache.org/jira/browse/$0"; /> - </IssueNavigationLink> - <IssueNavigationLink> - <option name="issueRegexp" value="#(\d+)"/> - <option name="linkRegexp" value="https://github.com/apache/hbase/pull/$1"/> - </IssueNavigationLink> - </list> - </option> - </component> -</project> diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java index dbbec5b342c..933c81333ec 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java @@ -803,7 +803,8 @@ public final class BackupCommands { } // Check if there is any other valid backup that can cover the PITR window - List<BackupInfo> allBackups = backupSystemTable.getBackupInfos(BackupState.COMPLETE); + List<BackupInfo> allBackups = backupSystemTable.getBackupHistory(withState( + BackupInfo.BackupState.COMPLETE)); boolean hasAnotherValidBackup = canAnyOtherBackupCover(allBackups, targetBackup, table, coveredPitrWindow.get(), continuousBackupStartTimes.get(table), maxAllowedPITRTime, currentTime); @@ -946,7 +947,8 @@ public final class BackupCommands { * @return cutoff timestamp or 0 if not found */ long determineWALCleanupCutoffTime(BackupSystemTable sysTable) throws IOException { - List<BackupInfo> backupInfos = sysTable.getBackupInfos(BackupState.COMPLETE); + List<BackupInfo> backupInfos = sysTable.getBackupHistory(withState( + BackupInfo.BackupState.COMPLETE)); Collections.reverse(backupInfos); // Start from oldest for (BackupInfo backupInfo : backupInfos) { diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/DefaultPitrRestoreHandler.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/DefaultPitrRestoreHandler.java index c6844ba96bd..d33e266503b 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/DefaultPitrRestoreHandler.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/DefaultPitrRestoreHandler.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.backup.BackupInfo; import org.apache.hadoop.hbase.backup.PointInTimeRestoreRequest; import org.apache.hadoop.hbase.client.Connection; import org.apache.yetus.audience.InterfaceAudience; +import static org.apache.hadoop.hbase.backup.BackupInfo.withState; /** * Default PITR restore handler that retrieves backup metadata from the system table. @@ -48,7 +49,8 @@ public class DefaultPitrRestoreHandler extends AbstractPitrRestoreHandler { protected List<PitrBackupMetadata> getBackupMetadata(PointInTimeRestoreRequest request) throws IOException { try (BackupSystemTable table = new BackupSystemTable(conn)) { - return table.getBackupInfos(BackupInfo.BackupState.COMPLETE).stream() + return table.getBackupHistory(withState( + BackupInfo.BackupState.COMPLETE)).stream() .map(BackupInfoAdapter::new).collect(Collectors.toList()); } } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java index 59418c86b81..2a5dab428dd 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.backup.impl; +import static org.apache.hadoop.hbase.backup.BackupInfo.withState; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONF_CONTINUOUS_BACKUP_WAL_DIR; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.JOB_NAME_CONF_KEY; @@ -381,16 +382,11 @@ public class IncrementalTableBackupClient extends TableBackupClient { // The table list in backupInfo is good for both full backup and incremental backup. // For incremental backup, it contains the incremental backup table set. backupManager.writeRegionServerLogTimestamp(backupInfo.getTables(), newTimestamps); - - Map<TableName, Map<String, Long>> newTableSetTimestampMap = - backupManager.readLogTimestampMap(); - - backupInfo.setTableSetTimestampMap(newTableSetTimestampMap); - Long newStartCode = - BackupUtils.getMinValue(BackupUtils.getRSLogTimestampMins(newTableSetTimestampMap)); - backupManager.writeBackupStartCode(newStartCode); } + Map<TableName, Map<String, Long>> newTableSetTimestampMap = + backupManager.readLogTimestampMap(); + List<BulkLoad> bulkLoads = handleBulkLoad(backupInfo.getTableNames(), tablesToWALFileList, tablesToPrevBackupTs); @@ -468,7 +464,8 @@ public class IncrementalTableBackupClient extends TableBackupClient { Path walBackupPath = new Path(walBackupDir); Set<TableName> tableSet = backupInfo.getTables(); currentBackupTs = backupInfo.getIncrCommittedWalTs(); - List<BackupInfo> backupInfos = backupManager.getBackupHistory(true); + List<BackupInfo> backupInfos = backupManager.getBackupHistory(withState( + BackupInfo.BackupState.COMPLETE)); for (TableName table : tableSet) { for (BackupInfo backup : backupInfos) { // find previous backup for this table diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java index b0665c23fa1..c765fd50474 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java @@ -342,6 +342,22 @@ public final class BackupUtils { } } + /** + * Get the min value for all the Values a map. + * @param map map + * @return the min value + */ + public static <T> Long getMinValue(Map<T, Long> map) { + Long minTimestamp = null; + if (map != null) { + ArrayList<Long> timestampList = new ArrayList<>(map.values()); + Collections.sort(timestampList); + // The min among all the RS log timestamps will be kept in backup system table table. + minTimestamp = timestampList.get(0); + } + return minTimestamp; + } + /** * Parses host name:port from archived WAL path * @param p path diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java index a82d28dc44a..16381325dbb 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java @@ -195,7 +195,7 @@ public class TestBackupBase { Map<TableName, Map<String, Long>> newTableSetTimestampMap = backupManager.readLogTimestampMap(); - handleBulkLoad(backupInfo.getTableNames()); + handleBulkLoad(backupInfo.getTableNames(), new HashMap<>(), new HashMap<>()); failStageIf(Stage.stage_4); // backup complete diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteWithContinuousBackupAndPITR.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteWithContinuousBackupAndPITR.java index 248e8e7b757..928dff1801c 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteWithContinuousBackupAndPITR.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteWithContinuousBackupAndPITR.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.backup; +import static org.apache.hadoop.hbase.backup.BackupInfo.withState; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONF_CONTINUOUS_BACKUP_PITR_WINDOW_DAYS; import static org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.ONE_DAY_IN_MILLISECONDS; import static org.junit.Assert.assertEquals; @@ -267,7 +268,7 @@ public class TestBackupDeleteWithContinuousBackupAndPITR extends TestBackupBase } private BackupInfo getBackupInfoById(String backupId) throws IOException { - return backupSystemTable.getBackupInfos(BackupInfo.BackupState.COMPLETE).stream() + return backupSystemTable.getBackupHistory(withState(BackupInfo.BackupState.COMPLETE)).stream() .filter(b -> b.getBackupId().equals(backupId)).findFirst() .orElseThrow(() -> new IllegalStateException("Backup should exist: " + backupId)); } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDescribe.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDescribe.java index 5d47135980b..c6453cca5cd 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDescribe.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDescribe.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONF_CONTINU import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_ENABLE_CONTINUOUS_BACKUP; import static org.apache.hadoop.hbase.replication.regionserver.ReplicationMarkerChore.REPLICATION_MARKER_ENABLED_DEFAULT; import static org.apache.hadoop.hbase.replication.regionserver.ReplicationMarkerChore.REPLICATION_MARKER_ENABLED_KEY; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -127,7 +128,7 @@ public class TestBackupDescribe extends TestBackupBase { String[] backupArgs = new String[] { "create", BackupType.FULL.name(), BACKUP_ROOT_DIR, "-t", table1.getNameAsString(), "-" + OPTION_ENABLE_CONTINUOUS_BACKUP }; int ret = ToolRunner.run(conf1, new BackupDriver(), backupArgs); - assertEquals("Backup should succeed", 0, ret); + assertEquals(0, ret, "Backup should succeed"); List<BackupInfo> backups = table.getBackupHistory(); String backupId = backups.get(0).getBackupId(); assertTrue(checkSucceeded(backupId)); @@ -163,7 +164,7 @@ public class TestBackupDescribe extends TestBackupBase { backupArgs = new String[] { "create", BackupType.INCREMENTAL.name(), BACKUP_ROOT_DIR, "-t", table1.getNameAsString() }; ret = ToolRunner.run(conf1, new BackupDriver(), backupArgs); - assertEquals("Incremental Backup should succeed", 0, ret); + assertEquals(0, ret, "Incremental Backup should succeed"); backups = table.getBackupHistory(); String incrBackupId = backups.get(0).getBackupId(); assertTrue(checkSucceeded(incrBackupId)); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupUtils.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupUtils.java index 08f15b37f68..397851fa716 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupUtils.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupUtils.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.backup; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -41,6 +42,7 @@ import org.apache.hadoop.hbase.util.Addressing; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.security.UserGroupInformation; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.slf4j.Logger; @@ -57,7 +59,7 @@ public class TestBackupUtils { private static FileSystem dummyFs; private static Path backupRootDir; - @BeforeClass + @BeforeAll public static void setUp() throws IOException { dummyFs = TEST_UTIL.getTestFileSystem(); backupRootDir = TEST_UTIL.getDataTestDirOnTestFS("backupUT"); @@ -196,21 +198,21 @@ public class TestBackupUtils { backupRootDir, startTime, endTime); // Verify the correct number of valid WAL dirs was found - assertEquals("The number of valid WAL dirs should be " + numExpectedValidWalDirs - + " for time zone " + timeZone, numExpectedValidWalDirs, validWalDirs.size()); + assertEquals(numExpectedValidWalDirs, validWalDirs.size(), + "The number of valid WAL dirs should be " + numExpectedValidWalDirs + + " for time zone " + timeZone); // Verify the list of valid WAL dirs is as expected for (String dirName : expectedValidWalDirs) { - assertTrue("Expected " + dirName + " to be a valid WAL dir", - validWalDirs.stream().anyMatch(path -> path.endsWith("/" + dirName))); + assertTrue(validWalDirs.stream().anyMatch(path -> path.endsWith("/" + dirName)), + "Expected " + dirName + " to be a valid WAL dir"); } // Verify the list of valid WAL dirs does not contain anything expected to be invalid List<String> expectedInvalidWalDirs = new ArrayList<>(availableWalDateDirs); expectedInvalidWalDirs.removeAll(expectedValidWalDirs); for (String dirName : expectedInvalidWalDirs) { - assertFalse("Expected " + dirName + " to NOT be a valid WAL dir", - validWalDirs.contains(dirName)); + assertFalse(validWalDirs.contains(dirName), "Expected " + dirName + " to NOT be a valid WAL dir"); } } } finally { diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/impl/TestBackupAdminImpl.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/impl/TestBackupAdminImpl.java index b78d44c144b..af9e91426a6 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/impl/TestBackupAdminImpl.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/impl/TestBackupAdminImpl.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.backup.impl; +import static org.apache.hadoop.hbase.backup.BackupInfo.withRoot; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -266,7 +267,7 @@ public class TestBackupAdminImpl { BackupInfo b3 = createBackupInfo("backup_003", 3000L, BackupType.INCREMENTAL, table); BackupInfo b4 = createBackupInfo("backup_004", 4000L, BackupType.INCREMENTAL, table); - when(mockTable.getBackupHistory("/backup/root")).thenReturn(List.of(b4, b3, b2, b1, b0)); + when(mockTable.getBackupHistory(withRoot("/backup/root"))).thenReturn(List.of(b4, b3, b2, b1, b0)); List<BackupInfo> result = backupAdminImpl.getAffectedBackupSessions(current, table, mockTable); @@ -293,7 +294,7 @@ public class TestBackupAdminImpl { BackupInfo b2 = createBackupInfo("backup_002", 2000L, BackupType.FULL, table); BackupInfo b3 = createBackupInfo("backup_003", 3000L, BackupType.INCREMENTAL, table); - when(mockTable.getBackupHistory("/backup/root")).thenReturn(List.of(b3, b2, b1, b0)); + when(mockTable.getBackupHistory(withRoot("/backup/root"))).thenReturn(List.of(b3, b2, b1, b0)); List<BackupInfo> result = backupAdminImpl.getAffectedBackupSessions(current, table, mockTable); @@ -320,7 +321,7 @@ public class TestBackupAdminImpl { TableName.valueOf("other_table")); BackupInfo b4 = createBackupInfo("backup_004", 4000L, BackupType.INCREMENTAL, table); - when(mockTable.getBackupHistory("/backup/root")).thenReturn(List.of(b4, b3, b2, b1, b0)); + when(mockTable.getBackupHistory(withRoot("/backup/root"))).thenReturn(List.of(b4, b3, b2, b1, b0)); List<BackupInfo> result = backupAdminImpl.getAffectedBackupSessions(current, table, mockTable); @@ -349,7 +350,7 @@ public class TestBackupAdminImpl { BackupInfo b3 = createBackupInfo("backup_003", 3000L, BackupType.INCREMENTAL, table); BackupInfo b4 = createBackupInfo("backup_004", 4000L, BackupType.INCREMENTAL, table); - when(mockTable.getBackupHistory("/backup/root")).thenReturn(List.of(b4, b3, b2, b1, b0)); + when(mockTable.getBackupHistory(withRoot("/backup/root"))).thenReturn(List.of(b4, b3, b2, b1, b0)); List<BackupInfo> result = backupAdminImpl.getAffectedBackupSessions(current, table, mockTable); @@ -633,7 +634,7 @@ public class TestBackupAdminImpl { BackupSystemTable table = mock(BackupSystemTable.class); when(table.readBackupInfo("b1")).thenReturn(b1); when(table.readBackupInfo("b2")).thenReturn(b2); - when(table.getBackupHistory(eq(-1), any(), any(), any(), any(), any())) + when(table.getBackupHistory(any())) .thenReturn(List.of(b1, b2)); new BackupAdminImpl(mock(Connection.class)).checkIfValidForMerge(ids, table); @@ -728,8 +729,7 @@ public class TestBackupAdminImpl { when(table.readBackupInfo("b2")).thenReturn(b2); when(table.readBackupInfo("b3")).thenReturn(b3); - when(table.getBackupHistory(eq(-1), any(), any(), any(), any(), any())) - .thenReturn(List.of(b1, b2, b3)); + when(table.getBackupHistory(any())).thenReturn(List.of(b1, b2, b3)); // Simulate a "hole" by omitting b2 from images String[] idsWithHole = { "b1", "b3" }; diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/impl/TestBackupCommands.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/impl/TestBackupCommands.java index 30ee495df55..fdcb637dbfe 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/impl/TestBackupCommands.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/impl/TestBackupCommands.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.backup.impl; +import static org.apache.hadoop.hbase.backup.BackupInfo.withState; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONF_CONTINUOUS_BACKUP_WAL_DIR; import static org.apache.hadoop.hbase.backup.TestBackupDeleteWithCleanup.logDirectoryStructure; import static org.apache.hadoop.hbase.backup.TestBackupDeleteWithCleanup.setupBackupFolders; @@ -85,7 +86,7 @@ public class TestBackupCommands extends TestBackupBase { // Ordered as newest to oldest, will be reversed in the method List<BackupInfo> backupInfos = List.of(full2, inc, full1); - when(sysTable.getBackupInfos(BackupInfo.BackupState.COMPLETE)) + when(sysTable.getBackupHistory(withState(BackupInfo.BackupState.COMPLETE))) .thenReturn(new ArrayList<>(backupInfos)); // WHEN
