This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 73484b641e OAK-11846 oak-segment-azure - recovery will recover from
.bak (#2452)
73484b641e is described below
commit 73484b641e9ca74d4587c7aadbb431536d2ef0ec
Author: Ieran Bogdan <[email protected]>
AuthorDate: Mon Aug 18 16:26:25 2025 +0300
OAK-11846 oak-segment-azure - recovery will recover from .bak (#2452)
* OAK-11846 oak-segment-azure - recovery will recover from .bak
---
.../oak/segment/azure/AzureArchiveManager.java | 15 +-
.../segment/azure/AzureSegmentArchiveReader.java | 6 +-
.../AzureArchiveManagerIgnoreSamePrefixTest.java | 211 +++++++++++++++++++++
3 files changed, 223 insertions(+), 9 deletions(-)
diff --git
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManager.java
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManager.java
index a2d6c4e908..061f9f3112 100644
---
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManager.java
+++
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManager.java
@@ -67,7 +67,7 @@ public class AzureArchiveManager implements
SegmentArchiveManager {
protected final IOMonitor ioMonitor;
protected final FileStoreMonitor monitor;
- private WriteAccessController writeAccessController;
+ private final WriteAccessController writeAccessController;
public AzureArchiveManager(BlobContainerClient readBlobContainerClient,
BlobContainerClient writeBlobContainerClient, String rootPrefix, IOMonitor
ioMonitor, FileStoreMonitor fileStoreMonitor, WriteAccessController
writeAccessController) {
this.readBlobContainerClient = readBlobContainerClient;
@@ -110,7 +110,7 @@ public class AzureArchiveManager implements
SegmentArchiveManager {
* @return true if the archive is empty (no 0000.* segment)
*/
private boolean isArchiveEmpty(String archiveName) throws
BlobStorageException {
- String fullBlobPrefix = String.format("%s/%s",
getDirectory(archiveName), "0000.");
+ String fullBlobPrefix = String.format("%s%s",
getDirectory(archiveName), "0000.");
ListBlobsOptions listBlobsOptions = new ListBlobsOptions();
listBlobsOptions.setPrefix(fullBlobPrefix);
return !readBlobContainerClient.listBlobs(listBlobsOptions,
null).iterator().hasNext();
@@ -119,7 +119,7 @@ public class AzureArchiveManager implements
SegmentArchiveManager {
@Override
public SegmentArchiveReader open(String archiveName) throws IOException {
try {
- String closedBlob = String.format("%s/%s",
getDirectory(archiveName), "closed");
+ String closedBlob = String.format("%s%s",
getDirectory(archiveName), "closed");
if (!readBlobContainerClient.getBlobClient(closedBlob).exists()) {
return null;
}
@@ -242,7 +242,7 @@ public class AzureArchiveManager implements
SegmentArchiveManager {
}
private void delete(String archiveName, Set<UUID> recoveredEntries) throws
IOException {
- getBlobs(archiveName + "/")
+ getBlobs(archiveName)
.forEach(blobItem -> {
if
(!recoveredEntries.contains(RemoteUtilities.getSegmentUUID(getName(blobItem))))
{
try {
@@ -265,8 +265,11 @@ public class AzureArchiveManager implements
SegmentArchiveManager {
delete(archiveName, recoveredEntries);
}
+ /**
+ * it must end with "/" otherwise we could overflow to other archives like
data00000a.tar.bak
+ */
protected String getDirectory(String archiveName) {
- return String.format("%s/%s", rootPrefix, archiveName);
+ return String.format("%s/%s/", rootPrefix, archiveName);
}
private List<BlobItem> getBlobs(String archiveName) throws IOException {
@@ -291,7 +294,7 @@ public class AzureArchiveManager implements
SegmentArchiveManager {
BlockBlobClient sourceBlobClient =
readBlobContainerClient.getBlobClient(blob.getName()).getBlockBlobClient();
- String destinationBlob = String.format("%s/%s", newParent,
AzureUtilities.getName(blob));
+ String destinationBlob = String.format("%s%s", newParent,
AzureUtilities.getName(blob));
BlockBlobClient destinationBlobClient =
writeBlobContainerClient.getBlobClient(destinationBlob).getBlockBlobClient();
PollResponse<BlobCopyInfo> response =
destinationBlobClient.beginCopy(sourceBlobClient.getBlobUrl(),
Duration.ofMillis(100)).waitForCompletion();
diff --git
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentArchiveReader.java
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentArchiveReader.java
index b5566f2f8c..5234a0bb7d 100644
---
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentArchiveReader.java
+++
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentArchiveReader.java
@@ -47,7 +47,7 @@ public class AzureSegmentArchiveReader extends
AbstractRemoteSegmentArchiveReade
super(ioMonitor);
this.blobContainerClient = blobContainerClient;
this.archiveName = archiveName;
- this.archivePath = String.format("%s/%s", rootPrefix, archiveName);
+ this.archivePath = String.format("%s/%s/", rootPrefix, archiveName);
this.length = computeArchiveIndexAndLength();
}
@@ -65,7 +65,7 @@ public class AzureSegmentArchiveReader extends
AbstractRemoteSegmentArchiveReade
protected long computeArchiveIndexAndLength() throws IOException {
long length = 0;
ListBlobsOptions listBlobsOptions = new ListBlobsOptions();
- listBlobsOptions.setPrefix(archivePath + "/");
+ listBlobsOptions.setPrefix(archivePath);
for (BlobItem blob : AzureUtilities.getBlobs(blobContainerClient,
listBlobsOptions)) {
Map<String, String> metadata = blob.getMetadata();
if (AzureBlobMetadata.isSegment(metadata)) {
@@ -95,7 +95,7 @@ public class AzureSegmentArchiveReader extends
AbstractRemoteSegmentArchiveReade
private BlockBlobClient getBlobClient(String name) throws IOException {
try {
- String fullName = String.format("%s/%s", archivePath, name);
+ String fullName = String.format("%s%s", archivePath, name);
return
blobContainerClient.getBlobClient(fullName).getBlockBlobClient();
} catch (BlobStorageException e) {
throw new IOException(e);
diff --git
a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerIgnoreSamePrefixTest.java
b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerIgnoreSamePrefixTest.java
new file mode 100644
index 0000000000..35cc9b0b8b
--- /dev/null
+++
b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerIgnoreSamePrefixTest.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.segment.azure;
+
+import com.azure.core.util.BinaryData;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.models.BlobStorageException;
+import com.azure.storage.blob.models.ListBlobsOptions;
+import org.apache.jackrabbit.oak.segment.remote.WriteAccessController;
+import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitorAdapter;
+import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter;
+import org.apache.jackrabbit.oak.segment.spi.monitor.RemoteStoreMonitorAdapter;
+import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager;
+import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveWriter;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.ClassRule;
+import org.junit.contrib.java.lang.system.ProvideSystemProperty;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.security.InvalidKeyException;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.UUID;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertEquals;
+
+public class AzureArchiveManagerIgnoreSamePrefixTest {
+
+ @ClassRule
+ public static AzuriteDockerRule azurite = new AzuriteDockerRule();
+
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
+ private BlobContainerClient readBlobContainerClient;
+ private BlobContainerClient writeBlobContainerClient;
+
+ private AzurePersistence azurePersistence;
+
+ private static final String rootPrefix = "oak";
+ private static final String segmentName =
"0004.44b4a246-50e0-470a-abe4-5a37a81c37c1";
+
+ @Before
+ public void setup() throws BlobStorageException, InvalidKeyException,
URISyntaxException, IOException {
+ readBlobContainerClient =
azurite.getReadBlobContainerClient("oak-test");
+ writeBlobContainerClient =
azurite.getWriteBlobContainerClient("oak-test");
+ BlobContainerClient noRetryBlobContainerClient =
azurite.getNoRetryBlobContainerClient("oak-test");
+
+ WriteAccessController writeAccessController = new
WriteAccessController();
+ writeAccessController.enableWriting();
+ azurePersistence = new AzurePersistence(readBlobContainerClient,
writeBlobContainerClient, noRetryBlobContainerClient, rootPrefix);
+ azurePersistence.setWriteAccessController(writeAccessController);
+ }
+
+ @Rule
+ public final ProvideSystemProperty systemPropertyRule = new
ProvideSystemProperty(AzureRepositoryLock.LEASE_DURATION_PROP, "15")
+ .and(AzureRepositoryLock.RENEWAL_INTERVAL_PROP, "3")
+ .and(AzureRepositoryLock.TIME_TO_WAIT_BEFORE_WRITE_BLOCK_PROP,
"9");
+
+ @Test
+ public void testRecoveryArchiveIgnoreArchiveSamePrefix() throws
BlobStorageException, IOException {
+ final String archiveName = "data00000a.tar";
+ final String bakArchiveName = archiveName + ".4.bak";
+
+ //create blob with same prefix as archiveName
+ writeBlobContainerClient.getBlobClient(rootPrefix + "/" +
bakArchiveName + "/" + segmentName)
+
.getBlockBlobClient().upload(BinaryData.fromString("test-data-segment-content"));
+
+ SegmentArchiveManager manager =
azurePersistence.createArchiveManager(false, false, new IOMonitorAdapter(), new
FileStoreMonitorAdapter(), new RemoteStoreMonitorAdapter());
+ SegmentArchiveWriter writer = manager.create(archiveName);
+
+ List<UUID> uuids = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ UUID u = UUID.randomUUID();
+ writer.writeSegment(u.getMostSignificantBits(),
u.getLeastSignificantBits(), new byte[10], 0, 10, 0, 0, false);
+ uuids.add(u);
+ }
+
+ writer.flush();
+ writer.close();
+
+ readBlobContainerClient.getBlobClient("oak/" + archiveName + "/0005."
+ uuids.get(5).toString()).delete();
+
+ LinkedHashMap<UUID, byte[]> recovered = new LinkedHashMap<>();
+ manager.recoverEntries(archiveName, recovered);
+ assertEquals(uuids.subList(0, 5), new ArrayList<>(recovered.keySet()));
+ }
+
+ @Test
+ public void testExistsArchiveIgnoreArchiveSamePrefix() {
+ final String archiveName = "data00001a.tar";
+ final String bakArchiveName = archiveName + ".4.bak";
+
+ writeBlobContainerClient.getBlobClient(rootPrefix + "/" +
bakArchiveName + "/" + segmentName)
+
.getBlockBlobClient().upload(BinaryData.fromString("test-data-segment-content"));
+
+ SegmentArchiveManager manager =
azurePersistence.createArchiveManager(false, false, new IOMonitorAdapter(), new
FileStoreMonitorAdapter(), new RemoteStoreMonitorAdapter());
+
+ assertFalse(manager.exists(archiveName));
+ }
+
+ @Test
+ public void testRenameToIgnoreBlobsSamePrefix() {
+ final String archiveName = "data00002a.tar";
+ final String bakArchiveName = archiveName + ".4.bak";
+ final String targetArchiveName = "data00003a.tar";
+
+ writeBlobContainerClient.getBlobClient(rootPrefix + "/" +
bakArchiveName + "/" + segmentName)
+
.getBlockBlobClient().upload(BinaryData.fromString("test-data-segment-content"));
+
+ SegmentArchiveManager manager =
azurePersistence.createArchiveManager(false, false, new IOMonitorAdapter(), new
FileStoreMonitorAdapter(), new RemoteStoreMonitorAdapter());
+ manager.renameTo(archiveName, targetArchiveName);
+
+ boolean blobExists = readBlobContainerClient.listBlobs(new
ListBlobsOptions().setPrefix(rootPrefix + "/" + targetArchiveName), null)
+ .iterator().hasNext();
+
+ assertFalse("blob from backup tar archive should not be renamed",
blobExists);
+ }
+
+ @Test
+ public void testCopyFileIgnoreOtherArchivesSamePrefix() throws IOException
{
+ final String archiveName = "data00003a.tar";
+ final String bakArchiveName = archiveName + ".4.bak";
+ final String targetArchiveName = "data00004a.tar";
+
+ writeBlobContainerClient.getBlobClient(rootPrefix + "/" +
bakArchiveName + "/" + segmentName)
+
.getBlockBlobClient().upload(BinaryData.fromString("test-data-segment-content"));
+
+ SegmentArchiveManager manager =
azurePersistence.createArchiveManager(false, false, new IOMonitorAdapter(), new
FileStoreMonitorAdapter(), new RemoteStoreMonitorAdapter());
+ manager.copyFile(archiveName, targetArchiveName);
+
+ boolean blobExistsInTargetArchive =
readBlobContainerClient.listBlobs(new ListBlobsOptions().setPrefix(rootPrefix +
"/" + targetArchiveName), null)
+ .iterator().hasNext();
+
+ assertFalse("blob from backup tar archive should not be copied",
blobExistsInTargetArchive);
+ }
+
+ @Test
+ public void testDeleteIgnoreOtherArchivesSamePrefix() {
+ final String archiveName = "data00004a.tar";
+ final String bakArchiveName = archiveName + ".4.bak";
+
+ writeBlobContainerClient.getBlobClient(rootPrefix + "/" +
bakArchiveName + "/" + segmentName)
+
.getBlockBlobClient().upload(BinaryData.fromString("test-data-segment-content"));
+ SegmentArchiveManager manager =
azurePersistence.createArchiveManager(false, false, new IOMonitorAdapter(), new
FileStoreMonitorAdapter(), new RemoteStoreMonitorAdapter());
+ manager.delete(archiveName);
+
+ boolean blobExists = readBlobContainerClient.listBlobs(new
ListBlobsOptions().setPrefix(rootPrefix + "/" + bakArchiveName + "/"), null)
+ .iterator().hasNext();
+
+ assertTrue("blob from backup tar archive should not be deleted",
blobExists);
+ }
+
+
+ @Test
+ public void testBackupWithRecoveredEntriesOverflow() throws
BlobStorageException, IOException {
+ final String archiveTestName = "data00005a.tar";
+ final String backupArchiveTestName = archiveTestName + ".bak";
+ final String extraBackupArchiveTestName = archiveTestName + ".4.bak";
+
+ writeBlobContainerClient.getBlobClient(rootPrefix + "/" +
extraBackupArchiveTestName + "/" + segmentName)
+ .getBlockBlobClient().getBlobOutputStream().close();
+
+ SegmentArchiveManager manager =
azurePersistence.createArchiveManager(false, false, new IOMonitorAdapter(), new
FileStoreMonitorAdapter(), new RemoteStoreMonitorAdapter());
+ SegmentArchiveWriter writer = manager.create(archiveTestName);
+
+ List<UUID> uuids = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ UUID u = UUID.randomUUID();
+ writer.writeSegment(u.getMostSignificantBits(),
u.getLeastSignificantBits(), new byte[10], 0, 10, 0, 0, false);
+ uuids.add(u);
+ }
+
+ writer.flush();
+ writer.close();
+
+ readBlobContainerClient.getBlobClient(rootPrefix + "/" +
archiveTestName + "/0005." + uuids.get(5).toString()).delete();
+
+ LinkedHashMap<UUID, byte[]> recovered = new LinkedHashMap<>();
+ manager.recoverEntries(archiveTestName, recovered);
+
+ manager.backup(archiveTestName, archiveTestName + ".bak",
recovered.keySet());
+
+ assertFalse("segment from extraBackupArchiveTestName should not be
copied to the new backup archive",
+ readBlobContainerClient.getBlobClient(rootPrefix + "/" +
backupArchiveTestName + "/" + segmentName).exists());
+ assertTrue("segment from extraBackupArchiveTestName should not be
cleaned",
+ readBlobContainerClient.getBlobClient(rootPrefix + "/" +
extraBackupArchiveTestName + "/" + segmentName).exists());
+ }
+}
