This is an automated email from the ASF dual-hosted git repository.
adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7c774d7b66 OAK-10898 - Allow AzureCheck and AzureCompact to be built
directly with a CloudBlobDirectory (#1540)
7c774d7b66 is described below
commit 7c774d7b66d767e52acd61121c0f0649fc958804
Author: Andrei Dulceanu <dulce...@users.noreply.github.com>
AuthorDate: Tue Jun 18 12:33:50 2024 +0200
OAK-10898 - Allow AzureCheck and AzureCompact to be built directly with a
CloudBlobDirectory (#1540)
---
.../oak/segment/azure/tool/AzureCheck.java | 27 +++++++++++++--
.../oak/segment/azure/tool/AzureCompact.java | 39 ++++++++++++++++++----
.../oak/segment/azure/tool/ToolUtils.java | 13 +++++---
3 files changed, 65 insertions(+), 14 deletions(-)
diff --git
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
index 2452e8b12b..8d0bca571e 100644
---
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
+++
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
@@ -17,6 +17,8 @@
package org.apache.jackrabbit.oak.segment.azure.tool;
import com.google.common.io.Files;
+import com.microsoft.azure.storage.blob.CloudBlobDirectory;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
import org.apache.jackrabbit.oak.segment.file.JournalReader;
import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
@@ -81,6 +83,8 @@ public class AzureCheck {
private Integer persistentCacheSizeGb;
+ private CloudBlobDirectory cloudBlobDirectory;
+
private Builder() {
// Prevent external instantiation.
}
@@ -270,6 +274,17 @@ public class AzureCheck {
return this;
}
+ /**
+ * The Azure blob directory to connect to.
+ * @param cloudBlobDirectory
+ * the Azure blob directory.
+ * @return this builder
+ */
+ public Builder withCloudBlobDirectory(CloudBlobDirectory
cloudBlobDirectory) {
+ this.cloudBlobDirectory = checkNotNull(cloudBlobDirectory);
+ return this;
+ }
+
/**
* Create an executable version of the {@link Check} command.
*
@@ -329,6 +344,8 @@ public class AzureCheck {
private final Integer persistentCacheSizeGb;
+ private final CloudBlobDirectory cloudBlobDirectory;
+
private AzureCheck(Builder builder) {
this.path = builder.path;
this.debugInterval = builder.debugInterval;
@@ -345,6 +362,7 @@ public class AzureCheck {
this.failFast = builder.failFast;
this.persistentCachePath = builder.persistentCachePath;
this.persistentCacheSizeGb = builder.persistentCacheSizeGb;
+ this.cloudBlobDirectory = builder.cloudBlobDirectory;
}
private static Integer revisionsToCheckCount(Integer revisionsCount) {
@@ -354,12 +372,17 @@ public class AzureCheck {
public int run() {
StatisticsIOMonitor ioMonitor = new StatisticsIOMonitor();
SegmentNodeStorePersistence persistence;
- if (persistentCachePath != null) {
- persistence =
ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE,
path, persistentCachePath, persistentCacheSizeGb);
+
+ if (cloudBlobDirectory != null) {
+ persistence = new AzurePersistence(cloudBlobDirectory);
} else {
persistence =
ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE,
path);
}
+ if (persistentCachePath != null) {
+ persistence = ToolUtils.decorateWithCache(persistence,
persistentCachePath, persistentCacheSizeGb);
+ }
+
FileStoreBuilder builder =
fileStoreBuilder(Files.createTempDir()).withCustomPersistence(persistence);
if (ioStatistics) {
diff --git
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
index d4942f63f1..ab18f4f282 100644
---
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
+++
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
@@ -19,11 +19,7 @@ package org.apache.jackrabbit.oak.segment.azure.tool;
import static
org.apache.jackrabbit.guava.common.base.Preconditions.checkArgument;
import static
org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull;
-import static
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.createArchiveManager;
-import static
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.createCloudBlobDirectory;
-import static
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.newFileStore;
-import static
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.newSegmentNodeStorePersistence;
-import static
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.printableStopwatch;
+import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.*;
import org.apache.jackrabbit.guava.common.base.Stopwatch;
import org.apache.jackrabbit.guava.common.io.Files;
@@ -35,6 +31,7 @@ import com.microsoft.azure.storage.blob.CloudBlobDirectory;
import com.microsoft.azure.storage.blob.ListBlobItem;
import org.apache.jackrabbit.oak.segment.SegmentCache;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.SegmentStoreType;
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.GCType;
import
org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType;
@@ -90,6 +87,10 @@ public class AzureCompact {
private Integer persistentCacheSizeGb;
+ private CloudBlobDirectory sourceCloudBlobDirectory;
+
+ private CloudBlobDirectory destinationCloudBlobDirectory;
+
private Builder() {
// Prevent external instantiation.
}
@@ -217,6 +218,16 @@ public class AzureCompact {
return this;
}
+ public Builder withSourceCloudBlobDirectory(CloudBlobDirectory
sourceCloudBlobDirectory) {
+ this.sourceCloudBlobDirectory =
checkNotNull(sourceCloudBlobDirectory);
+ return this;
+ }
+
+ public Builder withDestinationCloudBlobDirectory(CloudBlobDirectory
destinationCloudBlobDirectory) {
+ this.destinationCloudBlobDirectory =
checkNotNull(destinationCloudBlobDirectory);
+ return this;
+ }
+
/**
* Create an executable version of the {@link Compact} command.
*
@@ -248,6 +259,10 @@ public class AzureCompact {
private Integer persistentCacheSizeGb;
+ private CloudBlobDirectory sourceCloudBlobDirectory;
+
+ private CloudBlobDirectory destinationCloudBlobDirectory;
+
private AzureCompact(Builder builder) {
this.path = builder.path;
this.targetPath = builder.targetPath;
@@ -259,12 +274,22 @@ public class AzureCompact {
this.concurrency = builder.concurrency;
this.persistentCachePath = builder.persistentCachePath;
this.persistentCacheSizeGb = builder.persistentCacheSizeGb;
+ this.sourceCloudBlobDirectory = builder.sourceCloudBlobDirectory;
+ this.destinationCloudBlobDirectory =
builder.destinationCloudBlobDirectory;
}
public int run() throws IOException, StorageException, URISyntaxException {
Stopwatch watch = Stopwatch.createStarted();
- SegmentNodeStorePersistence roPersistence =
newSegmentNodeStorePersistence(SegmentStoreType.AZURE, path,
persistentCachePath, persistentCacheSizeGb);
- SegmentNodeStorePersistence rwPersistence =
newSegmentNodeStorePersistence(SegmentStoreType.AZURE, targetPath);
+
+ SegmentNodeStorePersistence roPersistence;
+ SegmentNodeStorePersistence rwPersistence;
+ if (sourceCloudBlobDirectory != null && destinationCloudBlobDirectory
!= null) {
+ roPersistence = decorateWithCache(new
AzurePersistence(sourceCloudBlobDirectory), persistentCachePath,
persistentCacheSizeGb);
+ rwPersistence = new
AzurePersistence(destinationCloudBlobDirectory);
+ } else {
+ roPersistence =
newSegmentNodeStorePersistence(SegmentStoreType.AZURE, path,
persistentCachePath, persistentCacheSizeGb);
+ rwPersistence =
newSegmentNodeStorePersistence(SegmentStoreType.AZURE, targetPath);
+ }
SegmentNodeStorePersistence splitPersistence = new
SplitPersistence(roPersistence, rwPersistence);
diff --git
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
index 4e73a79578..f1845bc0a6 100644
---
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
+++
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
@@ -134,11 +134,7 @@ public class ToolUtils {
switch (storeType) {
case AZURE:
CloudBlobDirectory cloudBlobDirectory =
createCloudBlobDirectory(pathOrUri.substring(3));
- SegmentNodeStorePersistence basePersistence = new
AzurePersistence(cloudBlobDirectory);
-
- PersistentCache persistentCache = new PersistentDiskCache(new
File(persistentCachePath),
- persistentCacheSize * 1024, new
DiskCacheIOMonitor(StatisticsProvider.NOOP));
- persistence = new CachingPersistence(persistentCache,
basePersistence);
+ persistence = decorateWithCache(new
AzurePersistence(cloudBlobDirectory), persistentCachePath, persistentCacheSize);
break;
default:
persistence = new TarPersistence(new File(pathOrUri));
@@ -147,6 +143,13 @@ public class ToolUtils {
return persistence;
}
+ public static SegmentNodeStorePersistence
decorateWithCache(SegmentNodeStorePersistence persistence,
+ String persistentCachePath, Integer persistentCacheSize) {
+ PersistentCache persistentCache = new PersistentDiskCache(new
File(persistentCachePath),
+ persistentCacheSize * 1024, new
DiskCacheIOMonitor(StatisticsProvider.NOOP));
+ return new CachingPersistence(persistentCache, persistence);
+ }
+
public static SegmentNodeStorePersistence
newSegmentNodeStorePersistence(SegmentStoreType storeType,
String pathOrUri) {
SegmentNodeStorePersistence persistence = null;
