(jackrabbit-oak) branch trunk updated: OAK-10978 - Skip Azure compaction when there's not enough garbage in the repository (#1606)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new c276352c10 OAK-10978 - Skip Azure compaction when there's not enough 
garbage in the repository (#1606)
c276352c10 is described below

commit c276352c105c3ce76c92b5403b1583077e36caaa
Author: Andrei Dulceanu 
AuthorDate: Mon Jul 29 16:24:27 2024 +0200

OAK-10978 - Skip Azure compaction when there's not enough garbage in the 
repository (#1606)
---
 .../apache/jackrabbit/oak/run/CompactCommand.java  | 18 ++
 .../oak/segment/azure/tool/AzureCompact.java   | 67 +-
 .../jackrabbit/oak/segment/file/FileStore.java |  2 +-
 3 files changed, 85 insertions(+), 2 deletions(-)

diff --git 
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java 
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
index ced75fc6be..0f8ee389fe 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
@@ -78,6 +78,16 @@ class CompactCommand implements Command {
 .withRequiredArg()
 .ofType(Integer.class)
 .defaultsTo(50);
+OptionSpec garbageThresholdGb = 
parser.accepts("garbage-threshold-gb", "Minimum amount of garbage in GB 
(defaults to 0 GB) for "
++ "compaction to run")
+.withRequiredArg()
+.ofType(Integer.class)
+.defaultsTo(0);
+OptionSpec garbageThresholdPercentage = 
parser.accepts("garbage-threshold-percentage", "Minimum amount of garbage in 
percentage (defaults to 0%) for "
++ "compaction to run")
+.withRequiredArg()
+.ofType(Integer.class)
+.defaultsTo(0);
 
 
 OptionSet options = parser.parse(args);
@@ -111,6 +121,14 @@ class CompactCommand implements Command {
 
azureBuilder.withPersistentCacheSizeGb(persistentCacheSizeGb.value(options));
 }
 
+if (options.has(garbageThresholdGb)) {
+
azureBuilder.withGarbageThresholdGb(garbageThresholdGb.value(options));
+}
+
+if (options.has(garbageThresholdPercentage)) {
+
azureBuilder.withGarbageThresholdPercentage(garbageThresholdPercentage.value(options));
+}
+
 if (options.has(tailArg)) {
 azureBuilder.withGCType(SegmentGCOptions.GCType.TAIL);
 }
diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
index a5db11950b..e11f30ca7b 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
@@ -37,6 +37,8 @@ import 
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.SegmentStoreType;
 import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.GCType;
 import 
org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType;
 import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.file.GCJournal;
+import org.apache.jackrabbit.oak.segment.spi.persistence.GCJournalFile;
 import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager;
 import 
org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
 import 
org.apache.jackrabbit.oak.segment.spi.persistence.split.SplitPersistence;
@@ -88,6 +90,10 @@ public class AzureCompact {
 
 private Integer persistentCacheSizeGb;
 
+private int garbageThresholdGb;
+
+private int garbageThresholdPercentage;
+
 private CloudBlobDirectory sourceCloudBlobDirectory;
 
 private CloudBlobDirectory destinationCloudBlobDirectory;
@@ -219,6 +225,29 @@ public class AzureCompact {
 return this;
 }
 
+/**
+ * The minimum garbage size in GB for the compaction to run.
+ * @param garbageThresholdGb
+ *   the minimum garbage size in GB for the compaction to run.
+ *
+ * @return this builder
+ */
+public Builder withGarbageThresholdGb(int garbageThresholdGb) {
+this.garbageThresholdGb = garbageThresholdGb;
+return this;
+}
+
+/**
+ * The minimum garbage size in percentage for the compaction to run.
+ * @param garbageThresholdPercentage
+ *  the minimum garbage size in percentage for the compaction 
to run.
+

(jackrabbit-oak) branch issues/OAK-10978 created (now f43f4d3c4a)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10978
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at f43f4d3c4a OAK-10978 - Skip Azure compaction when there's not enough 
garbage in the repository

This branch includes the following new commits:

 new f43f4d3c4a OAK-10978 - Skip Azure compaction when there's not enough 
garbage in the repository

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

(jackrabbit-oak) 01/01: OAK-10978 - Skip Azure compaction when there's not enough garbage in the repository

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-10978
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit f43f4d3c4a3253929b3f1b4d49e0c69b1bf0e83d
Author: Andrei Dulceanu 
AuthorDate: Mon Jul 29 12:00:33 2024 +0300

OAK-10978 - Skip Azure compaction when there's not enough garbage in the 
repository
---
 .../apache/jackrabbit/oak/run/CompactCommand.java  | 18 ++
 .../oak/segment/azure/tool/AzureCompact.java   | 67 +-
 .../jackrabbit/oak/segment/file/FileStore.java |  2 +-
 3 files changed, 85 insertions(+), 2 deletions(-)

diff --git 
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java 
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
index ced75fc6be..0f8ee389fe 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
@@ -78,6 +78,16 @@ class CompactCommand implements Command {
 .withRequiredArg()
 .ofType(Integer.class)
 .defaultsTo(50);
+OptionSpec garbageThresholdGb = 
parser.accepts("garbage-threshold-gb", "Minimum amount of garbage in GB 
(defaults to 0 GB) for "
++ "compaction to run")
+.withRequiredArg()
+.ofType(Integer.class)
+.defaultsTo(0);
+OptionSpec garbageThresholdPercentage = 
parser.accepts("garbage-threshold-percentage", "Minimum amount of garbage in 
percentage (defaults to 0%) for "
++ "compaction to run")
+.withRequiredArg()
+.ofType(Integer.class)
+.defaultsTo(0);
 
 
 OptionSet options = parser.parse(args);
@@ -111,6 +121,14 @@ class CompactCommand implements Command {
 
azureBuilder.withPersistentCacheSizeGb(persistentCacheSizeGb.value(options));
 }
 
+if (options.has(garbageThresholdGb)) {
+
azureBuilder.withGarbageThresholdGb(garbageThresholdGb.value(options));
+}
+
+if (options.has(garbageThresholdPercentage)) {
+
azureBuilder.withGarbageThresholdPercentage(garbageThresholdPercentage.value(options));
+}
+
 if (options.has(tailArg)) {
 azureBuilder.withGCType(SegmentGCOptions.GCType.TAIL);
 }
diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
index a5db11950b..e11f30ca7b 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
@@ -37,6 +37,8 @@ import 
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.SegmentStoreType;
 import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.GCType;
 import 
org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType;
 import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.file.GCJournal;
+import org.apache.jackrabbit.oak.segment.spi.persistence.GCJournalFile;
 import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager;
 import 
org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
 import 
org.apache.jackrabbit.oak.segment.spi.persistence.split.SplitPersistence;
@@ -88,6 +90,10 @@ public class AzureCompact {
 
 private Integer persistentCacheSizeGb;
 
+private int garbageThresholdGb;
+
+private int garbageThresholdPercentage;
+
 private CloudBlobDirectory sourceCloudBlobDirectory;
 
 private CloudBlobDirectory destinationCloudBlobDirectory;
@@ -219,6 +225,29 @@ public class AzureCompact {
 return this;
 }
 
+/**
+ * The minimum garbage size in GB for the compaction to run.
+ * @param garbageThresholdGb
+ *   the minimum garbage size in GB for the compaction to run.
+ *
+ * @return this builder
+ */
+public Builder withGarbageThresholdGb(int garbageThresholdGb) {
+this.garbageThresholdGb = garbageThresholdGb;
+return this;
+}
+
+/**
+ * The minimum garbage size in percentage for the compaction to run.
+ * @param garbageThresholdPercentage
+ *  the minimum garbage size in percentage for the compaction 
to run.
+ * @return this builder
+ */
+public Builder withGarbageThresholdPercentage(int 
garbageThresholdPercentage) {
+this.garbageThresholdPercentage = garbageThresholdPercentage;
+ 

(jackrabbit-oak) branch trunk updated (ffbd5daa29 -> a344090ca0)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


from ffbd5daa29 OAK-10976 - Avoid unnecessary call to PathUtils.getName in 
IndexDefinition (#1603)
 add a344090ca0 OAK-10904: Close token refresh executor service after 
access token is no longer needed (#1545)

No new revisions were added by this update.

Summary of changes:
 .../oak/run/cli/SegmentTarFixtureProvider.java |   5 +-
 .../explorer/AzureSegmentStoreExplorerBackend.java |  11 +-
 .../jackrabbit/oak/run/FileStoreDiffCommand.java   |  29 ++--
 .../jackrabbit/oak/run/DataStoreCommandTest.java   |   5 +-
 .../segment/azure/AzureSegmentStoreService.java|  12 +-
 .../azure/AzureStorageCredentialManager.java   | 147 +
 .../oak/segment/azure/AzureUtilities.java  |  94 -
 .../jackrabbit/oak/segment/azure/package-info.java |   2 +-
 .../oak/segment/azure/tool/AzureCheck.java |   7 +-
 .../oak/segment/azure/tool/AzureCompact.java   |  11 +-
 .../oak/segment/azure/tool/SegmentCopy.java|  13 +-
 .../oak/segment/azure/tool/ToolUtils.java  | 107 +--
 .../SegmentCopyAzureServicePrincipalToTarTest.java |   7 +-
 .../segment/azure/tool/SegmentCopyTestBase.java|   2 +-
 .../oak/segment/azure/tool/ToolUtilsTest.java  |  24 +++-
 .../oak/upgrade/cli/node/SegmentAzureFactory.java  |  20 ++-
 ...entAzureServicePrincipalNodeStoreContainer.java |   8 +-
 .../upgrade/cli/node/SegmentAzureFactoryTest.java  |  61 +
 18 files changed, 325 insertions(+), 240 deletions(-)
 create mode 100644 
oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureStorageCredentialManager.java

(jackrabbit-oak) branch trunk updated: OAK-10917 - Make persistent cache related arguments optional for Azure compaction (#1555)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 2b734cfba6 OAK-10917 - Make persistent cache related arguments 
optional for Azure compaction (#1555)
2b734cfba6 is described below

commit 2b734cfba6e8300e9846097c459398c65d75cdb0
Author: Andrei Dulceanu 
AuthorDate: Mon Jun 24 16:47:24 2024 +0200

OAK-10917 - Make persistent cache related arguments optional for Azure 
compaction (#1555)
---
 .../org/apache/jackrabbit/oak/run/CompactCommand.java| 14 ++
 .../jackrabbit/oak/segment/azure/tool/AzureCompact.java  | 16 ++--
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git 
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java 
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
index f955278931..ced75fc6be 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
@@ -99,24 +99,22 @@ class CompactCommand implements Command {
 System.exit(-1);
 }
 
-if (persistentCachePath.value(options) == null) {
-System.err.println("A path for the persistent disk cache needs 
to be specified");
-parser.printHelpOn(System.err);
-System.exit(-1);
-}
-
 AzureCompact.Builder azureBuilder = AzureCompact.builder()
 .withPath(path)
 .withTargetPath(targetPath.value(options))
-
.withPersistentCachePath(persistentCachePath.value(options))
-
.withPersistentCacheSizeGb(persistentCacheSizeGb.value(options))
 .withForce(options.has(forceArg))
 .withGCLogInterval(Long.getLong("compaction-progress-log", 
15))
 .withConcurrency(nThreads.value(options));
 
+if (options.has(persistentCachePath)) {
+
azureBuilder.withPersistentCachePath(persistentCachePath.value(options));
+
azureBuilder.withPersistentCacheSizeGb(persistentCacheSizeGb.value(options));
+}
+
 if (options.has(tailArg)) {
 azureBuilder.withGCType(SegmentGCOptions.GCType.TAIL);
 }
+
 if (options.has(compactor)) {
 
azureBuilder.withCompactorType(CompactorType.fromDescription(compactor.value(options)));
 }
diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
index 747a44fa66..3e79f11837 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
@@ -258,13 +258,13 @@ public class AzureCompact {
 
 private final int concurrency;
 
-private String persistentCachePath;
+private final String persistentCachePath;
 
-private Integer persistentCacheSizeGb;
+private final Integer persistentCacheSizeGb;
 
-private CloudBlobDirectory sourceCloudBlobDirectory;
+private final CloudBlobDirectory sourceCloudBlobDirectory;
 
-private CloudBlobDirectory destinationCloudBlobDirectory;
+private final CloudBlobDirectory destinationCloudBlobDirectory;
 
 private AzureCompact(Builder builder) {
 this.path = builder.path;
@@ -287,13 +287,17 @@ public class AzureCompact {
 SegmentNodeStorePersistence roPersistence;
 SegmentNodeStorePersistence rwPersistence;
 if (sourceCloudBlobDirectory != null && destinationCloudBlobDirectory 
!= null) {
-roPersistence = decorateWithCache(new 
AzurePersistence(sourceCloudBlobDirectory), persistentCachePath, 
persistentCacheSizeGb);
+roPersistence = new AzurePersistence(sourceCloudBlobDirectory);
 rwPersistence = new 
AzurePersistence(destinationCloudBlobDirectory);
 } else {
-roPersistence = 
newSegmentNodeStorePersistence(SegmentStoreType.AZURE, path, 
persistentCachePath, persistentCacheSizeGb);
+roPersistence = 
newSegmentNodeStorePersistence(SegmentStoreType.AZURE, path);
 rwPersistence = 
newSegmentNodeStorePersistence(SegmentStoreType.AZURE, targetPath);
 }
 
+if (persistentCachePath != null) {
+roPersistence = decorateWithCache(roPersistence, 
persistentCachePath, persistentCacheSizeGb);
+}
+
 SegmentNodeStorePersistence splitPersistence = new 
SplitPersistence(roPersistence, rwPersistence);
 
 SegmentArchiveManager roArchiveManager = 
createArchiveManager(roPersistence);

(jackrabbit-oak) 01/01: OAK-10917 - Make persistent cache related arguments optional for Azure compaction

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-10917
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit 262973e58288f843f2bd76d0d69a7223f1580f7e
Author: Andrei Dulceanu 
AuthorDate: Mon Jun 24 16:02:13 2024 +0200

OAK-10917 - Make persistent cache related arguments optional for Azure 
compaction
---
 .../org/apache/jackrabbit/oak/run/CompactCommand.java| 14 ++
 .../jackrabbit/oak/segment/azure/tool/AzureCompact.java  | 16 ++--
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git 
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java 
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
index f955278931..ced75fc6be 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
@@ -99,24 +99,22 @@ class CompactCommand implements Command {
 System.exit(-1);
 }
 
-if (persistentCachePath.value(options) == null) {
-System.err.println("A path for the persistent disk cache needs 
to be specified");
-parser.printHelpOn(System.err);
-System.exit(-1);
-}
-
 AzureCompact.Builder azureBuilder = AzureCompact.builder()
 .withPath(path)
 .withTargetPath(targetPath.value(options))
-
.withPersistentCachePath(persistentCachePath.value(options))
-
.withPersistentCacheSizeGb(persistentCacheSizeGb.value(options))
 .withForce(options.has(forceArg))
 .withGCLogInterval(Long.getLong("compaction-progress-log", 
15))
 .withConcurrency(nThreads.value(options));
 
+if (options.has(persistentCachePath)) {
+
azureBuilder.withPersistentCachePath(persistentCachePath.value(options));
+
azureBuilder.withPersistentCacheSizeGb(persistentCacheSizeGb.value(options));
+}
+
 if (options.has(tailArg)) {
 azureBuilder.withGCType(SegmentGCOptions.GCType.TAIL);
 }
+
 if (options.has(compactor)) {
 
azureBuilder.withCompactorType(CompactorType.fromDescription(compactor.value(options)));
 }
diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
index 747a44fa66..3e79f11837 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
@@ -258,13 +258,13 @@ public class AzureCompact {
 
 private final int concurrency;
 
-private String persistentCachePath;
+private final String persistentCachePath;
 
-private Integer persistentCacheSizeGb;
+private final Integer persistentCacheSizeGb;
 
-private CloudBlobDirectory sourceCloudBlobDirectory;
+private final CloudBlobDirectory sourceCloudBlobDirectory;
 
-private CloudBlobDirectory destinationCloudBlobDirectory;
+private final CloudBlobDirectory destinationCloudBlobDirectory;
 
 private AzureCompact(Builder builder) {
 this.path = builder.path;
@@ -287,13 +287,17 @@ public class AzureCompact {
 SegmentNodeStorePersistence roPersistence;
 SegmentNodeStorePersistence rwPersistence;
 if (sourceCloudBlobDirectory != null && destinationCloudBlobDirectory 
!= null) {
-roPersistence = decorateWithCache(new 
AzurePersistence(sourceCloudBlobDirectory), persistentCachePath, 
persistentCacheSizeGb);
+roPersistence = new AzurePersistence(sourceCloudBlobDirectory);
 rwPersistence = new 
AzurePersistence(destinationCloudBlobDirectory);
 } else {
-roPersistence = 
newSegmentNodeStorePersistence(SegmentStoreType.AZURE, path, 
persistentCachePath, persistentCacheSizeGb);
+roPersistence = 
newSegmentNodeStorePersistence(SegmentStoreType.AZURE, path);
 rwPersistence = 
newSegmentNodeStorePersistence(SegmentStoreType.AZURE, targetPath);
 }
 
+if (persistentCachePath != null) {
+roPersistence = decorateWithCache(roPersistence, 
persistentCachePath, persistentCacheSizeGb);
+}
+
 SegmentNodeStorePersistence splitPersistence = new 
SplitPersistence(roPersistence, rwPersistence);
 
 SegmentArchiveManager roArchiveManager = 
createArchiveManager(roPersistence);

(jackrabbit-oak) branch issues/OAK-10917 created (now 262973e582)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10917
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at 262973e582 OAK-10917 - Make persistent cache related arguments 
optional for Azure compaction

This branch includes the following new commits:

 new 262973e582 OAK-10917 - Make persistent cache related arguments 
optional for Azure compaction

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

(jackrabbit-oak) branch trunk updated (5fd5eb9f6d -> f0170604e3)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


from 5fd5eb9f6d OAK-10881: Expand oak/docs/participating with some more 
guidelines (#1544)
 add f0170604e3 OAK-10898 - Allow AzureCheck and AzureCompact to be built 
directly with a CloudBlobDirectory (#1550)

No new revisions were added by this update.

Summary of changes:
 .../jackrabbit/oak/segment/azure/tool/AzureCompact.java | 13 +
 1 file changed, 9 insertions(+), 4 deletions(-)

(jackrabbit-oak) 01/01: OAK-10898 - Allow AzureCheck and AzureCompact to be built directly with a CloudBlobDirectory Minor changes wrt. printing source and destination and the size of the target conta

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-10898-3
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit aca85cd8b54075721d754aa224ee9045dac02935
Author: Andrei Dulceanu 
AuthorDate: Thu Jun 20 16:04:37 2024 +0200

OAK-10898 - Allow AzureCheck and AzureCompact to be built directly with a 
CloudBlobDirectory
Minor changes wrt. printing source and destination and the size of the 
target container
---
 .../jackrabbit/oak/segment/azure/tool/AzureCompact.java | 13 +
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
index 5e84b21799..747a44fa66 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
@@ -299,8 +299,8 @@ public class AzureCompact {
 SegmentArchiveManager roArchiveManager = 
createArchiveManager(roPersistence);
 SegmentArchiveManager rwArchiveManager = 
createArchiveManager(rwPersistence);
 
-System.out.printf("Compacting %s\n", path);
-System.out.printf(" to %s\n", targetPath);
+System.out.printf("Compacting %s\n", path != null ? path : 
sourceCloudBlobDirectory.getUri().toString());
+System.out.printf(" to %s\n", targetPath != null ? targetPath : 
destinationCloudBlobDirectory.getUri().toString());
 System.out.printf("before\n");
 List beforeArchives = Collections.emptyList();
 try {
@@ -348,8 +348,13 @@ public class AzureCompact {
 printArchives(System.out, afterArchives);
 System.out.printf("Compaction succeeded in %s.\n", 
printableStopwatch(watch));
 
-CloudBlobDirectory targetDirectory = 
createCloudBlobDirectory(targetPath.substring(3));
-CloudBlobContainer targetContainer = targetDirectory.getContainer();
+CloudBlobContainer targetContainer = null;
+if (targetPath != null) {
+CloudBlobDirectory targetDirectory = 
createCloudBlobDirectory(targetPath.substring(3));
+targetContainer = targetDirectory.getContainer();
+} else {
+targetContainer = destinationCloudBlobDirectory.getContainer();
+}
 printTargetRepoSizeInfo(targetContainer);
 
 return 0;

(jackrabbit-oak) branch issues/OAK-10898-3 created (now aca85cd8b5)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10898-3
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at aca85cd8b5 OAK-10898 - Allow AzureCheck and AzureCompact to be built 
directly with a CloudBlobDirectory Minor changes wrt. printing source and 
destination and the size of the target container

This branch includes the following new commits:

 new aca85cd8b5 OAK-10898 - Allow AzureCheck and AzureCompact to be built 
directly with a CloudBlobDirectory Minor changes wrt. printing source and 
destination and the size of the target container

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

(jackrabbit-oak) branch trunk updated: OAK-10898 - Allow AzureCheck and AzureCompact to be built directly with a CloudBlobDirectory (#1549)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 6110406d65 OAK-10898 - Allow AzureCheck and AzureCompact to be built 
directly with a CloudBlobDirectory (#1549)
6110406d65 is described below

commit 6110406d65566a3d04318fcafbacce40be7488ff
Author: Andrei Dulceanu 
AuthorDate: Thu Jun 20 12:16:13 2024 +0200

OAK-10898 - Allow AzureCheck and AzureCompact to be built directly with a 
CloudBlobDirectory (#1549)

Allow paths to be null when CloudBlobDirectories are specified
---
 .../org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java | 4 +++-
 .../org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java   | 5 -
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
index 8d0bca571e..f484515574 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
@@ -291,7 +291,9 @@ public class AzureCheck {
  * @return an instance of {@link Runnable}.
  */
 public AzureCheck build() {
-checkNotNull(path);
+if (cloudBlobDirectory == null) {
+checkNotNull(path);
+}
 return new AzureCheck(this);
 }
 
diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
index ab18f4f282..5e84b21799 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
@@ -234,7 +234,10 @@ public class AzureCompact {
  * @return an instance of {@link Runnable}.
  */
 public AzureCompact build() {
-checkNotNull(path);
+if (sourceCloudBlobDirectory == null || 
destinationCloudBlobDirectory == null) {
+checkNotNull(path);
+checkNotNull(targetPath);
+}
 return new AzureCompact(this);
 }
 }

(jackrabbit-oak) 01/01: OAK-10898 - Allow AzureCheck and AzureCompact to be built directly with a CloudBlobDirectory Allow paths to be null when CloudBlobDirectories are specified

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-10898-2
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit 616b448439f1a6c2dcd5f1aac97540e4916ca1da
Author: Andrei Dulceanu 
AuthorDate: Thu Jun 20 12:13:47 2024 +0200

OAK-10898 - Allow AzureCheck and AzureCompact to be built directly with a 
CloudBlobDirectory
Allow paths to be null when CloudBlobDirectories are specified
---
 .../org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java | 4 +++-
 .../org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java   | 5 -
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
index 8d0bca571e..f484515574 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
@@ -291,7 +291,9 @@ public class AzureCheck {
  * @return an instance of {@link Runnable}.
  */
 public AzureCheck build() {
-checkNotNull(path);
+if (cloudBlobDirectory == null) {
+checkNotNull(path);
+}
 return new AzureCheck(this);
 }
 
diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
index ab18f4f282..5e84b21799 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
@@ -234,7 +234,10 @@ public class AzureCompact {
  * @return an instance of {@link Runnable}.
  */
 public AzureCompact build() {
-checkNotNull(path);
+if (sourceCloudBlobDirectory == null || 
destinationCloudBlobDirectory == null) {
+checkNotNull(path);
+checkNotNull(targetPath);
+}
 return new AzureCompact(this);
 }
 }

(jackrabbit-oak) branch issues/OAK-10898-2 created (now 616b448439)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10898-2
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at 616b448439 OAK-10898 - Allow AzureCheck and AzureCompact to be built 
directly with a CloudBlobDirectory Allow paths to be null when 
CloudBlobDirectories are specified

This branch includes the following new commits:

 new 616b448439 OAK-10898 - Allow AzureCheck and AzureCompact to be built 
directly with a CloudBlobDirectory Allow paths to be null when 
CloudBlobDirectories are specified

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

(jackrabbit-oak) branch trunk updated: OAK-10898 - Allow AzureCheck and AzureCompact to be built directly with a CloudBlobDirectory (#1540)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 7c774d7b66 OAK-10898 - Allow AzureCheck and AzureCompact to be built 
directly with a CloudBlobDirectory (#1540)
7c774d7b66 is described below

commit 7c774d7b66d767e52acd61121c0f0649fc958804
Author: Andrei Dulceanu 
AuthorDate: Tue Jun 18 12:33:50 2024 +0200

OAK-10898 - Allow AzureCheck and AzureCompact to be built directly with a 
CloudBlobDirectory (#1540)
---
 .../oak/segment/azure/tool/AzureCheck.java | 27 +--
 .../oak/segment/azure/tool/AzureCompact.java   | 39 ++
 .../oak/segment/azure/tool/ToolUtils.java  | 13 +---
 3 files changed, 65 insertions(+), 14 deletions(-)

diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
index 2452e8b12b..8d0bca571e 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
@@ -17,6 +17,8 @@
 package org.apache.jackrabbit.oak.segment.azure.tool;
 
 import com.google.common.io.Files;
+import com.microsoft.azure.storage.blob.CloudBlobDirectory;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
 import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
 import org.apache.jackrabbit.oak.segment.file.JournalReader;
 import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
@@ -81,6 +83,8 @@ public class AzureCheck {
 
 private Integer persistentCacheSizeGb;
 
+private CloudBlobDirectory cloudBlobDirectory;
+
 private Builder() {
 // Prevent external instantiation.
 }
@@ -270,6 +274,17 @@ public class AzureCheck {
 return this;
 }
 
+/**
+ * The Azure blob directory to connect to.
+ * @param cloudBlobDirectory
+ *  the Azure blob directory.
+ * @return this builder
+ */
+public Builder withCloudBlobDirectory(CloudBlobDirectory 
cloudBlobDirectory) {
+this.cloudBlobDirectory = checkNotNull(cloudBlobDirectory);
+return this;
+}
+
 /**
  * Create an executable version of the {@link Check} command.
  *
@@ -329,6 +344,8 @@ public class AzureCheck {
 
 private final Integer persistentCacheSizeGb;
 
+private final CloudBlobDirectory cloudBlobDirectory;
+
 private AzureCheck(Builder builder) {
 this.path = builder.path;
 this.debugInterval = builder.debugInterval;
@@ -345,6 +362,7 @@ public class AzureCheck {
 this.failFast = builder.failFast;
 this.persistentCachePath = builder.persistentCachePath;
 this.persistentCacheSizeGb = builder.persistentCacheSizeGb;
+this.cloudBlobDirectory = builder.cloudBlobDirectory;
 }
 
 private static Integer revisionsToCheckCount(Integer revisionsCount) {
@@ -354,12 +372,17 @@ public class AzureCheck {
 public int run() {
 StatisticsIOMonitor ioMonitor = new StatisticsIOMonitor();
 SegmentNodeStorePersistence persistence;
-if (persistentCachePath != null) {
-persistence = 
ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, 
path, persistentCachePath, persistentCacheSizeGb);
+
+if (cloudBlobDirectory != null) {
+persistence = new AzurePersistence(cloudBlobDirectory);
 } else {
 persistence = 
ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, 
path);
 }
 
+if (persistentCachePath != null) {
+persistence = ToolUtils.decorateWithCache(persistence, 
persistentCachePath, persistentCacheSizeGb);
+}
+
 FileStoreBuilder builder = 
fileStoreBuilder(Files.createTempDir()).withCustomPersistence(persistence);
 
 if (ioStatistics) {
diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
index d4942f63f1..ab18f4f282 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
@@ -19,11 +19,7 @@ package org.apache.jackrabbit.oak.segment.azure.tool;
 
 import static 
org.apache.jackrabbit.guava.common.base.Preconditions.checkArgument;
 import static 
org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull;
-import static

(jackrabbit-oak) branch issues/OAK-10898 created (now eb73debd55)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10898
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at eb73debd55 OAK-10898 - Allow AzureCheck and AzureCompact to be built 
directly with a CloudBlobDirectory

This branch includes the following new commits:

 new eb73debd55 OAK-10898 - Allow AzureCheck and AzureCompact to be built 
directly with a CloudBlobDirectory

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

(jackrabbit-oak) 01/01: OAK-10898 - Allow AzureCheck and AzureCompact to be built directly with a CloudBlobDirectory

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-10898
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit eb73debd5539f0cccfe6541fb674b1af74d7cb08
Author: Andrei Dulceanu 
AuthorDate: Mon Jun 17 14:50:43 2024 +0200

OAK-10898 - Allow AzureCheck and AzureCompact to be built directly with a 
CloudBlobDirectory
---
 .../oak/segment/azure/tool/AzureCheck.java | 27 +--
 .../oak/segment/azure/tool/AzureCompact.java   | 39 ++
 .../oak/segment/azure/tool/ToolUtils.java  | 13 +---
 3 files changed, 65 insertions(+), 14 deletions(-)

diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
index 2452e8b12b..8d0bca571e 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
@@ -17,6 +17,8 @@
 package org.apache.jackrabbit.oak.segment.azure.tool;
 
 import com.google.common.io.Files;
+import com.microsoft.azure.storage.blob.CloudBlobDirectory;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
 import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
 import org.apache.jackrabbit.oak.segment.file.JournalReader;
 import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
@@ -81,6 +83,8 @@ public class AzureCheck {
 
 private Integer persistentCacheSizeGb;
 
+private CloudBlobDirectory cloudBlobDirectory;
+
 private Builder() {
 // Prevent external instantiation.
 }
@@ -270,6 +274,17 @@ public class AzureCheck {
 return this;
 }
 
+/**
+ * The Azure blob directory to connect to.
+ * @param cloudBlobDirectory
+ *  the Azure blob directory.
+ * @return this builder
+ */
+public Builder withCloudBlobDirectory(CloudBlobDirectory 
cloudBlobDirectory) {
+this.cloudBlobDirectory = checkNotNull(cloudBlobDirectory);
+return this;
+}
+
 /**
  * Create an executable version of the {@link Check} command.
  *
@@ -329,6 +344,8 @@ public class AzureCheck {
 
 private final Integer persistentCacheSizeGb;
 
+private final CloudBlobDirectory cloudBlobDirectory;
+
 private AzureCheck(Builder builder) {
 this.path = builder.path;
 this.debugInterval = builder.debugInterval;
@@ -345,6 +362,7 @@ public class AzureCheck {
 this.failFast = builder.failFast;
 this.persistentCachePath = builder.persistentCachePath;
 this.persistentCacheSizeGb = builder.persistentCacheSizeGb;
+this.cloudBlobDirectory = builder.cloudBlobDirectory;
 }
 
 private static Integer revisionsToCheckCount(Integer revisionsCount) {
@@ -354,12 +372,17 @@ public class AzureCheck {
 public int run() {
 StatisticsIOMonitor ioMonitor = new StatisticsIOMonitor();
 SegmentNodeStorePersistence persistence;
-if (persistentCachePath != null) {
-persistence = 
ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, 
path, persistentCachePath, persistentCacheSizeGb);
+
+if (cloudBlobDirectory != null) {
+persistence = new AzurePersistence(cloudBlobDirectory);
 } else {
 persistence = 
ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, 
path);
 }
 
+if (persistentCachePath != null) {
+persistence = ToolUtils.decorateWithCache(persistence, 
persistentCachePath, persistentCacheSizeGb);
+}
+
 FileStoreBuilder builder = 
fileStoreBuilder(Files.createTempDir()).withCustomPersistence(persistence);
 
 if (ioStatistics) {
diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
index d4942f63f1..ab18f4f282 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
@@ -19,11 +19,7 @@ package org.apache.jackrabbit.oak.segment.azure.tool;
 
 import static 
org.apache.jackrabbit.guava.common.base.Preconditions.checkArgument;
 import static 
org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull;
-import static 
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.createArchiveManager;
-import static 
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.createCloudBlobDirectory;
-import static 
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.newFileStore;
-import static

(jackrabbit-oak) branch trunk updated: OAK-9455 - Improve oak-run check to allow fast remote consistency checks (#1508)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 24179e5dd8 OAK-9455 - Improve oak-run check to allow fast remote 
consistency checks (#1508)
24179e5dd8 is described below

commit 24179e5dd8a2b6c7eea0851e0815bc5f86644502
Author: Andrei Dulceanu 
AuthorDate: Fri Jun 7 10:41:07 2024 +0200

OAK-9455 - Improve oak-run check to allow fast remote consistency checks 
(#1508)
---
 .../site/markdown/nodestore/segment/overview.md|   7 +
 .../apache/jackrabbit/oak/run/CheckCommand.java|  91 +++--
 .../oak/segment/azure/tool/AzureCheck.java | 408 +
 .../apache/jackrabbit/oak/segment/tool/Check.java  | 284 ++
 .../tool/{Check.java => check/CheckHelper.java}| 370 +--
 5 files changed, 611 insertions(+), 549 deletions(-)

diff --git a/oak-doc/src/site/markdown/nodestore/segment/overview.md 
b/oak-doc/src/site/markdown/nodestore/segment/overview.md
index 7e07b02ca2..b350f8e470 100644
--- a/oak-doc/src/site/markdown/nodestore/segment/overview.md
+++ b/oak-doc/src/site/markdown/nodestore/segment/overview.md
@@ -803,6 +803,8 @@ The `--bin` option has no effect on binary properties 
stored in an external Blob
 
 The optional `--last [Integer]` argument can be used to control the maximum 
number of revisions to be verified (default is `1`).
 
+The optional `--fail-fast` argument can be used to stop the check as soon as 
an inconsistency is found. If not specified, the tool will continue to check 
the entire journal.
+
 If the `--head` option is specified, the tool will scan **only** the head 
state, ignoring any available checkpoints.
 
 If the `--checkpoints` option is specified, the tool will scan **only** the 
specified checkpoints, ignoring the head state. At least one argument is 
expected with this option; multiple arguments need to be comma-separated.
@@ -820,6 +822,11 @@ If the option is not specified, the full traversal of the 
repository (rooted at
 If the `--io-stats` option is specified, the tool will print some statistics 
about the I/O operations performed during the execution of the check command.
 This option is optional and is disabled by default.
 
+The optional `--persistent-cache-path PERSISTENT_CACHE_PATH` argument allows 
to specify the path for the persistent disk cache. `PERSISTENT_CACHE_PATH` must 
be a valid path.
+
+The optional `--persistent-cache-size-gb ` argument 
allows to limit the maximum size of the persistent disk cache to 
``. If not specified, the default size will be 
limited to `50` GB.
+
+
 ###  Compact
 
 ```
diff --git 
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java 
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java
index 7a0815679e..601e23cebf 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java
@@ -27,6 +27,7 @@ import joptsimple.OptionParser;
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.segment.azure.tool.AzureCheck;
 import org.apache.jackrabbit.oak.segment.tool.Check;
 
 class CheckCommand implements Command {
@@ -38,9 +39,9 @@ class CheckCommand implements Command {
 .withOptionalArg()
 .ofType(Boolean.class)
 .defaultsTo(true);
-OptionSpec journal = parser.accepts("journal", "journal file")
+OptionSpec journal = parser.accepts("journal", "journal file")
 .withRequiredArg()
-.ofType(File.class);
+.ofType(String.class);
 OptionSpec notify = parser.accepts("notify", "number of seconds 
between progress notifications")
 .withRequiredArg()
 .ofType(Long.class)
@@ -61,9 +62,22 @@ class CheckCommand implements Command {
 .withValuesSeparatedBy(',')
 .defaultsTo("all");
 OptionSpec ioStatistics = parser.accepts("io-stats", "Print I/O 
statistics (only for oak-segment-tar)");
-OptionSpec dir = parser.nonOptions()
-.describedAs("path")
-.ofType(File.class);
+OptionSpec dir = parser.nonOptions()
+.describedAs("Path/URI to TAR/remote segment store (required)")
+.ofType(String.class);
+OptionSpec failFast = parser.accepts("fail-fast", "eagerly 
fail if first path/revision checked is inconsistent (default: false)")
+.withOptionalArg()
+.ofType(Boolean.class)
+.defaultsTo(false);
+OptionSpec persistentCachePath = 
parser.

(jackrabbit-oak) 01/01: OAK-9455 - Improve oak-run check to allow fast remote consistency checks

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-9455
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit 60eda2c96661ac0a16dff53af531199247ace7db
Author: Andrei Dulceanu 
AuthorDate: Tue Jun 4 13:00:02 2024 +0200

OAK-9455 - Improve oak-run check to allow fast remote consistency checks
---
 .../site/markdown/nodestore/segment/overview.md|   7 +
 .../apache/jackrabbit/oak/run/CheckCommand.java|  91 +++--
 .../oak/segment/azure/tool/AzureCheck.java | 408 +
 .../apache/jackrabbit/oak/segment/tool/Check.java  | 284 ++
 .../tool/{Check.java => check/CheckHelper.java}| 370 +--
 5 files changed, 611 insertions(+), 549 deletions(-)

diff --git a/oak-doc/src/site/markdown/nodestore/segment/overview.md 
b/oak-doc/src/site/markdown/nodestore/segment/overview.md
index 7e07b02ca2..b350f8e470 100644
--- a/oak-doc/src/site/markdown/nodestore/segment/overview.md
+++ b/oak-doc/src/site/markdown/nodestore/segment/overview.md
@@ -803,6 +803,8 @@ The `--bin` option has no effect on binary properties 
stored in an external Blob
 
 The optional `--last [Integer]` argument can be used to control the maximum 
number of revisions to be verified (default is `1`).
 
+The optional `--fail-fast` argument can be used to stop the check as soon as 
an inconsistency is found. If not specified, the tool will continue to check 
the entire journal.
+
 If the `--head` option is specified, the tool will scan **only** the head 
state, ignoring any available checkpoints.
 
 If the `--checkpoints` option is specified, the tool will scan **only** the 
specified checkpoints, ignoring the head state. At least one argument is 
expected with this option; multiple arguments need to be comma-separated.
@@ -820,6 +822,11 @@ If the option is not specified, the full traversal of the 
repository (rooted at
 If the `--io-stats` option is specified, the tool will print some statistics 
about the I/O operations performed during the execution of the check command.
 This option is optional and is disabled by default.
 
+The optional `--persistent-cache-path PERSISTENT_CACHE_PATH` argument allows 
to specify the path for the persistent disk cache. `PERSISTENT_CACHE_PATH` must 
be a valid path.
+
+The optional `--persistent-cache-size-gb ` argument 
allows to limit the maximum size of the persistent disk cache to 
``. If not specified, the default size will be 
limited to `50` GB.
+
+
 ###  Compact
 
 ```
diff --git 
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java 
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java
index 7a0815679e..601e23cebf 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java
@@ -27,6 +27,7 @@ import joptsimple.OptionParser;
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.segment.azure.tool.AzureCheck;
 import org.apache.jackrabbit.oak.segment.tool.Check;
 
 class CheckCommand implements Command {
@@ -38,9 +39,9 @@ class CheckCommand implements Command {
 .withOptionalArg()
 .ofType(Boolean.class)
 .defaultsTo(true);
-OptionSpec journal = parser.accepts("journal", "journal file")
+OptionSpec journal = parser.accepts("journal", "journal file")
 .withRequiredArg()
-.ofType(File.class);
+.ofType(String.class);
 OptionSpec notify = parser.accepts("notify", "number of seconds 
between progress notifications")
 .withRequiredArg()
 .ofType(Long.class)
@@ -61,9 +62,22 @@ class CheckCommand implements Command {
 .withValuesSeparatedBy(',')
 .defaultsTo("all");
 OptionSpec ioStatistics = parser.accepts("io-stats", "Print I/O 
statistics (only for oak-segment-tar)");
-OptionSpec dir = parser.nonOptions()
-.describedAs("path")
-.ofType(File.class);
+OptionSpec dir = parser.nonOptions()
+.describedAs("Path/URI to TAR/remote segment store (required)")
+.ofType(String.class);
+OptionSpec failFast = parser.accepts("fail-fast", "eagerly 
fail if first path/revision checked is inconsistent (default: false)")
+.withOptionalArg()
+.ofType(Boolean.class)
+.defaultsTo(false);
+OptionSpec persistentCachePath = 
parser.accepts("persistent-cache-path", "Path/URI to persistent cache where " +
+"resulting segments will be written")
+.withRequiredA

(jackrabbit-oak) branch issues/OAK-9455 created (now 60eda2c966)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-9455
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at 60eda2c966 OAK-9455 - Improve oak-run check to allow fast remote 
consistency checks

This branch includes the following new commits:

 new 60eda2c966 OAK-9455 - Improve oak-run check to allow fast remote 
consistency checks

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

(jackrabbit-oak) branch trunk updated: OAK-10807 - Add oak-run diff to Azure repositories (#1496)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 53fd887f88 OAK-10807 - Add oak-run diff to Azure repositories (#1496)
53fd887f88 is described below

commit 53fd887f886a96fb3b6122d10f5cab0b3c8d6683
Author: Andrei Dulceanu 
AuthorDate: Thu May 30 11:30:22 2024 +0200

OAK-10807 - Add oak-run diff to Azure repositories (#1496)
---
 .../jackrabbit/oak/run/FileStoreDiffCommand.java   |  67 ++---
 .../oak/segment/azure/tool/ToolUtils.java  |  33 -
 .../apache/jackrabbit/oak/segment/tool/Diff.java   | 150 -
 .../jackrabbit/oak/segment/tool/Revisions.java |  20 +--
 .../apache/jackrabbit/oak/segment/tool/Utils.java  |  10 +-
 5 files changed, 186 insertions(+), 94 deletions(-)

diff --git 
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java 
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java
index 5b0871b216..3bc7d27d0c 100644
--- 
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java
+++ 
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java
@@ -18,13 +18,19 @@
 package org.apache.jackrabbit.oak.run;
 
 import static java.util.Arrays.asList;
+import static 
org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
+import static 
org.apache.jackrabbit.oak.segment.tool.Utils.newBasicReadOnlyBlobStore;
 
 import java.io.File;
 
+import com.google.common.io.Files;
 import joptsimple.OptionParser;
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils;
+import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
+import 
org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
 import org.apache.jackrabbit.oak.segment.tool.Diff;
 import org.apache.jackrabbit.oak.segment.tool.Revisions;
 
@@ -34,7 +40,7 @@ class FileStoreDiffCommand implements Command {
 public void execute(String... args) throws Exception {
 OptionParser parser = new OptionParser();
 OptionSpec help = parser.acceptsAll(asList("h", "?", "help"), "show 
help").forHelp();
-OptionSpec storeO = parser.nonOptions("Path to segment store 
(required)").ofType(File.class);
+OptionSpec pathOrURI0 = parser.nonOptions("Path/URI to segment 
store (required)").ofType(String.class);
 OptionSpec outO = parser.accepts("output", "Output 
file").withRequiredArg().ofType(File.class).defaultsTo(defaultOutFile());
 OptionSpec listOnlyO = parser.accepts("list", "Lists available 
revisions");
 OptionSpec intervalO = parser.accepts("diff", "Revision diff 
interval. Ex '--diff=R0..R1'. 'HEAD' can be used to reference the latest head 
revision, ie. '--diff=R0..HEAD'").withRequiredArg().ofType(String.class);
@@ -48,9 +54,9 @@ class FileStoreDiffCommand implements Command {
 System.exit(0);
 }
 
-File store = storeO.value(options);
+String pathOrURI = pathOrURI0.value(options);
 
-if (store == null) {
+if (pathOrURI == null) {
 parser.printHelpOn(System.out);
 System.exit(1);
 }
@@ -65,21 +71,48 @@ class FileStoreDiffCommand implements Command {
 
 int statusCode;
 if (listOnly) {
-statusCode = Revisions.builder()
-.withPath(store)
-.withOutput(out)
-.build()
-.run();
+if (pathOrURI.startsWith("az:")) {
+statusCode = Revisions.builder()
+.withPath(pathOrURI)
+.withOutput(out)
+.build()
+.run(ToolUtils::readRevisions);
+} else {
+statusCode = Revisions.builder()
+.withPath(pathOrURI)
+.withOutput(out)
+.build()
+
.run(org.apache.jackrabbit.oak.segment.tool.Utils::readRevisions);
+}
 } else {
-statusCode = Diff.builder()
-.withPath(store)
-.withOutput(out)
-.withInterval(interval)
-.withIncremental(incremental)
-.withFilter(path)
-.withIgnoreMissingSegments(ignoreSNFEs)
-.build()
-.run();
+if (pathOrURI.startsWith("az:")) {
+SegmentNodeStorePersistence azurePersistence = 
ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, 
pathOrURI

(jackrabbit-oak) branch issues/OAK-10807 created (now c721c7a173)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10807
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at c721c7a173 OAK-10807 - Add oak-run diff to Azure repositories

This branch includes the following new commits:

 new c721c7a173 OAK-10807 - Add oak-run diff to Azure repositories

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

(jackrabbit-oak) 01/01: OAK-10807 - Add oak-run diff to Azure repositories

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-10807
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit c721c7a17352486500e669436cd35aaef6144c96
Author: Andrei Dulceanu 
AuthorDate: Thu May 16 09:15:37 2024 +0200

OAK-10807 - Add oak-run diff to Azure repositories
---
 .../jackrabbit/oak/run/FileStoreDiffCommand.java   |  67 ++---
 .../oak/segment/azure/tool/ToolUtils.java  |  33 -
 .../apache/jackrabbit/oak/segment/tool/Diff.java   | 150 -
 .../jackrabbit/oak/segment/tool/Revisions.java |  20 +--
 .../apache/jackrabbit/oak/segment/tool/Utils.java  |  10 +-
 5 files changed, 186 insertions(+), 94 deletions(-)

diff --git 
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java 
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java
index 5b0871b216..3bc7d27d0c 100644
--- 
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java
+++ 
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java
@@ -18,13 +18,19 @@
 package org.apache.jackrabbit.oak.run;
 
 import static java.util.Arrays.asList;
+import static 
org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
+import static 
org.apache.jackrabbit.oak.segment.tool.Utils.newBasicReadOnlyBlobStore;
 
 import java.io.File;
 
+import com.google.common.io.Files;
 import joptsimple.OptionParser;
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils;
+import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
+import 
org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
 import org.apache.jackrabbit.oak.segment.tool.Diff;
 import org.apache.jackrabbit.oak.segment.tool.Revisions;
 
@@ -34,7 +40,7 @@ class FileStoreDiffCommand implements Command {
 public void execute(String... args) throws Exception {
 OptionParser parser = new OptionParser();
 OptionSpec help = parser.acceptsAll(asList("h", "?", "help"), "show 
help").forHelp();
-OptionSpec storeO = parser.nonOptions("Path to segment store 
(required)").ofType(File.class);
+OptionSpec pathOrURI0 = parser.nonOptions("Path/URI to segment 
store (required)").ofType(String.class);
 OptionSpec outO = parser.accepts("output", "Output 
file").withRequiredArg().ofType(File.class).defaultsTo(defaultOutFile());
 OptionSpec listOnlyO = parser.accepts("list", "Lists available 
revisions");
 OptionSpec intervalO = parser.accepts("diff", "Revision diff 
interval. Ex '--diff=R0..R1'. 'HEAD' can be used to reference the latest head 
revision, ie. '--diff=R0..HEAD'").withRequiredArg().ofType(String.class);
@@ -48,9 +54,9 @@ class FileStoreDiffCommand implements Command {
 System.exit(0);
 }
 
-File store = storeO.value(options);
+String pathOrURI = pathOrURI0.value(options);
 
-if (store == null) {
+if (pathOrURI == null) {
 parser.printHelpOn(System.out);
 System.exit(1);
 }
@@ -65,21 +71,48 @@ class FileStoreDiffCommand implements Command {
 
 int statusCode;
 if (listOnly) {
-statusCode = Revisions.builder()
-.withPath(store)
-.withOutput(out)
-.build()
-.run();
+if (pathOrURI.startsWith("az:")) {
+statusCode = Revisions.builder()
+.withPath(pathOrURI)
+.withOutput(out)
+.build()
+.run(ToolUtils::readRevisions);
+} else {
+statusCode = Revisions.builder()
+.withPath(pathOrURI)
+.withOutput(out)
+.build()
+
.run(org.apache.jackrabbit.oak.segment.tool.Utils::readRevisions);
+}
 } else {
-statusCode = Diff.builder()
-.withPath(store)
-.withOutput(out)
-.withInterval(interval)
-.withIncremental(incremental)
-.withFilter(path)
-.withIgnoreMissingSegments(ignoreSNFEs)
-.build()
-.run();
+if (pathOrURI.startsWith("az:")) {
+SegmentNodeStorePersistence azurePersistence = 
ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, 
pathOrURI);
+ReadOnlyFileStore store = 
fileStoreBuilder(Files.createTempDir()).withCustomPersistence(azurePersistence).withBlobStore(newBasicReadOnlyBlobStore()).buildReadOnly

(jackrabbit-oak) branch trunk updated: Issues/oak 10781 (#1485)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 3c97422858 Issues/oak 10781 (#1485)
3c97422858 is described below

commit 3c974228584433932873f1ad2279affe2ee14618
Author: Tushar <145645280+t-r...@users.noreply.github.com>
AuthorDate: Tue May 28 18:47:09 2024 +0530

Issues/oak 10781 (#1485)

* OAK-10675: add service principal support in oak-blob-cloud-azure

* OAK-10675: use user delegation key signed sas for service principal and 
add license to new files

* OAK-10675: some self review changes

* OAK-10675: set null blob headers as empty string

* OAK-10675: update pom to resolve package dependency

* OAK-10675: add import packages to resolve bundle

* OAK-10770: embed runtime dependencies of azure identity in 
oak-segment-azure

* OAK-10770: remove unused imports, refactor dependencies

* OAK-10770: add imports, refactor dependencies

* OAK-10675: fix runtime dependency issues, code review

* OAK-10781: add access token refresh mechanism

* close executor

* reduce token refresh delauy to 1 minute

* retrigger build
---
 .../blobstorage/AzureBlobContainerProvider.java| 70 +-
 1 file changed, 67 insertions(+), 3 deletions(-)

diff --git 
a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java
 
b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java
index c8600594cd..2f6d6b4f47 100644
--- 
a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java
+++ 
b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java
@@ -18,6 +18,7 @@
  */
 package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
 
+import com.azure.core.credential.AccessToken;
 import com.azure.core.credential.TokenRequestContext;
 import com.azure.identity.ClientSecretCredential;
 import com.azure.identity.ClientSecretCredentialBuilder;
@@ -34,18 +35,30 @@ import 
com.microsoft.azure.storage.blob.SharedAccessBlobPermissions;
 import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import java.io.Closeable;
+import java.io.IOException;
 import java.net.URISyntaxException;
 import java.security.InvalidKeyException;
 import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.Date;
 import java.util.EnumSet;
 import java.util.Optional;
 import java.util.Properties;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 
-public class AzureBlobContainerProvider {
+public class AzureBlobContainerProvider implements Closeable {
+private static final Logger log = 
LoggerFactory.getLogger(AzureBlobContainerProvider.class);
 private static final String DEFAULT_ENDPOINT_SUFFIX = "core.windows.net";
 private static final String AZURE_DEFAULT_SCOPE = 
"https://storage.azure.com/.default;;
 private final String azureConnectionString;
@@ -57,6 +70,9 @@ public class AzureBlobContainerProvider {
 private final String tenantId;
 private final String clientId;
 private final String clientSecret;
+private static final long TOKEN_REFRESHER_INITIAL_DELAY = 45L;
+private static final long TOKEN_REFRESHER_DELAY = 1L;
+private static final ScheduledExecutorService executorService = 
Executors.newSingleThreadScheduledExecutor();
 
 private AzureBlobContainerProvider(Builder builder) {
 this.azureConnectionString = builder.azureConnectionString;
@@ -194,8 +210,15 @@ public class AzureBlobContainerProvider {
 .clientSecret(clientSecret)
 .tenantId(tenantId)
 .build();
-String accessToken = clientSecretCredential.getTokenSync(new 
TokenRequestContext().addScopes(AZURE_DEFAULT_SCOPE)).getToken();
-return new StorageCredentialsToken(accountName, accessToken);
+AccessToken accessToken = clientSecretCredential.getTokenSync(new 
TokenRequestContext().addScopes(AZURE_DEFAULT_SCOPE));
+if (accessToken == null || 
StringUtils.isBlank(accessToken.getToken())) {
+log.error("Access token is null or empty

(jackrabbit-oak) branch trunk updated (a9d0c1c812 -> 62eee36beb)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


from a9d0c1c812 Revert "OAK-10830: Upgrade jackson-databind dependency to 
2.17.1 (#1486)"
 add 62eee36beb OAK-10833 - Consistency check reports success for 
repository with SNFE in checkpoints (#1487)

No new revisions were added by this update.

Summary of changes:
 .../java/org/apache/jackrabbit/oak/segment/tool/Check.java | 10 +-
 1 file changed, 9 insertions(+), 1 deletion(-)

(jackrabbit-oak) branch issues/OAK-10833 created (now 67658f1ce4)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10833
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at 67658f1ce4 OAK-10833 - Consistency check reports success for 
repository with SNFE in checkpoints

This branch includes the following new commits:

 new 67658f1ce4 OAK-10833 - Consistency check reports success for 
repository with SNFE in checkpoints

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

(jackrabbit-oak) 01/01: OAK-10833 - Consistency check reports success for repository with SNFE in checkpoints

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-10833
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit 67658f1ce4968c3c11ec62a03452d6cdf955b1ba
Author: Andrei Dulceanu 
AuthorDate: Fri May 24 15:28:06 2024 +0200

OAK-10833 - Consistency check reports success for repository with SNFE in 
checkpoints
---
 .../java/org/apache/jackrabbit/oak/segment/tool/Check.java | 10 +-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java
 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java
index 268697b3cf..cafb684aa1 100644
--- 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java
+++ 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java
@@ -586,7 +586,7 @@ public class Check {
 }
 
 private static boolean hasAllRevision(ConsistencyCheckResult result) {
-return hasAnyHeadRevision(result) && hasAnyCheckpointRevision(result);
+return hasAnyHeadRevision(result) && hasAllCheckpointRevision(result);
 }
 
 private static boolean hasAnyHeadRevision(ConsistencyCheckResult result) {
@@ -604,6 +604,14 @@ public class Check {
 .anyMatch(Objects::nonNull);
 }
 
+private static boolean hasAllCheckpointRevision(ConsistencyCheckResult 
result) {
+return result.getCheckpointRevisions()
+.values()
+.stream()
+.flatMap(m -> m.values().stream())
+.allMatch(Objects::nonNull);
+}
+
 private void printRevision(int indent, String path, Revision revision) {
 Optional r = Optional.ofNullable(revision);
 print(

(jackrabbit-oak) branch sprout-test updated: Embedded jackson-core in both oak-segment-azure and oak-blob-cloud-azure

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch sprout-test
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/sprout-test by this push:
 new 7e23e6b5a1 Embedded jackson-core in both oak-segment-azure and 
oak-blob-cloud-azure
7e23e6b5a1 is described below

commit 7e23e6b5a1e6b449386a25ac0583aba206feaea2
Author: dulceanu 
AuthorDate: Sun Jan 28 07:33:08 2024 +0100

Embedded jackson-core in both oak-segment-azure and oak-blob-cloud-azure
---
 oak-blob-cloud-azure/pom.xml | 16 
 oak-segment-azure/pom.xml|  6 ++
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/oak-blob-cloud-azure/pom.xml b/oak-blob-cloud-azure/pom.xml
index 7c8dfac2bf..144da0d447 100644
--- a/oak-blob-cloud-azure/pom.xml
+++ b/oak-blob-cloud-azure/pom.xml
@@ -38,10 +38,11 @@
 
 
 
-  !com.google.*,
-  !org.checkerframework.*,
-  !android.os,
-  *
+!com.fasterxml.jackson.core.*,
+!com.google.*,
+!org.checkerframework.*,
+!android.os,
+*
 
 
 
org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage
@@ -51,6 +52,8 @@
 azure-storage,
 azure-keyvault-core,
 guava,
+jsr305,
+jackson-core
 jsr305
 
 
@@ -156,6 +159,11 @@
 3.0.2
 
 
+
+com.fasterxml.jackson.core
+jackson-core
+
+
 
 
 org.apache.jackrabbit
diff --git a/oak-segment-azure/pom.xml b/oak-segment-azure/pom.xml
index 39afa9723d..2d0a5de9fa 100644
--- a/oak-segment-azure/pom.xml
+++ b/oak-segment-azure/pom.xml
@@ -58,6 +58,7 @@
 com.nimbusds.secevent.sdk*;resolution:=optional,
 com.sun.jna*;resolution:=optional,
 !org.apache.jackrabbit.oak.segment*,
+!com.fasterxml.jackson.core.*,
 !com.google.*,
 !android.os,
 !org.checkerframework.*,
@@ -84,6 +85,7 @@
 azure-identity-broker,
 azure-json,
 azure-xml,
+jackson-core,
 guava,
 jsr305
 
@@ -198,6 +200,10 @@
 com.fasterxml.jackson.core
 jackson-databind
 
+
+com.fasterxml.jackson.core
+jackson-core
+
 
 com.azure
 azure-xml

(jackrabbit-oak) branch sprout-test updated (8d1a1b4633 -> f2b0cbf937)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch sprout-test
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


from 8d1a1b4633 Merge pull request #1279 from apache/OAK-10281
 add 0a6f99367f OAK-10615 - Azure Service Principal Support in oak-run 
segment-copy, compact, console (#1280)
 add f2b0cbf937 OAK-10626 - Rework oak-segment-azure OSGi wirings after 
embedding azure libraries needed for service principals support

No new revisions were added by this update.

Summary of changes:
 .../cloud/azure/blobstorage/AzuriteDockerRule.java |   2 +-
 .../site/markdown/nodestore/segment/overview.md|   2 +-
 oak-run/pom.xml|   7 +-
 oak-segment-azure/pom.xml  |   6 +-
 .../segment/azure/AzureSegmentStoreService.java|  13 +--
 .../oak/segment/azure/AzureUtilities.java  |  22 +
 .../jackrabbit/oak/segment/azure/package-info.java |   2 +-
 .../oak/segment/azure/tool/ToolUtils.java  |  37 ++-
 .../SegmentCopyAzureServicePrincipalToTarTest.java |  80 +++
 .../segment/azure/tool/SegmentCopyTestBase.java|  13 ++-
 .../azure/AzureSegmentStoreServiceTest.java|  10 +-
 .../oak/segment/azure/tool/ToolUtilsTest.java  | 108 ++---
 12 files changed, 257 insertions(+), 45 deletions(-)
 create mode 100644 
oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java

(jackrabbit-oak) branch trunk updated: OAK-10626 - Rework oak-segment-azure OSGi wirings after embedding azure libraries needed for service principals support

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new f2b0cbf937 OAK-10626 - Rework oak-segment-azure OSGi wirings after 
embedding azure libraries needed for service principals support
f2b0cbf937 is described below

commit f2b0cbf937ca550343d3f1d244c879c9936433c1
Author: dulceanu 
AuthorDate: Fri Jan 26 17:02:39 2024 +0100

OAK-10626 - Rework oak-segment-azure OSGi wirings after embedding azure 
libraries needed for service principals support
---
 oak-segment-azure/pom.xml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/oak-segment-azure/pom.xml b/oak-segment-azure/pom.xml
index b285fde0e6..39afa9723d 100644
--- a/oak-segment-azure/pom.xml
+++ b/oak-segment-azure/pom.xml
@@ -57,13 +57,13 @@
 
com.nimbusds.openid.connect.sdk*;resolution:=optional,
 com.nimbusds.secevent.sdk*;resolution:=optional,
 com.sun.jna*;resolution:=optional,
-org.reactivestreams;resolution:=optional,
-reactor.core*;resolution:=optional,
-reactor.util*;resolution:=optional,
 !org.apache.jackrabbit.oak.segment*,
 !com.google.*,
 !android.os,
 !org.checkerframework.*,
+!org.reactivestreams.*,
+!reactor.core.*,
+!reactor.util.*,
 *
 
 

(jackrabbit-oak) branch trunk updated: OAK-10615 - Azure Service Principal Support in oak-run segment-copy, compact, console (#1280)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 0a6f99367f OAK-10615 - Azure Service Principal Support in oak-run 
segment-copy, compact, console (#1280)
0a6f99367f is described below

commit 0a6f99367fd20bf215b4154a76e4a6e422acd2a5
Author: Andrei Dulceanu 
AuthorDate: Fri Jan 26 16:00:04 2024 +0200

OAK-10615 - Azure Service Principal Support in oak-run segment-copy, 
compact, console (#1280)

* OAK-10615 - Azure Service Principal Support in oak-run
First un-tested implementation

* Fixed ToolUtilsTest

* Added test for warning when connecting with AZURE_SECRET_KEY instead of 
service principal

* Added SegmentCopyAzureServicePrincipalToTarTest

* Fixed warning test when connecting with AZURE_SECRET_KEY
Bumped azurite version to 3.29.0
Increased oak.segment.azure package version after refactorings

* Bumped azurite to 3.29.0
Increased again oak-run size to include apache commons lang3 needed for 
running azure commands

* Updated Jackrabbit Oak Site Documentation
---
 .../cloud/azure/blobstorage/AzuriteDockerRule.java |   2 +-
 .../site/markdown/nodestore/segment/overview.md|   2 +-
 oak-run/pom.xml|   7 +-
 .../segment/azure/AzureSegmentStoreService.java|  13 +--
 .../oak/segment/azure/AzureUtilities.java  |  22 +
 .../jackrabbit/oak/segment/azure/package-info.java |   2 +-
 .../oak/segment/azure/tool/ToolUtils.java  |  37 ++-
 .../SegmentCopyAzureServicePrincipalToTarTest.java |  80 +++
 .../segment/azure/tool/SegmentCopyTestBase.java|  13 ++-
 .../azure/AzureSegmentStoreServiceTest.java|  10 +-
 .../oak/segment/azure/tool/ToolUtilsTest.java  | 108 ++---
 11 files changed, 254 insertions(+), 42 deletions(-)

diff --git 
a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java
 
b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java
index 3eaf27980a..cb709aca29 100644
--- 
a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java
+++ 
b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java
@@ -38,7 +38,7 @@ import java.util.concurrent.atomic.AtomicReference;
 
 public class AzuriteDockerRule extends ExternalResource {
 
-private static final DockerImageName DOCKER_IMAGE_NAME = 
DockerImageName.parse("mcr.microsoft.com/azure-storage/azurite:3.19.0");
+private static final DockerImageName DOCKER_IMAGE_NAME = 
DockerImageName.parse("mcr.microsoft.com/azure-storage/azurite:3.29.0");
 public static final String ACCOUNT_KEY = 
"Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
 public static final String ACCOUNT_NAME = "devstoreaccount1";
 private static final AtomicReference STARTUP_EXCEPTION = new 
AtomicReference<>();
diff --git a/oak-doc/src/site/markdown/nodestore/segment/overview.md 
b/oak-doc/src/site/markdown/nodestore/segment/overview.md
index cd2133c058..7e07b02ca2 100644
--- a/oak-doc/src/site/markdown/nodestore/segment/overview.md
+++ b/oak-doc/src/site/markdown/nodestore/segment/overview.md
@@ -688,7 +688,7 @@ Besides the local storage in TAR files (previously known as 
TarMK), support for
 
 **Connection Instructions**:
 
-* **Microsoft Azure** The `cloud-prefix` for MS Azure is `az`, therefore a 
valid connection argument would be 
`az:https://myaccount.blob.core.windows.net/container/repository`, where the 
part after `:` is the Azure URL identifier for the _repository_ directory 
inside the specified _container_ of the _myaccount_ Azure storage account. The 
last missing piece is the secret key which will be supplied as an environment 
variable, i.e. `AZURE_SECRET_KEY`.
+* **Microsoft Azure** The `cloud-prefix` for MS Azure is `az`, therefore a 
valid connection argument would be 
`az:https://myaccount.blob.core.windows.net/container/repository`, where the 
part after `:` is the Azure URL identifier for the _repository_ directory 
inside the specified _container_ of the _myaccount_ Azure storage account. 
Default authentication to Microsoft Entra ID with service principal credentials 
supplied via `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET` and `AZURE_TENANT_ID`  
[...]
 
 * **Amazon AWS** The `cloud-prefix` for Amazon AWS is `aws`, therefore a valid 
connection argument would be 
`aws:bucket;root_directory;journal_table;lock_table` where the part after `:` 
defines the _root_directory_ inside the specified _bucket_ in S3 and the 
_journal_table_ and _lock_table_ table

(jackrabbit-oak) branch sprout-test created (now 8d1a1b4633)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch sprout-test
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at 8d1a1b4633 Merge pull request #1279 from apache/OAK-10281

No new revisions were added by this update.

(jackrabbit-oak) branch trunk updated: Increased again oak-run jar size due to size check complaining for local testing

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new c3fae64fb9 Increased again oak-run jar size due to size check 
complaining for local testing
c3fae64fb9 is described below

commit c3fae64fb99ee18a033b35da280ed27ba7235bc9
Author: dulceanu 
AuthorDate: Tue Jan 23 15:08:36 2024 +0100

Increased again oak-run jar size due to size check complaining for local 
testing
---
 oak-run/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oak-run/pom.xml b/oak-run/pom.xml
index 791488196b..0c2057422f 100644
--- a/oak-run/pom.xml
+++ b/oak-run/pom.xml
@@ -48,7 +48,7 @@
   + 41 MB build failing on the release profile (OAK-6250)
   + 38 MB. Initial value. Current 35MB plus a 10%
 -->
-7810
+7830
   
 
   

(jackrabbit-oak) branch trunk updated: OAK-10604 - Azure Service Principal Support in oak-segment-azure (#1268)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 5389ea4d75 OAK-10604 - Azure Service Principal Support in 
oak-segment-azure (#1268)
5389ea4d75 is described below

commit 5389ea4d7560b7ede65aa382765bfa1d83201967
Author: Andrei Dulceanu 
AuthorDate: Mon Jan 22 17:24:06 2024 +0200

OAK-10604 - Azure Service Principal Support in oak-segment-azure (#1268)

* OAK-10604 - Azure Service Principal Support in oak-segment-azure
Added service principal configuration properties
Added Azure Identity client library for Java as a dependency and updated 
all oak-run* poms to adjust size

* OAK-10604 - Azure Service Principal Support in oak-segment-azure
First untested implementation

* OAK-10604 - Azure Service Principal Support in oak-segment-azure
Added scope to token request
Added test case using environment variables

* OAK-10604 - Azure Service Principal Support in oak-segment-azure
Nitpick - avoid wildcard imports

* OAK-10604 - Azure Service Principal Support in oak-segment-azure
Corrected token request scope and fixed the test

* OAK-10604 - Azure Service Principal Support in oak-segment-azure
OSGi wiring fix - wip

* OSGi wiring - wip

* OAK-10604 - Azure Service Principal Support in oak-segment-azure
Fixed OSGi wiring

* OAK-10604 - Azure Service Principal Support in oak-segment-azure
Increased size for oak-run and oak-run-elastic jars after embedding more 
libraries in oak-segment-azure

* OAK-10604 - Azure Service Principal Support in oak-segment-azure
Increased again oak-run jar size due to sonar complaining about it during CI
---
 oak-parent/pom.xml | 25 
 oak-run-elastic/pom.xml|  3 +-
 oak-run/pom.xml|  3 +-
 oak-segment-azure/pom.xml  | 55 -
 .../segment/azure/AzureSegmentStoreService.java| 70 ++---
 .../oak/segment/azure/Configuration.java   | 15 +
 .../jackrabbit/oak/segment/azure/package-info.java |  2 +-
 .../azure/AzureSegmentStoreServiceTest.java| 72 +++---
 8 files changed, 209 insertions(+), 36 deletions(-)

diff --git a/oak-parent/pom.xml b/oak-parent/pom.xml
index 1037b115e1..44772a7ee0 100644
--- a/oak-parent/pom.xml
+++ b/oak-parent/pom.xml
@@ -721,6 +721,31 @@
 azure-keyvault-core
 1.2.6
   
+  
+com.azure
+azure-identity
+1.11.1
+  
+  
+com.azure
+azure-identity-broker
+1.0.1
+  
+  
+com.azure
+azure-core
+1.45.1
+  
+  
+com.azure
+azure-json
+1.0.1
+  
+  
+com.azure
+azure-xml
+1.0.0-beta.2
+  
 
   
   
diff --git a/oak-run-elastic/pom.xml b/oak-run-elastic/pom.xml
index a6312ba5c8..391a02c024 100644
--- a/oak-run-elastic/pom.xml
+++ b/oak-run-elastic/pom.xml
@@ -38,8 +38,9 @@
 121 MB : add Elasticsearch Java client along with RHLC: the latter can 
be removed when the code can be fully migrated to use the new client
 125 MB : shaded Guava
 85 MB : remove Elasticsearch RHLC
+103.5 MB: Azure Identity client library for Java (OAK-10604)
 -->
-8500
+10350
   
 
 
diff --git a/oak-run/pom.xml b/oak-run/pom.xml
index b58f2d379e..791488196b 100644
--- a/oak-run/pom.xml
+++ b/oak-run/pom.xml
@@ -34,6 +34,7 @@
 9.4.53.v20231009
 
-62914560
+7810
   
 
   
diff --git a/oak-segment-azure/pom.xml b/oak-segment-azure/pom.xml
index f8c7f6512f..b285fde0e6 100644
--- a/oak-segment-azure/pom.xml
+++ b/oak-segment-azure/pom.xml
@@ -42,6 +42,24 @@
 
 org.apache.jackrabbit.oak.segment.spi*,
 org.apache.jackrabbit.oak.segment.remote*,
+
com.fasterxml.jackson.annotation;resolution:=optional,
+
com.fasterxml.jackson.databind*;resolution:=optional,
+
com.fasterxml.jackson.dataformat.xml;resolution:=optional,
+
com.fasterxml.jackson.datatype*;resolution:=optional,
+com.microsoft.aad.msal4j*;resolution:=optional,
+
com.nimbusds.common.contenttype;resolution:=optional,
+com.nimbusds.jose*;resolution:=optional,
+com.nimbusds.jwt;resolution:=optional,
+com.nimbusds.jwt.util;resolution:=optional,
+com.nimbusds.oauth2.sdk*;resolut

(jackrabbit-oak) branch issues/OAK-10604 updated (fca5643691 -> 6c50d26a56)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10604
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


from fca5643691 OAK-10604 - Azure Service Principal Support in 
oak-segment-azure Increased size for oak-run and oak-run-elastic jars after 
embedding more libraries in oak-segment-azure
 add 6c50d26a56 OAK-10604 - Azure Service Principal Support in 
oak-segment-azure Increased again oak-run jar size due to sonar complaining 
about it during CI

No new revisions were added by this update.

Summary of changes:
 oak-run/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(jackrabbit-oak) branch issues/OAK-10604 updated (ff172bd6b3 -> fca5643691)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10604
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


from ff172bd6b3 OAK-10604 - Azure Service Principal Support in 
oak-segment-azure Fixed OSGi wiring
 add fca5643691 OAK-10604 - Azure Service Principal Support in 
oak-segment-azure Increased size for oak-run and oak-run-elastic jars after 
embedding more libraries in oak-segment-azure

No new revisions were added by this update.

Summary of changes:
 oak-run-elastic/pom.xml | 4 ++--
 oak-run/pom.xml | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

(jackrabbit-oak) branch issues/OAK-10604 updated (2ce8114a1e -> ff172bd6b3)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10604
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


from 2ce8114a1e OAK-10604 - Azure Service Principal Support in 
oak-segment-azure OSGi wiring fix - wip
 add 625b9b9149 OSGi wiring - wip
 add ff172bd6b3 OAK-10604 - Azure Service Principal Support in 
oak-segment-azure Fixed OSGi wiring

No new revisions were added by this update.

Summary of changes:
 oak-parent/pom.xml| 10 +
 oak-segment-azure/pom.xml | 95 ---
 2 files changed, 41 insertions(+), 64 deletions(-)

(jackrabbit-oak) branch issues/OAK-10604 updated (e80cd5b6bb -> 2ce8114a1e)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10604
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


from e80cd5b6bb OAK-10604 - Azure Service Principal Support in 
oak-segment-azure Corrected token request scope and fixed the test
 add 2ce8114a1e OAK-10604 - Azure Service Principal Support in 
oak-segment-azure OSGi wiring fix - wip

No new revisions were added by this update.

Summary of changes:
 oak-parent/pom.xml| 10 ++
 oak-segment-azure/pom.xml | 81 ++-
 2 files changed, 90 insertions(+), 1 deletion(-)

(jackrabbit-oak) branch issues/OAK-10604 updated (80370e2a5e -> e80cd5b6bb)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10604
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


from 80370e2a5e OAK-10604 - Azure Service Principal Support in 
oak-segment-azure Added scope to token request Added test case using 
environment variables
 add 546e11ef83 OAK-10604 - Azure Service Principal Support in 
oak-segment-azure Nitpick - avoid wildcard imports
 add e80cd5b6bb OAK-10604 - Azure Service Principal Support in 
oak-segment-azure Corrected token request scope and fixed the test

No new revisions were added by this update.

Summary of changes:
 .../segment/azure/AzureSegmentStoreService.java|  5 +--
 .../azure/AzureSegmentStoreServiceTest.java| 45 --
 2 files changed, 35 insertions(+), 15 deletions(-)

(jackrabbit-oak) branch issues/OAK-10604 updated (57d3611375 -> 80370e2a5e)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10604
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


from 57d3611375 OAK-10604 - Azure Service Principal Support in 
oak-segment-azure First untested implementation
 add 80370e2a5e OAK-10604 - Azure Service Principal Support in 
oak-segment-azure Added scope to token request Added test case using 
environment variables

No new revisions were added by this update.

Summary of changes:
 .../segment/azure/AzureSegmentStoreService.java|  3 +-
 .../azure/AzureSegmentStoreServiceTest.java| 41 +++---
 2 files changed, 38 insertions(+), 6 deletions(-)

(jackrabbit-oak) branch issues/OAK-10604 updated (07b6bc462b -> 57d3611375)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10604
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


omit 07b6bc462b OAK-10604 - Azure Service Principal Support in 
oak-segment-azure Added service principal configuration properties Added Azure 
Identity client library for Java as a dependency and updated all oak-run* poms 
to adjust size
 add 3d2c1437f7 OAK-10605: Update (shaded) Guava to 33.0.0 (#1270)
 add 5aa6d3d75e OAK-10606: azure/oak-blob-cloud-azure: update embedded 
Guava to 33.0.0 (#1272)
 add deeccbe0d4 OAK-10607 Rename Maven property "java.version" (#1271)
 add e0b46fedf4 OAK-10604 - Azure Service Principal Support in 
oak-segment-azure Added service principal configuration properties Added Azure 
Identity client library for Java as a dependency and updated all oak-run* poms 
to adjust size
 add 57d3611375 OAK-10604 - Azure Service Principal Support in 
oak-segment-azure First untested implementation

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (07b6bc462b)
\
 N -- N -- N   refs/heads/issues/OAK-10604 (57d3611375)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

No new revisions were added by this update.

Summary of changes:
 oak-blob-cloud-azure/pom.xml   |  2 +-
 oak-parent/pom.xml | 10 ++--
 oak-segment-azure/pom.xml  |  2 +-
 .../segment/azure/AzureSegmentStoreService.java| 68 ++
 .../jackrabbit/oak/segment/azure/package-info.java |  2 +-
 oak-shaded-guava/pom.xml   |  4 +-
 6 files changed, 53 insertions(+), 35 deletions(-)

(jackrabbit-oak) branch issues/OAK-10604 created (now 07b6bc462b)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10604
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at 07b6bc462b OAK-10604 - Azure Service Principal Support in 
oak-segment-azure Added service principal configuration properties Added Azure 
Identity client library for Java as a dependency and updated all oak-run* poms 
to adjust size

This branch includes the following new commits:

 new 07b6bc462b OAK-10604 - Azure Service Principal Support in 
oak-segment-azure Added service principal configuration properties Added Azure 
Identity client library for Java as a dependency and updated all oak-run* poms 
to adjust size

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

(jackrabbit-oak) 01/01: OAK-10604 - Azure Service Principal Support in oak-segment-azure Added service principal configuration properties Added Azure Identity client library for Java as a dependency a

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-10604
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit 07b6bc462bcedaefe23710af6694d5a87bf9cebb
Author: dulceanu 
AuthorDate: Mon Jan 15 14:01:38 2024 +0100

OAK-10604 - Azure Service Principal Support in oak-segment-azure
Added service principal configuration properties
Added Azure Identity client library for Java as a dependency and updated 
all oak-run* poms to adjust size
---
 oak-parent/pom.xml|  5 +
 oak-run-elastic/pom.xml   |  3 ++-
 oak-run/pom.xml   |  3 ++-
 oak-segment-azure/pom.xml |  7 +++
 .../oak/segment/azure/AzureSegmentStoreService.java   |  8 
 .../jackrabbit/oak/segment/azure/Configuration.java   | 15 +++
 6 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/oak-parent/pom.xml b/oak-parent/pom.xml
index 4cd4f042fa..1fd5dd6703 100644
--- a/oak-parent/pom.xml
+++ b/oak-parent/pom.xml
@@ -721,6 +721,11 @@
 azure-keyvault-core
 1.2.6
   
+  
+com.azure
+azure-identity
+1.11.1
+  
 
   
   
diff --git a/oak-run-elastic/pom.xml b/oak-run-elastic/pom.xml
index a6312ba5c8..c60e4ba1bb 100644
--- a/oak-run-elastic/pom.xml
+++ b/oak-run-elastic/pom.xml
@@ -38,8 +38,9 @@
 121 MB : add Elasticsearch Java client along with RHLC: the latter can 
be removed when the code can be fully migrated to use the new client
 125 MB : shaded Guava
 85 MB : remove Elasticsearch RHLC
+100.5 MB: Azure Identity client library for Java (OAK-10604)
 -->
-8500
+10050
   
 
 
diff --git a/oak-run/pom.xml b/oak-run/pom.xml
index b58f2d379e..4de7922b61 100644
--- a/oak-run/pom.xml
+++ b/oak-run/pom.xml
@@ -34,6 +34,7 @@
 9.4.53.v20231009
 
-62914560
+7504
   
 
   
diff --git a/oak-segment-azure/pom.xml b/oak-segment-azure/pom.xml
index c4d06a11c1..8799b3ba22 100644
--- a/oak-segment-azure/pom.xml
+++ b/oak-segment-azure/pom.xml
@@ -151,6 +151,13 @@
 com.microsoft.azure
 azure-keyvault-core
 
+
+
+
+com.azure
+azure-identity
+
+
 
 
 com.google.guava
diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
index e76372f99c..6bb19292ab 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
@@ -84,6 +84,9 @@ public class AzureSegmentStoreService {
 if (!StringUtils.isBlank(configuration.connectionURL())) {
 return createPersistenceFromConnectionURL(configuration);
 }
+if (!StringUtils.isBlank(configuration.clientId())) {
+return 
createAzurePersistenceFromServicePrincipalCredentials(configuration);
+}
 if (!StringUtils.isBlank(configuration.sharedAccessSignature())) {
 return createPersistenceFromSasUri(configuration);
 }
@@ -145,6 +148,11 @@ public class AzureSegmentStoreService {
 }
 }
 
+@NotNull
+private static AzurePersistence 
createAzurePersistenceFromServicePrincipalCredentials(Configuration 
configuration) throws IOException {
+return null;
+}
+
 @NotNull
 private static String normalizePath(@NotNull String rootPath) {
 if (rootPath.length() > 0 && rootPath.charAt(0) == '/') {
diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/Configuration.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/Configuration.java
index 2eaf628bc1..f0aa1e4b1c 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/Configuration.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/Configuration.java
@@ -68,6 +68,21 @@ import static 
org.apache.jackrabbit.oak.segment.azure.Configuration.PID;
 description = "Blob Endpoint URL used to connect to the Azure Storage")
 String blobEndpoint() default "";
 
+@AttributeDefinition(
+name = "Azure Service Principal ID (optional)",
+description = "Azure Service Principal ID for Azure Storage 
authentication")
+String clientId() default "";
+
+@AttributeDefinition(
+name = "Azure Service Principal Password (optional)",
+description = "Azure Service Principal 

(jackrabbit-oak) branch 1.22 updated: OAK-10591 - Bump netty dependency from 4.1.96.Final to 4.1.104.Final

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch 1.22
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/1.22 by this push:
 new b348485e4c OAK-10591 - Bump netty dependency from 4.1.96.Final to 
4.1.104.Final
b348485e4c is described below

commit b348485e4c938a685136f56849c06235f1209aaa
Author: dulceanu 
AuthorDate: Thu Dec 21 17:30:00 2023 +0100

OAK-10591 - Bump netty dependency from 4.1.96.Final to 4.1.104.Final
---
 oak-segment-tar/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oak-segment-tar/pom.xml b/oak-segment-tar/pom.xml
index 392a6196f4..acc252bf25 100644
--- a/oak-segment-tar/pom.xml
+++ b/oak-segment-tar/pom.xml
@@ -33,7 +33,7 @@
 Oak Segment Tar
 
 
-4.1.86.Final
+4.1.104.Final
 
1.4.2
 
 

(jackrabbit-oak) branch trunk updated: OAK-10591 - Bump netty dependency from 4.1.96.Final to 4.1.104.Final (#1257)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 6e5b58060b OAK-10591 - Bump netty dependency from 4.1.96.Final to 
4.1.104.Final (#1257)
6e5b58060b is described below

commit 6e5b58060bca300e3a0e9a43e8be1c7a52b95840
Author: Andrei Dulceanu 
AuthorDate: Thu Dec 21 19:28:58 2023 +0200

OAK-10591 - Bump netty dependency from 4.1.96.Final to 4.1.104.Final (#1257)
---
 oak-segment-tar/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oak-segment-tar/pom.xml b/oak-segment-tar/pom.xml
index e9d1fbd4a0..ff4b6803ef 100644
--- a/oak-segment-tar/pom.xml
+++ b/oak-segment-tar/pom.xml
@@ -33,7 +33,7 @@
 Oak Segment Tar
 
 
-4.1.96.Final
+4.1.104.Final
 
1.4.2
 
 

(jackrabbit-oak) branch issues/OAK-10591 created (now 1a8c668c28)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10591
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at 1a8c668c28 OAK-10591 - Bump netty dependency from 4.1.96.Final to 
4.1.104.Final

This branch includes the following new commits:

 new 1a8c668c28 OAK-10591 - Bump netty dependency from 4.1.96.Final to 
4.1.104.Final

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

(jackrabbit-oak) 01/01: OAK-10591 - Bump netty dependency from 4.1.96.Final to 4.1.104.Final

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-10591
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit 1a8c668c28f1597fab5fa2c6e47e116285449f28
Author: dulceanu 
AuthorDate: Thu Dec 21 17:30:00 2023 +0100

OAK-10591 - Bump netty dependency from 4.1.96.Final to 4.1.104.Final
---
 oak-segment-tar/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oak-segment-tar/pom.xml b/oak-segment-tar/pom.xml
index e9d1fbd4a0..ff4b6803ef 100644
--- a/oak-segment-tar/pom.xml
+++ b/oak-segment-tar/pom.xml
@@ -33,7 +33,7 @@
 Oak Segment Tar
 
 
-4.1.96.Final
+4.1.104.Final
 
1.4.2
 
 

(jackrabbit-oak) branch trunk updated: OAK-3349 - Partial compaction Fixed license header

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 3df72cbeef OAK-3349 - Partial compaction Fixed license header
3df72cbeef is described below

commit 3df72cbeeffbb052359bf4a4bbed2631cac24e8a
Author: dulceanu 
AuthorDate: Wed Nov 15 18:31:56 2023 +0100

OAK-3349 - Partial compaction
Fixed license header
---
 .../file/DefaultGarbageCollectionStrategyTest.java  | 17 +
 1 file changed, 17 insertions(+)

diff --git 
a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/DefaultGarbageCollectionStrategyTest.java
 
b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/DefaultGarbageCollectionStrategyTest.java
index 9dc8508236..953271cde8 100644
--- 
a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/DefaultGarbageCollectionStrategyTest.java
+++ 
b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/DefaultGarbageCollectionStrategyTest.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
 package org.apache.jackrabbit.oak.segment.file;
 
 import org.apache.jackrabbit.oak.segment.RecordId;

(jackrabbit-oak) branch trunk updated: OAK-10311 - Optimize SegmentBlob#equals for segment blobs that origin… (#1150)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 21b542d2e8 OAK-10311 - Optimize SegmentBlob#equals for segment blobs 
that origin… (#1150)
21b542d2e8 is described below

commit 21b542d2e87c6e3351f23310e928f0dfbea33575
Author: Andrei Dulceanu 
AuthorDate: Wed Nov 15 18:24:35 2023 +0200

OAK-10311 - Optimize SegmentBlob#equals for segment blobs that origin… 
(#1150)

* OAK-10311 - Optimize SegmentBlob#equals for segment blobs that originate 
from the same blob store

* OAK-10311 - Optimize SegmentBlob#equals for segment blobs that originate 
from the same blob store
Switched from content identity to blob id in comparison
---
 .../java/org/apache/jackrabbit/oak/segment/SegmentBlob.java | 13 +
 1 file changed, 13 insertions(+)

diff --git 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
index 3bc438bcbb..9ab2ffea8d 100644
--- 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
+++ 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
@@ -30,6 +30,7 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.commons.properties.SystemPropertySupplier;
 import org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob;
 import org.apache.jackrabbit.oak.plugins.blob.datastore.InMemoryDataRecord;
 import org.apache.jackrabbit.oak.plugins.memory.AbstractBlob;
@@ -41,6 +42,10 @@ import org.jetbrains.annotations.Nullable;
  * A BLOB (stream of bytes). This is a record of type "VALUE".
  */
 public class SegmentBlob extends Record implements Blob {
+private static final boolean FAST_EQUALS_SAME_BLOBSTORE = 
SystemPropertySupplier
+.create("oak.segment.blob.fastEquals.same.blobstore", false)
+.formatSetMessage( (name, value) -> String.format("%s set to: %s", 
name, value) )
+.get();
 
 @Nullable
 private final BlobStore blobStore;
@@ -192,6 +197,14 @@ public class SegmentBlob extends Record implements Blob {
 }
 }
 
+if (FAST_EQUALS_SAME_BLOBSTORE) {
+if (blobStore != null && this.blobStore.equals(that.blobStore) 
&& this.isExternal() && that.isExternal()) {
+if (this.getBlobId() != null && that.getBlobId() != null) {
+return this.getBlobId().equals(that.getBlobId());
+}
+}
+}
+
 if (this.length() != that.length()) {
 return false;
 }

(jackrabbit-oak) branch issues/OAK-3349 updated (92d96801e2 -> 057ccb8555)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-3349
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


 discard 92d96801e2 OAK-3349 Partial Compaction
omit 5136d82b9a OAK-9949 Offline Tail Compaction
omit b375d8fa57 OAK-9922 Parallel Compaction Export package 
org.apache.jackrabbit.oak.plugins.index.counter from oak-core
omit 1d9095709b OAK-9922 Parallel Compaction
 add 563c038a43 OAK-10452 - Mongo query that downloads ancestors of base 
path for regex filtering is doing a column scan (#1129)
 add 8752a2eaa7 OAK-10429: Implementation for IncrementalFFS (#1123)
 add 7be9852aa2 OAK-10456: log index name when bulk update fails (#1132)
 add 189caed371 OAK-10450: bump testcontainers (#1131)
 add ebd83ac64f OAK-10461: oak-search-elastic does not build under Java 17 
- disable Felix SCR plugin (#1134)
 add ad64ecbcfb Set LZ4 as the default compression algorithm for the 
indexing job. (#1133)
 add cef242abb2 Revert "Set LZ4 as the default compression algorithm for 
the indexing job. (#1133)"
 add 0b6e538736 OAK-10453 - Pipelined strategy: enforce size limit on 
memory taken by objects in the queue between download and transform thread 
(#1130)
 add c6b4b08da6 OAK-9922: Parallel Compaction (#1043)
 add 1f3cfa3b3d OAK-10465 : added io.netty 
netty-transport-native-unix-common embedded dependency for oak-segment-tar 
(#1137)
 add f4343950c1 OAK-10458 - Set LZ4 as the default compression algorithm 
for the indexing job. (#1136)
 add b2ffbb446c OAK-10463: Retrieve flatFileStore for indexing from 
baseFlatFileStore (#1135)
 add c747535b92 OAK-9922: segment-tar: parallel compaction - remove 
oak-core dependency
 add 8ef2cf89db OAK-10475 - Expose the Mongo client connection in 
MongoDocumentNodeStoreBuilderBase (#1142)
 add 6b8aba2e76 OAK-10474: Release Oak 1.58.0 (#1145)
 add 25568d29fe [maven-release-plugin] prepare release jackrabbit-oak-1.58.0
 add 82ba25c3fc [maven-release-plugin] prepare for next development 
iteration
 add 62a3f776cb OAK-10471: Implement ConflictHandler for 
UserPrincipalProvider Cache
 add 763031951c OAK-10490 Suggest queries return duplicate entries if 
prefetch is enabled (#1148)
 add 6a341e9dd1 OAK-10492: Update (shaded) Guava to 32.1.3 (#1153)
 add ab772d73a9 OAK-10474: Release Oak 1.58.0 - version cleanup
 add da19433ebe OAK-10486 : Resolution of inherited groups may terminate 
pre-maturely for external users (#1147)
 add d7816c62b9 OAK-10491 - Indexing: pass a MongoDatabase instance instead 
of MongoConnection to indexing logic (#1151)
 add 9c1fd98d68 OAK-10471 Implement ConflictHandler for 
UserPrincipalProvider Cache - fix tests (#1154)
 add 13889a17d1 OAK-10495: Ignore DocumentStoreIndexerIT#parallelReindex* 
tests  (#1157)
 add 31a3341ac3 OAK-10487: replaced lsm estimator with cacheable count 
(#1149)
 add 4c7e9f4144 OAK-10502: oak-segment-azure/oak-blob-cloud-azure: update 
embedded Guava to 31.0 (#1162)
 add 274398d483 OAK-10507: oak-segment-azure/oak-blob-cloud-azure: update 
embedded Guava to 31.1 (#1163)
 add 9aeef93e59 OAK-10508: oak-segment-azure/oak-blob-cloud-azure: update 
embedded Guava to 32.0.0 (#1164)
 add e3c2dd6303 OAK-10510: oak-segment-azure/oak-blob-cloud-azure: update 
embedded Guava to 32.0.1 (#1167)
 add 63aef2f4a1 OAK-10497 Properties order in FFS can be different across 
runs: option to sort (#1159)
 add 1b31a22066 OAK-10512: oak-segment-azure/oak-blob-cloud-azure: update 
embedded Guava to 32.1.0 (#1170)
 add c255d16fed OAK-10497 Properties order in FFS can be different across 
runs: sort (#1175)
 add 2a10b89f3e OAK-10513: oak-segment-azure/oak-blob-cloud-azure: update 
embedded Guava to 32.1.1 (#1176)
 add 9d54f6f23f OAK-10515: oak-segment-azure/oak-blob-cloud-azure: update 
embedded Guava to 32.1.2 (#1177)
 add e904e45942 OAK-10503 | Handle Exception in case of diff failures while 
building incremental FFS and handling Add operations during merging of 
incremental FFS. (#1172)
 add bb98a83554 OAK-10516: oak-segment-azure/oak-blob-cloud-azure: update 
embedded Guava to 32.1.3 (#1178)
 add 8d307fcdaa OAK-6760: Convert oak-blob-cloud to OSGi R6 annotations 
(#1144)
 add d9ab316b2b OAK-10505 - Make PIPELINED the default indexing strategy. 
(#1168)
 add abcfb39909 Deprecate all Mongo download strategies except for 
Pipelined. (#1169)
 add b0ec30536f OAK-10514: Utility method to remove unmerged branch changes
 add 9377d05a39 Merge pull request #1173 from mreutegg/OAK-10514
 add 380467dbfe OAK-10504- Add log messages with start and end of the 
indexing job. (#1165)
 add 9013717676 OAK-301 Improve documentation - fix the lucene documentation
 add e682bc06c6 OAK-301 Improve documentation - fix the lucene documentation
 add 546e889da6 OAK-10460 - PIPELINED strategy: support eag

(jackrabbit-oak) branch issues/OAK-10311 updated: OAK-10311 - Optimize SegmentBlob#equals for segment blobs that originate from the same blob store Switched from content identity to blob id in compari

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-10311
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/issues/OAK-10311 by this push:
 new 7699ac150d OAK-10311 - Optimize SegmentBlob#equals for segment blobs 
that originate from the same blob store Switched from content identity to blob 
id in comparison
7699ac150d is described below

commit 7699ac150d78ee54d428389996f70120225ad6c6
Author: dulceanu 
AuthorDate: Wed Nov 15 11:57:39 2023 +0100

OAK-10311 - Optimize SegmentBlob#equals for segment blobs that originate 
from the same blob store
Switched from content identity to blob id in comparison
---
 .../src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
index 25d525236a..9ab2ffea8d 100644
--- 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
+++ 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
@@ -199,8 +199,8 @@ public class SegmentBlob extends Record implements Blob {
 
 if (FAST_EQUALS_SAME_BLOBSTORE) {
 if (blobStore != null && this.blobStore.equals(that.blobStore) 
&& this.isExternal() && that.isExternal()) {
-if (this.getContentIdentity() != null && 
that.getContentIdentity() != null) {
-return 
this.getContentIdentity().equals(that.getContentIdentity());
+if (this.getBlobId() != null && that.getBlobId() != null) {
+return this.getBlobId().equals(that.getBlobId());
 }
 }
 }

(jackrabbit-oak) branch trunk updated: OAK-10511 - Get blobId without network roundtrip (#1171)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 0de5e850ec OAK-10511 - Get blobId without network roundtrip (#1171)
0de5e850ec is described below

commit 0de5e850ec8f7a0ca482529e4adbe20f8ef8afa2
Author: Axel Hanikel 
AuthorDate: Tue Nov 7 11:29:46 2023 +0100

OAK-10511 - Get blobId without network roundtrip (#1171)

* OAK-10511 - Get blobId without network roundtrip

* Add a test

* Lazy init: as in https://stackoverflow.com/a/3580658

-

Co-authored-by: Axel Hanikel 
---
 .../oak/segment/DefaultSegmentWriter.java  |  8 +
 .../oak/segment/DefaultSegmentWriterTest.java  | 40 +-
 .../oak/segment/test/TemporaryBlobStore.java   | 33 +++---
 .../oak/segment/test/TemporaryFileStore.java   | 37 
 4 files changed, 105 insertions(+), 13 deletions(-)

diff --git 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java
 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java
index 3211151ef8..edef45810b 100644
--- 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java
+++ 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java
@@ -65,6 +65,7 @@ import org.apache.jackrabbit.oak.api.Blob;
 import org.apache.jackrabbit.oak.api.PropertyState;
 import org.apache.jackrabbit.oak.api.Type;
 import org.apache.jackrabbit.oak.commons.Buffer;
+import org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob;
 import org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState;
 import org.apache.jackrabbit.oak.segment.RecordWriters.RecordWriter;
 import org.apache.jackrabbit.oak.segment.WriteOperationHandler.WriteOperation;
@@ -576,6 +577,13 @@ public class DefaultSegmentWriter implements SegmentWriter 
{
 }
 }
 
+if (blob instanceof BlobStoreBlob) {
+String blobId = ((BlobStoreBlob) blob).getBlobId();
+if (blobId != null) {
+return writeBlobId(blobId);
+}
+}
+
 String reference = blob.getReference();
 if (reference != null && blobStore != null) {
 String blobId = blobStore.getBlobId(reference);
diff --git 
a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java
 
b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java
index 49e7ebb795..d2e96c1707 100644
--- 
a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java
+++ 
b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java
@@ -24,10 +24,12 @@ import static 
org.apache.jackrabbit.guava.common.collect.Maps.newHashMap;
 import static 
org.apache.jackrabbit.oak.segment.DefaultSegmentWriterBuilder.defaultSegmentWriterBuilder;
 import static org.apache.jackrabbit.oak.segment.ListRecord.LEVEL_SIZE;
 import static org.apache.jackrabbit.oak.segment.ListRecord.MAX_ELEMENTS;
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import java.io.ByteArrayInputStream;
 import java.io.File;
@@ -40,6 +42,10 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob;
+import org.apache.jackrabbit.oak.segment.test.TemporaryBlobStore;
 import org.apache.jackrabbit.oak.segment.test.TemporaryFileStore;
 import org.jetbrains.annotations.NotNull;
 import org.junit.Before;
@@ -68,13 +74,24 @@ public class DefaultSegmentWriterTest {
 
 private static final int SMALL_BINARIES_INLINE_THRESHOLD = 4;
 
+private TemporaryFolder blobFolder = new TemporaryFolder(new 
File("target"));
+
+private TemporaryBlobStore blobStore = new TemporaryBlobStore(blobFolder);
+
 private TemporaryFolder folder = new TemporaryFolder(new File("target"));
 
-private TemporaryFileStore store = new TemporaryFileStore(folder, 
SMALL_BINARIES_INLINE_THRESHOLD);
+private TemporaryFileStore store = new TemporaryFileStore(folder, null, 
SMALL_BINARIES_INLINE_THRESHOLD);
+
+private TemporaryFileStore storeWithBlobStore = new 
TemporaryFileStore(folder, blobStore, SMALL_BINARIES_INLINE_THRESHOLD);
 
 @Rule
 public RuleChain rules = RuleChain.outerRule(folder).around(store);
 
+   

(jackrabbit-oak) branch trunk updated: OAK-9949 Offline Tail Compaction (#1166)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new a9307fc658 OAK-9949 Offline Tail Compaction (#1166)
a9307fc658 is described below

commit a9307fc658bf3cfbc5a7954ac70ca981b699d1b8
Author: Andrei Dulceanu 
AuthorDate: Mon Nov 6 12:04:46 2023 +0200

OAK-9949 Offline Tail Compaction (#1166)

Co-authored-by: Lucas Weitzendorf 
---
 .../site/markdown/nodestore/segment/overview.md|  6 ++-
 .../apache/jackrabbit/oak/run/CompactCommand.java  | 23 +++
 .../oak/segment/aws/tool/AwsCompact.java   | 28 -
 .../oak/segment/azure/tool/AzureCompact.java   | 28 -
 .../oak/segment/CheckpointCompactor.java   | 46 +-
 .../jackrabbit/oak/segment/tool/Compact.java   | 28 -
 6 files changed, 129 insertions(+), 30 deletions(-)

diff --git a/oak-doc/src/site/markdown/nodestore/segment/overview.md 
b/oak-doc/src/site/markdown/nodestore/segment/overview.md
index 53f10542ab..499bcb3cf2 100644
--- a/oak-doc/src/site/markdown/nodestore/segment/overview.md
+++ b/oak-doc/src/site/markdown/nodestore/segment/overview.md
@@ -823,14 +823,16 @@ This option is optional and is disabled by default.
 ###  Compact
 
 ```
-java -jar oak-run.jar compact [--force] [--mmap] [--compactor] [--threads] 
SOURCE [--target-path DESTINATION] [--persistent-cache-path 
PERSISTENT_CACHE_PATH] [--persistent-cache-size-gb ]
+java -jar oak-run.jar compact [--force] [--mmap] [--tail] [--compactor] 
[--threads] SOURCE [--target-path DESTINATION] [--persistent-cache-path 
PERSISTENT_CACHE_PATH] [--persistent-cache-size-gb ]
 ```
 
 The `compact` command performs offline compaction of the local/remote Segment 
Store at `SOURCE`. 
 `SOURCE` must be a valid path/uri to an existing Segment Store. Currently, 
Azure Segment Store and AWS Segment Store the supported remote Segment Stores. 
 Please refer to the [Remote Segment Stores](#remote-segment-stores) section 
for details on how to correctly specify connection URIs.
 
-If the optional `--force [Boolean]` argument is set to `true` the tool ignores 
a non-matching Segment Store version. *CAUTION*: this will upgrade the Segment 
Store to the 
+With the optional `--tail` flag, only tail compaction is performed instead of 
the full repository.
+
+If the optional `--force` flag is set, the tool ignores a non-matching Segment 
Store version. *CAUTION*: this will upgrade the Segment Store to the 
 latest version, which is incompatible with older versions. *There is no way to 
downgrade 
 an accidentally upgraded Segment Store*.  
 
diff --git 
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java 
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
index 47257982d3..f955278931 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
@@ -25,6 +25,7 @@ import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 import org.apache.jackrabbit.oak.run.commons.Command;
 import org.apache.jackrabbit.oak.segment.azure.tool.AzureCompact;
+import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
 import 
org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType;
 import org.apache.jackrabbit.oak.segment.aws.tool.AwsCompact;
 import org.apache.jackrabbit.oak.segment.tool.Compact;
@@ -48,12 +49,11 @@ class CompactCommand implements Command {
 "is always enforced and this option is ignored.")
 .withOptionalArg()
 .ofType(Boolean.class);
-OptionSpec forceArg = parser.accepts("force",
+OptionSpec forceArg = parser.accepts("force",
 "Force compaction and ignore a non matching segment store 
version. " +
 "CAUTION: this will upgrade the segment store to the 
latest version, " +
-"which is incompatible with older versions of Oak.")
-.withOptionalArg()
-.ofType(Boolean.class);
+"which is incompatible with older versions of Oak.");
+OptionSpec tailArg = parser.accepts("tail", "Use tail compaction 
instead of a full repository rewrite.");
 OptionSpec compactor = parser.accepts("compactor",
 "Allow the user to control compactor type to be used. Valid 
choices are \"classic\", \"diff\", \"parallel\". " +
 "While \"classic\" is slower, it might be more stable, 
due to lack of optimisations employed " +
@@ -110,10 +110,13 @@ class CompactCommand imple

[jackrabbit-oak] branch issues/OAK-9949-final created (now a2309992a2)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-9949-final
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at a2309992a2 OAK-9949 Offline Tail Compaction

This branch includes the following new commits:

 new a2309992a2 OAK-9949 Offline Tail Compaction

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[jackrabbit-oak] 01/01: OAK-9949 Offline Tail Compaction

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-9949-final
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit a2309992a222b8740b2fc2fd10eea0892db7ca3f
Author: Lucas Weitzendorf 
AuthorDate: Wed Sep 28 14:59:13 2022 +0200

OAK-9949 Offline Tail Compaction
---
 .../site/markdown/nodestore/segment/overview.md|  6 ++-
 .../apache/jackrabbit/oak/run/CompactCommand.java  | 23 +++
 .../oak/segment/aws/tool/AwsCompact.java   | 28 -
 .../oak/segment/azure/tool/AzureCompact.java   | 28 -
 .../oak/segment/CheckpointCompactor.java   | 46 +-
 .../jackrabbit/oak/segment/tool/Compact.java   | 28 -
 6 files changed, 129 insertions(+), 30 deletions(-)

diff --git a/oak-doc/src/site/markdown/nodestore/segment/overview.md 
b/oak-doc/src/site/markdown/nodestore/segment/overview.md
index 53f10542ab..499bcb3cf2 100644
--- a/oak-doc/src/site/markdown/nodestore/segment/overview.md
+++ b/oak-doc/src/site/markdown/nodestore/segment/overview.md
@@ -823,14 +823,16 @@ This option is optional and is disabled by default.
 ###  Compact
 
 ```
-java -jar oak-run.jar compact [--force] [--mmap] [--compactor] [--threads] 
SOURCE [--target-path DESTINATION] [--persistent-cache-path 
PERSISTENT_CACHE_PATH] [--persistent-cache-size-gb ]
+java -jar oak-run.jar compact [--force] [--mmap] [--tail] [--compactor] 
[--threads] SOURCE [--target-path DESTINATION] [--persistent-cache-path 
PERSISTENT_CACHE_PATH] [--persistent-cache-size-gb ]
 ```
 
 The `compact` command performs offline compaction of the local/remote Segment 
Store at `SOURCE`. 
 `SOURCE` must be a valid path/uri to an existing Segment Store. Currently, 
Azure Segment Store and AWS Segment Store the supported remote Segment Stores. 
 Please refer to the [Remote Segment Stores](#remote-segment-stores) section 
for details on how to correctly specify connection URIs.
 
-If the optional `--force [Boolean]` argument is set to `true` the tool ignores 
a non-matching Segment Store version. *CAUTION*: this will upgrade the Segment 
Store to the 
+With the optional `--tail` flag, only tail compaction is performed instead of 
the full repository.
+
+If the optional `--force` flag is set, the tool ignores a non-matching Segment 
Store version. *CAUTION*: this will upgrade the Segment Store to the 
 latest version, which is incompatible with older versions. *There is no way to 
downgrade 
 an accidentally upgraded Segment Store*.  
 
diff --git 
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java 
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
index 47257982d3..f955278931 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
@@ -25,6 +25,7 @@ import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 import org.apache.jackrabbit.oak.run.commons.Command;
 import org.apache.jackrabbit.oak.segment.azure.tool.AzureCompact;
+import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
 import 
org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType;
 import org.apache.jackrabbit.oak.segment.aws.tool.AwsCompact;
 import org.apache.jackrabbit.oak.segment.tool.Compact;
@@ -48,12 +49,11 @@ class CompactCommand implements Command {
 "is always enforced and this option is ignored.")
 .withOptionalArg()
 .ofType(Boolean.class);
-OptionSpec forceArg = parser.accepts("force",
+OptionSpec forceArg = parser.accepts("force",
 "Force compaction and ignore a non matching segment store 
version. " +
 "CAUTION: this will upgrade the segment store to the 
latest version, " +
-"which is incompatible with older versions of Oak.")
-.withOptionalArg()
-.ofType(Boolean.class);
+"which is incompatible with older versions of Oak.");
+OptionSpec tailArg = parser.accepts("tail", "Use tail compaction 
instead of a full repository rewrite.");
 OptionSpec compactor = parser.accepts("compactor",
 "Allow the user to control compactor type to be used. Valid 
choices are \"classic\", \"diff\", \"parallel\". " +
 "While \"classic\" is slower, it might be more stable, 
due to lack of optimisations employed " +
@@ -110,10 +110,13 @@ class CompactCommand implements Command {
 .withTargetPath(targetPath.value(options))
 
.withPersistentCachePath(persistentCachePath.value(options))
 
.withPersistentCacheSi

[jackrabbit-oak] branch issues/OAK-10311 created (now 4000431899)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10311
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at 4000431899 OAK-10311 - Optimize SegmentBlob#equals for segment blobs 
that originate from the same blob store

This branch includes the following new commits:

 new 4000431899 OAK-10311 - Optimize SegmentBlob#equals for segment blobs 
that originate from the same blob store

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[jackrabbit-oak] 01/01: OAK-10311 - Optimize SegmentBlob#equals for segment blobs that originate from the same blob store

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-10311
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit 4000431899e3afb83240d085377398a2bd2afd58
Author: dulceanu 
AuthorDate: Fri Oct 13 11:31:00 2023 +0200

OAK-10311 - Optimize SegmentBlob#equals for segment blobs that originate 
from the same blob store
---
 .../java/org/apache/jackrabbit/oak/segment/SegmentBlob.java | 13 +
 1 file changed, 13 insertions(+)

diff --git 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
index 3bc438bcbb..25d525236a 100644
--- 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
+++ 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
@@ -30,6 +30,7 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.commons.properties.SystemPropertySupplier;
 import org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob;
 import org.apache.jackrabbit.oak.plugins.blob.datastore.InMemoryDataRecord;
 import org.apache.jackrabbit.oak.plugins.memory.AbstractBlob;
@@ -41,6 +42,10 @@ import org.jetbrains.annotations.Nullable;
  * A BLOB (stream of bytes). This is a record of type "VALUE".
  */
 public class SegmentBlob extends Record implements Blob {
+private static final boolean FAST_EQUALS_SAME_BLOBSTORE = 
SystemPropertySupplier
+.create("oak.segment.blob.fastEquals.same.blobstore", false)
+.formatSetMessage( (name, value) -> String.format("%s set to: %s", 
name, value) )
+.get();
 
 @Nullable
 private final BlobStore blobStore;
@@ -192,6 +197,14 @@ public class SegmentBlob extends Record implements Blob {
 }
 }
 
+if (FAST_EQUALS_SAME_BLOBSTORE) {
+if (blobStore != null && this.blobStore.equals(that.blobStore) 
&& this.isExternal() && that.isExternal()) {
+if (this.getContentIdentity() != null && 
that.getContentIdentity() != null) {
+return 
this.getContentIdentity().equals(that.getContentIdentity());
+}
+}
+}
+
 if (this.length() != that.length()) {
 return false;
 }

[jackrabbit-oak] branch issues/OAK-10472 updated (ffd7bf2650 -> 915a07cc59)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10472
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


 discard ffd7bf2650 OAK-10472 - Move ApproximateCounter from oak-core to a 
place where it can be used by oak-segment-tar
 add b2ffbb446c OAK-10463: Retrieve flatFileStore for indexing from 
baseFlatFileStore (#1135)
 add 915a07cc59 OAK-10472 - Move ApproximateCounter from oak-core to a 
place where it can be used by oak-segment-tar

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (ffd7bf2650)
\
 N -- N -- N   refs/heads/issues/OAK-10472 (915a07cc59)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

No new revisions were added by this update.

Summary of changes:
 .../jackrabbit/oak/commons/sort/ExternalSort.java  | 229 +++---
 .../jackrabbit/oak/commons/sort/package-info.java  |   2 +-
 .../oak/commons/sort/ExternalSortTest.java | 289 +
 .../jackrabbit/oak/commons/sort/test-file-2.csv|   6 +
 .../plugins/index/counter/ApproximateCounter.java  |   0
 .../jackrabbit/oak/index/IndexerSupport.java   |  51 +++
 .../indexer/document/DocumentStoreIndexerBase.java |  42 +--
 .../incrementalstore/IncrementalStoreIT.java   | 342 +++--
 .../oak/segment}/ApproximateCounter.java   |  24 +-
 .../jackrabbit/oak/segment/ParallelCompactor.java  |   1 -
 oak-store-spi/pom.xml  |   1 -
 11 files changed, 677 insertions(+), 310 deletions(-)
 copy {oak-store-spi => 
oak-core}/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/ApproximateCounter.java
 (100%)
 rename 
{oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter => 
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment}/ApproximateCounter.java
 (97%)

[jackrabbit-oak] branch issues/OAK-10472 created (now ffd7bf2650)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10472
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at ffd7bf2650 OAK-10472 - Move ApproximateCounter from oak-core to a 
place where it can be used by oak-segment-tar

This branch includes the following new commits:

 new ffd7bf2650 OAK-10472 - Move ApproximateCounter from oak-core to a 
place where it can be used by oak-segment-tar

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[jackrabbit-oak] 01/01: OAK-10472 - Move ApproximateCounter from oak-core to a place where it can be used by oak-segment-tar

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-10472
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit ffd7bf2650a350f43a25f88413fda9ea8b05c19b
Author: dulceanu 
AuthorDate: Fri Oct 6 17:52:43 2023 +0200

OAK-10472 - Move ApproximateCounter from oak-core to a place where it can 
be used by oak-segment-tar
---
 oak-core/pom.xml| 1 -
 oak-segment-tar/pom.xml | 2 ++
 oak-store-spi/pom.xml   | 1 +
 .../apache/jackrabbit/oak/plugins/index/counter/ApproximateCounter.java | 0
 4 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/oak-core/pom.xml b/oak-core/pom.xml
index 4a59a93528..0f24f7c1b5 100644
--- a/oak-core/pom.xml
+++ b/oak-core/pom.xml
@@ -55,7 +55,6 @@
   org.apache.jackrabbit.oak.plugins.identifier,
   org.apache.jackrabbit.oak.plugins.index,
   org.apache.jackrabbit.oak.plugins.index.aggregate,
-  org.apache.jackrabbit.oak.plugins.index.counter,
   org.apache.jackrabbit.oak.plugins.index.cursor,
   org.apache.jackrabbit.oak.plugins.index.fulltext,
   org.apache.jackrabbit.oak.plugins.index.importer,
diff --git a/oak-segment-tar/pom.xml b/oak-segment-tar/pom.xml
index d345cf2922..69f83a04ef 100644
--- a/oak-segment-tar/pom.xml
+++ b/oak-segment-tar/pom.xml
@@ -436,6 +436,8 @@
 org.apache.jackrabbit
 oak-core
 ${project.version}
+tests
+test
 
 
 
diff --git a/oak-store-spi/pom.xml b/oak-store-spi/pom.xml
index 600465085a..2b0cc8b0f7 100644
--- a/oak-store-spi/pom.xml
+++ b/oak-store-spi/pom.xml
@@ -45,6 +45,7 @@
 org.apache.jackrabbit.oak.spi.filter,
 org.apache.jackrabbit.oak.spi.lifecycle,
 org.apache.jackrabbit.oak.spi.state,
+org.apache.jackrabbit.oak.plugins.index.counter,
 org.apache.jackrabbit.oak.plugins.memory,
 org.apache.jackrabbit.oak.plugins.value,
 org.apache.jackrabbit.oak.plugins.value.jcr
diff --git 
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/ApproximateCounter.java
 
b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/ApproximateCounter.java
similarity index 100%
rename from 
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/ApproximateCounter.java
rename to 
oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/ApproximateCounter.java

[jackrabbit-oak] branch issues/OAK-9922 deleted (was b375d8fa57)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-9922
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


 was b375d8fa57 OAK-9922 Parallel Compaction Export package 
org.apache.jackrabbit.oak.plugins.index.counter from oak-core

The revisions that were on this branch are still contained in
other references; therefore, this change does not discard any commits
from the repository.

[jackrabbit-oak] branch trunk updated (0b6e538736 -> c6b4b08da6)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


from 0b6e538736 OAK-10453 - Pipelined strategy: enforce size limit on 
memory taken by objects in the queue between download and transform thread 
(#1130)
 add c6b4b08da6 OAK-9922: Parallel Compaction (#1043)

No new revisions were added by this update.

Summary of changes:
 oak-core/pom.xml   |   1 +
 .../site/markdown/nodestore/segment/overview.md|  27 +-
 .../apache/jackrabbit/oak/run/CompactCommand.java  |  50 +--
 .../oak/segment/aws/tool/AwsCompact.java   |  34 +-
 .../oak/segment/aws/tool/AwsToolUtils.java |  19 +-
 .../oak/segment/azure/tool/AzureCompact.java   |  21 +-
 .../oak/segment/azure/tool/ToolUtils.java  |  26 +-
 oak-segment-tar/pom.xml|   2 -
 .../oak/segment/CheckpointCompactor.java   |  31 +-
 .../jackrabbit/oak/segment/ClassicCompactor.java   |  21 +-
 .../jackrabbit/oak/segment/CompactorUtils.java |  21 +-
 .../oak/segment/DefaultSegmentWriterBuilder.java   |  71 ++--
 .../jackrabbit/oak/segment/ParallelCompactor.java  | 370 
 .../apache/jackrabbit/oak/segment/RecordCache.java | 113 +++---
 .../oak/segment/SegmentBufferWriterPool.java   | 379 ++---
 .../oak/segment/compaction/SegmentGCOptions.java   |  47 ++-
 .../segment/file/AbstractCompactionStrategy.java   |  25 +-
 .../jackrabbit/oak/segment/file/FileStore.java |   3 +-
 .../oak/segment/file/GCNodeWriteMonitor.java   |   2 +
 .../jackrabbit/oak/segment/file/PriorityCache.java | 272 +--
 .../jackrabbit/oak/segment/tool/Compact.java   |  22 +-
 ...java => AbstractCompactorExternalBlobTest.java} |  29 +-
 ...mpactorTest.java => AbstractCompactorTest.java} |  27 +-
 .../CheckpointCompactorExternalBlobTest.java   | 135 +---
 .../oak/segment/CheckpointCompactorTest.java   | 111 ++
 ...actorTestUtils.java => CompactorTestUtils.java} |  20 +-
 .../segment/ParallelCompactorExternalBlobTest.java |  63 
 .../oak/segment/ParallelCompactorTest.java |  63 
 .../jackrabbit/oak/segment/RecordCacheTest.java|  54 ++-
 .../oak/segment/SegmentBufferWriterPoolTest.java   |  23 +-
 .../segment/file/ConcurrentPriorityCacheTest.java  | 236 +
 31 files changed, 1629 insertions(+), 689 deletions(-)
 copy 
oak-upgrade/src/main/java/org/apache/jackrabbit/oak/segment/CheckpointAccessor.java
 => 
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/CompactorUtils.java
 (68%)
 create mode 100644 
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ParallelCompactor.java
 copy 
oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/{CheckpointCompactorExternalBlobTest.java
 => AbstractCompactorExternalBlobTest.java} (83%)
 copy 
oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/{CheckpointCompactorTest.java
 => AbstractCompactorTest.java} (83%)
 rename 
oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/{CheckpointCompactorTestUtils.java
 => CompactorTestUtils.java} (88%)
 create mode 100644 
oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ParallelCompactorExternalBlobTest.java
 create mode 100644 
oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ParallelCompactorTest.java
 create mode 100644 
oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/ConcurrentPriorityCacheTest.java

[jackrabbit-oak] branch issues/OAK-3349 created (now 92d96801e2)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-3349
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at 92d96801e2 OAK-3349 Partial Compaction

This branch includes the following new commits:

 new 92d96801e2 OAK-3349 Partial Compaction

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[jackrabbit-oak] 01/01: OAK-9949 Offline Tail Compaction

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-9949
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit 5136d82b9a0ac9701d144fda47d6ad4cd5b68a27
Author: Lucas Weitzendorf 
AuthorDate: Wed Sep 28 14:59:13 2022 +0200

OAK-9949 Offline Tail Compaction
---
 .../site/markdown/nodestore/segment/overview.md|  6 ++-
 .../apache/jackrabbit/oak/run/CompactCommand.java  | 23 +++
 .../oak/segment/aws/tool/AwsCompact.java   | 28 -
 .../oak/segment/azure/tool/AzureCompact.java   | 28 -
 .../oak/segment/CheckpointCompactor.java   | 46 +-
 .../jackrabbit/oak/segment/tool/Compact.java   | 28 -
 6 files changed, 129 insertions(+), 30 deletions(-)

diff --git a/oak-doc/src/site/markdown/nodestore/segment/overview.md 
b/oak-doc/src/site/markdown/nodestore/segment/overview.md
index 53f10542ab..499bcb3cf2 100644
--- a/oak-doc/src/site/markdown/nodestore/segment/overview.md
+++ b/oak-doc/src/site/markdown/nodestore/segment/overview.md
@@ -823,14 +823,16 @@ This option is optional and is disabled by default.
 ###  Compact
 
 ```
-java -jar oak-run.jar compact [--force] [--mmap] [--compactor] [--threads] 
SOURCE [--target-path DESTINATION] [--persistent-cache-path 
PERSISTENT_CACHE_PATH] [--persistent-cache-size-gb ]
+java -jar oak-run.jar compact [--force] [--mmap] [--tail] [--compactor] 
[--threads] SOURCE [--target-path DESTINATION] [--persistent-cache-path 
PERSISTENT_CACHE_PATH] [--persistent-cache-size-gb ]
 ```
 
 The `compact` command performs offline compaction of the local/remote Segment 
Store at `SOURCE`. 
 `SOURCE` must be a valid path/uri to an existing Segment Store. Currently, 
Azure Segment Store and AWS Segment Store the supported remote Segment Stores. 
 Please refer to the [Remote Segment Stores](#remote-segment-stores) section 
for details on how to correctly specify connection URIs.
 
-If the optional `--force [Boolean]` argument is set to `true` the tool ignores 
a non-matching Segment Store version. *CAUTION*: this will upgrade the Segment 
Store to the 
+With the optional `--tail` flag, only tail compaction is performed instead of 
the full repository.
+
+If the optional `--force` flag is set, the tool ignores a non-matching Segment 
Store version. *CAUTION*: this will upgrade the Segment Store to the 
 latest version, which is incompatible with older versions. *There is no way to 
downgrade 
 an accidentally upgraded Segment Store*.  
 
diff --git 
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java 
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
index 47257982d3..f955278931 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
@@ -25,6 +25,7 @@ import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 import org.apache.jackrabbit.oak.run.commons.Command;
 import org.apache.jackrabbit.oak.segment.azure.tool.AzureCompact;
+import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
 import 
org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType;
 import org.apache.jackrabbit.oak.segment.aws.tool.AwsCompact;
 import org.apache.jackrabbit.oak.segment.tool.Compact;
@@ -48,12 +49,11 @@ class CompactCommand implements Command {
 "is always enforced and this option is ignored.")
 .withOptionalArg()
 .ofType(Boolean.class);
-OptionSpec forceArg = parser.accepts("force",
+OptionSpec forceArg = parser.accepts("force",
 "Force compaction and ignore a non matching segment store 
version. " +
 "CAUTION: this will upgrade the segment store to the 
latest version, " +
-"which is incompatible with older versions of Oak.")
-.withOptionalArg()
-.ofType(Boolean.class);
+"which is incompatible with older versions of Oak.");
+OptionSpec tailArg = parser.accepts("tail", "Use tail compaction 
instead of a full repository rewrite.");
 OptionSpec compactor = parser.accepts("compactor",
 "Allow the user to control compactor type to be used. Valid 
choices are \"classic\", \"diff\", \"parallel\". " +
 "While \"classic\" is slower, it might be more stable, 
due to lack of optimisations employed " +
@@ -110,10 +110,13 @@ class CompactCommand implements Command {
 .withTargetPath(targetPath.value(options))
 
.withPersistentCachePath(persistentCachePath.value(options))
 
.withPersistentCacheSi

[jackrabbit-oak] branch issues/OAK-9949 created (now 5136d82b9a)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-9949
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at 5136d82b9a OAK-9949 Offline Tail Compaction

This branch includes the following new commits:

 new 5136d82b9a OAK-9949 Offline Tail Compaction

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[jackrabbit-oak] branch issues/OAK-9922 updated (9cd831128d -> b375d8fa57)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-9922
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


omit 9cd831128d OAK-9922 Parallel Compaction Export package 
org.apache.jackrabbit.oak.plugins.index.counter from oak-core
omit 73f07ceb77 OAK-9922 Parallel Compaction
 add 2c83efbefc OAK-10334: Node.addMixin() may overwrite existing mixins
 add 0b8223f113 OAK-10334: Node.addMixin() may overwrite existing mixins
 add ce2eb51d4b OAK-301 Improve documentation
 add 6016723174 OAK-10333 Improved logging for queries that that traverse 
more than 10'000 nodes
 add 7ba70edd2b Merge pull request #1012 from apache/OAK-10333
 add 958e1ff69d OAK-10372: oak-search-elastic: similarity queries produce 
no relevant results (#1046)
 add 09e1ff4dd7 OAK-10347 : Adding small util class for exporting a tree as 
flat file (#1021)
 add e421f1adf6 OAK-10381: DocumentStoreIndexerIT failure resolved (#1052)
 add d6c402c944 OAK-10367: update org.apache.sling.testing.osgi-mock 
dependency (#1044)
 add 1982aebf69 OAK-10379: DynamicGroupsTest#testCrossIDPMembership() fails 
occasionally (#1054)
 add c10ce87e3f OAK-10385: segment-azure: add javax.activation-api to test 
class path (#1055)
 add f2a8101b28 OAK-10386: Bump netty dependency from 4.1.52.Final to 
4.1.96.Final (#1056)
 add b83ccfe6c0 OAK-10389: remove commons-lang dependency (#1058)
 add 49204bd6ac OAK-10390: Update commons-io dependency to 2.13.0 (#1059)
 add 860576b593 OAK-10391: improve 
TarFilesTest.testCollectBlobReferencesWithGenerationFilter diagnostics (#1060)
 add 10b8be63ed OAK-10392: Update (shaded) Guava to 32.1.2 (#1061)
 add f6e2728914 OAK-10394: Update commons-lang3 dependency to 3.13.0 (#1062)
 add 6cd46d48d0 OAK-10395: Update commons-codec dependency to 1.16.0 (#1063)
 add 4807553544 OAK-10396: segment-aws: remove unused commons-lang3 
dependency (#1064)
 add 713df5fc7b OAK-10397: oak-benchmarks/oak-it-osgi: update 
commons-compress dependency to 1.23.0 (#1065)
 add de516dd49f OAK-10400: auth-ldap/benchmarks: use default dependency for 
commons-lang3 (#1067)
 add 57cd862d47 OAK-10401: search-mt/solr-osgi: use default versions of 
commons-math3 (#1068)
 add 5b75d52f64 OAK-10402: benchmarks: remove commons-math dependency (use 
math3 instead) (#1069)
 add da22ffe40e OAK-10403: update Tomcat JDBC dependency to 8.5.92 (#1072)
 add ac3d8066df OAK-10272 reverse OAK-10022
 add b724dee4a3 Merge pull request #1070 from 
jelmini/OAK-10272_reverse_OAK-10022
 add 501f27c9ad OAK-10404: improve TarFilesTest.testCollectBlobReferences 
diagnostics (#1074)
 add 7171b3eab0 OAK-10405: document-store: potential NPE in 
AbstractTwoNodeTest teardown (#1076)
 add 3800a6514b OAK-10407: run/lucene: update gson dependencies to 2.10.1 
(#1078)
 add cf521f072e Merge branch 'trunk' into OAK-10334
 add 028e8d3618 OAK-10334: Node.addMixin() may overwrite existing mixins
 add 6d2fbee0f8 OAK-10412: AbstractTwoNodeTest: check preconditions in 
setup (#1081)
 add ae7a422c5b OAK-10415: AbstractTwoNodeTest: make teardown more robust 
(#1083)
 add 03f034c04c OAK-10416: AbstractTwoNodeTest: ensure RDB fixture uses 
empty DB (#1084)
 add fbf7413a01 OAK-10414: RDB*Store: update Derby dependency to 10.15.2.0 
(#1082)
 add 46109d1d84 Merge branch 'trunk' into OAK-10334
 add 5b05f24ecb OAK-10357: Path option for documentstore-check
 add 9299e4f8f1 Merge pull request #1031 from mreutegg/OAK-10357
 add 27cee262b3 OAK-10419: Release Oak 1.56.0 - Candidate Release Notes 
(#1088)
 add afb9d8d303 [maven-release-plugin] prepare release jackrabbit-oak-1.56.0
 add 04ad212826 [maven-release-plugin] prepare for next development 
iteration
 add 921467d8ef OAK-10399 Automatically pick a merged index over multiple 
levels
 add 79bd999aef OAK-10399 Automatically pick a merged index over multiple 
levels
 add cb5df597c8 OAK-10420 Tool to compare Lucene index content
 add 81e49e5f8d Revert "OAK-10399 Automatically pick a merged index over 
multiple levels"
 add 1d50d20bd6 Revert "Revert "OAK-10399 Automatically pick a merged index 
over multiple levels""
 add 482c5ac821 Revert "OAK-10420 Tool to compare Lucene index content"
 add 77f36b3252 OAK-10399 Automatically pick a merged index over multiple 
levels
 add 009ef2c185 Merge pull request #1066 from apache/OAK-10399
 add 8c3116832b Clarified a pitfall with --merge-paths
 add e11d20cb3d Merge pull request #161 from sbluhm/patch-1
 add 5ee4afcaa9 OAK-10419: Release Oak 1.56.0  - fix project versions
 add e91361faa5 Merge branch 'trunk' into OAK-10334
 add dcb47fc0bc Merge pull request #1011 from mreutegg/OAK-10334
 add 634cb27d07 OAK-10413: DocumentStoreIndexerIT hanging
 add 42eee1060b OAK-10427: Clear c

[jackrabbit-oak] branch issues/OAK-9922 updated (73f07ceb77 -> 9cd831128d)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-9922
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


from 73f07ceb77 OAK-9922 Parallel Compaction
 add 9cd831128d OAK-9922 Parallel Compaction Export package 
org.apache.jackrabbit.oak.plugins.index.counter from oak-core

No new revisions were added by this update.

Summary of changes:
 oak-core/pom.xml | 1 +
 1 file changed, 1 insertion(+)

[jackrabbit-oak] branch issues/OAK-9922 updated (7d9196f713 -> 73f07ceb77)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-9922
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


 discard 7d9196f713 OAK-9922 Parallel Compaction
 add 73f07ceb77 OAK-9922 Parallel Compaction

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (7d9196f713)
\
 N -- N -- N   refs/heads/issues/OAK-9922 (73f07ceb77)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

No new revisions were added by this update.

Summary of changes:
 .../strategy/UniqueEntryStoreStrategy.java | 42 +++---
 .../index/counter/ApproximateCounterTest.java  | 20 +--
 .../strategy/ContentMirrorStoreStrategyTest.java   |  4 +--
 .../jackrabbit/oak/segment/file/PriorityCache.java |  6 ++--
 4 files changed, 36 insertions(+), 36 deletions(-)

[jackrabbit-oak] branch issues/OAK-9922 updated (b0a7bbada8 -> 7d9196f713)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-9922
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


omit b0a7bbada8 OAK-10181: Flaky test: DataStoreBlobStoreStatsTest - ensure 
that the two created blobs are different (#1045)
omit bf1c1a093b OAK-9922 Parallel Compaction Address further comments
omit 625fd1c771 OAK-9922 Parallel Compaction Address review comments
omit 9402ecb9fe OAK-9922 Parallel Compaction Make SegmentBufferWriterPool 
implementation a configurable option
omit 4ec09454fe OAK-9922 Parallel Compaction
 add 045d982b39 OAK-10003: Upgrade tika to 1.28.5 (#758)
 add 191687c2b4 OAK-10181: Flaky test: DataStoreBlobStoreStatsTest - ensure 
that the two created blobs are different (#1045)
 add e64acee349 OAK-10316: Importer exceptions always should mention path - 
add missing SP character
 add dc19d269fa OAK-301 Improve documentation
 new 7d9196f713 OAK-9922 Parallel Compaction

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (b0a7bbada8)
\
 N -- N -- N   refs/heads/issues/OAK-9922 (7d9196f713)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../plugins/index/counter/ApproximateCounter.java  |   9 +-
 .../plugins/index/counter/NodeCounterEditor.java   |   1 -
 .../oak/plugins/index/counter/jmx/NodeCounter.java |   2 +-
 .../index/property/PropertyIndexInfoProvider.java  |   2 +-
 .../strategy/ContentMirrorStoreStrategy.java   |   2 +-
 .../strategy/UniqueEntryStoreStrategy.java |  44 ++---
 .../index/counter}/ApproximateCounterTest.java |  29 ++--
 .../strategy/ContentMirrorStoreStrategyTest.java   |   6 +-
 oak-doc/src/site/markdown/query/lucene.md  |   8 +-
 .../jackrabbit/oak/osgi/TikaExtractionOsgiIT.java  |   6 +-
 oak-it-osgi/src/test/resources/versions.properties |   6 +-
 .../jackrabbit/oak/jcr/xml/ImporterImpl.java   |   2 +-
 oak-parent/pom.xml |   2 +-
 .../apache/jackrabbit/oak/run/CompactCommand.java  |   2 +-
 oak-segment-tar/pom.xml|   2 -
 .../oak/segment/CheckpointCompactor.java   |   1 -
 .../jackrabbit/oak/segment/ParallelCompactor.java  |   2 +-
 .../apache/jackrabbit/oak/segment/RecordCache.java |   7 +-
 .../oak/segment/SegmentBufferWriterPool.java   |  18 +-
 .../segment/file/AbstractCompactionStrategy.java   |   8 +-
 .../jackrabbit/oak/segment/file/FileStore.java |   4 +-
 .../oak/segment/file/FileStoreBuilder.java |  14 +-
 .../oak/segment/file/GCNodeWriteMonitor.java   |  68 
 .../jackrabbit/oak/segment/memory/MemoryStore.java |  14 +-
 .../oak/segment/CompactionAndCleanupIT.java|  13 +-
 .../oak/segment/RecordCacheStatsTest.java  |  16 +-
 .../oak/segment/SegmentBufferWriterPoolTest.java   |  44 +++--
 .../oak/segment/file/PriorityCacheTest.java|  20 ++-
 .../oak/plugins/index/ApproximateCounter.java  | 184 -
 29 files changed, 194 insertions(+), 342 deletions(-)
 rename {oak-store-spi/src/test/java/org/apache/jackrabbit/oak/plugins/index => 
oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/counter}/ApproximateCounterTest.java
 (94%)
 delete mode 100644 
oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/index/ApproximateCounter.java

[jackrabbit-oak] 03/04: OAK-9922 Parallel Compaction Address review comments

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-9922
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit 625fd1c7716d4db05247fc611ca32e770137e3c8
Author: Lucas Weitzendorf 
AuthorDate: Fri Oct 14 13:35:51 2022 +0200

OAK-9922 Parallel Compaction
Address review comments
---
 .../plugins/index/counter/ApproximateCounter.java  |   9 +-
 .../plugins/index/counter/NodeCounterEditor.java   |   1 +
 .../oak/plugins/index/counter/jmx/NodeCounter.java |   2 +-
 .../index/property/PropertyIndexInfoProvider.java  |   2 +-
 .../strategy/ContentMirrorStoreStrategy.java   |   2 +-
 .../strategy/UniqueEntryStoreStrategy.java |   2 +-
 .../strategy/ContentMirrorStoreStrategyTest.java   |   2 +-
 .../apache/jackrabbit/oak/run/CompactCommand.java  |   4 +-
 .../oak/segment/aws/tool/AwsCompact.java   |   2 +-
 .../oak/segment/azure/tool/AzureCompact.java   |   2 +-
 oak-segment-tar/pom.xml|   2 +
 .../oak/segment/CheckpointCompactor.java   |   4 +-
 .../jackrabbit/oak/segment/ClassicCompactor.java   |  11 +-
 .../jackrabbit/oak/segment/CompactorUtils.java |  35 +++
 .../oak/segment/DefaultSegmentWriterBuilder.java   |   9 +-
 .../jackrabbit/oak/segment/ParallelCompactor.java  |  19 +-
 .../apache/jackrabbit/oak/segment/RecordCache.java |  32 +--
 .../oak/segment/SegmentBufferWriterPool.java   |   4 +
 .../segment/file/AbstractCompactionStrategy.java   |   8 +-
 .../jackrabbit/oak/segment/file/FileStore.java |  11 +-
 .../oak/segment/file/FileStoreBuilder.java |  14 +-
 .../oak/segment/file/GCNodeWriteMonitor.java   |  13 +-
 .../jackrabbit/oak/segment/file/PriorityCache.java | 194 +
 .../jackrabbit/oak/segment/memory/MemoryStore.java |  18 +-
 .../jackrabbit/oak/segment/tool/Compact.java   |   2 +-
 .../segment/AbstractCompactorExternalBlobTest.java |   3 +-
 .../oak/segment/AbstractCompactorTest.java |   9 +-
 .../jackrabbit/oak/segment/NodeRecordTest.java |   2 +-
 .../segment/ParallelCompactorExternalBlobTest.java |  20 +-
 .../oak/segment/ParallelCompactorTest.java |  20 +-
 .../oak/segment/RecordCacheStatsTest.java  |   7 +-
 .../jackrabbit/oak/segment/RecordCacheTest.java|  54 -
 .../segment/file/ConcurrentPriorityCacheTest.java  | 236 +
 .../oak/segment/file/PriorityCacheTest.java|  20 +-
 .../oak/plugins/index}/ApproximateCounter.java |   2 +-
 .../oak/plugins/index}/ApproximateCounterTest.java |   9 +-
 36 files changed, 581 insertions(+), 205 deletions(-)

diff --git 
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/ApproximateCounter.java
 
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/ApproximateCounter.java
index bfad0a2561..a62070a6c8 100644
--- 
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/ApproximateCounter.java
+++ 
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/ApproximateCounter.java
@@ -18,17 +18,18 @@
  */
 package org.apache.jackrabbit.oak.plugins.index.counter;
 
-import java.util.Random;
-import java.util.UUID;
-
 import org.apache.jackrabbit.oak.api.PropertyState;
 import org.apache.jackrabbit.oak.api.Type;
 import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
 import org.apache.jackrabbit.oak.spi.state.NodeState;
 
+import java.util.Random;
+import java.util.UUID;
+
 /**
- * An approximate counter algorithm.
+ * Moved to oak-store-spi
  */
+@Deprecated
 public class ApproximateCounter {
 
 public static final String COUNT_PROPERTY_PREFIX = ":count_";
diff --git 
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditor.java
 
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditor.java
index cde5acec08..6df3524341 100644
--- 
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditor.java
+++ 
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditor.java
@@ -22,6 +22,7 @@ import org.apache.jackrabbit.oak.api.CommitFailedException;
 import org.apache.jackrabbit.oak.api.PropertyState;
 import org.apache.jackrabbit.oak.api.Type;
 import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.index.ApproximateCounter;
 import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback;
 import org.apache.jackrabbit.oak.plugins.index.counter.jmx.NodeCounter;
 import org.apache.jackrabbit.oak.plugins.index.property.Multiplexers;
diff --git 
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/jmx/NodeCounter.java
 
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/jmx/NodeCounter.java
index fa11e76f27..5815abb350 100644
--- 
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/jmx/NodeCo

[jackrabbit-oak] 02/04: OAK-9922 Parallel Compaction Make SegmentBufferWriterPool implementation a configurable option

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-9922
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit 9402ecb9fe028ad615f7a8a965d17f5b267d4888
Author: Lucas Weitzendorf 
AuthorDate: Tue Oct 11 16:32:42 2022 +0200

OAK-9922 Parallel Compaction
Make SegmentBufferWriterPool implementation a configurable option
---
 .../oak/segment/DefaultSegmentWriterBuilder.java   |  60 ++--
 .../oak/segment/SegmentBufferWriterPool.java   | 314 +
 .../jackrabbit/oak/segment/file/FileStore.java |  12 +-
 .../jackrabbit/oak/segment/memory/MemoryStore.java |  16 +-
 .../jackrabbit/oak/segment/NodeRecordTest.java |   2 +-
 .../segment/ParallelCompactorExternalBlobTest.java |   2 +-
 .../oak/segment/ParallelCompactorTest.java |   2 +-
 .../oak/segment/SegmentBufferWriterPoolTest.java   |  25 +-
 8 files changed, 310 insertions(+), 123 deletions(-)

diff --git 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java
 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java
index 94ae636a4b..4d70c23c2c 100644
--- 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java
+++ 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java
@@ -20,6 +20,7 @@
 package org.apache.jackrabbit.oak.segment;
 
 import static 
org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull;
+import static 
org.apache.jackrabbit.oak.segment.SegmentBufferWriterPool.PoolType;
 
 import org.apache.jackrabbit.guava.common.base.Supplier;
 import org.apache.jackrabbit.guava.common.base.Suppliers;
@@ -29,10 +30,11 @@ import 
org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
 import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration;
 import org.apache.jackrabbit.oak.segment.memory.MemoryStore;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Builder for building {@link DefaultSegmentWriter} instances.
- * The returned instances are thread safe if {@link #withWriterPool()}
+ * The returned instances are thread safe if {@link #withWriterPool(PoolType)}
  * was specified and not thread sage if {@link #withoutWriterPool()}
  * was specified (default).
  * 
@@ -56,7 +58,7 @@ public final class DefaultSegmentWriterBuilder {
 @NotNull
 private Supplier generation = 
Suppliers.ofInstance(GCGeneration.NULL);
 
-private boolean pooled = false;
+private PoolType poolType = null;
 
 @NotNull
 private WriterCacheManager cacheManager = new WriterCacheManager.Default();
@@ -81,7 +83,7 @@ public final class DefaultSegmentWriterBuilder {
  * If {@link #withoutWriterPool()} was specified all segments will be 
written
  * at the generation that {@code generation.get()} returned at the time
  * any of the {@code build()} methods is called.
- * If {@link #withWriterPool()} was specified a segments will be written
+ * If {@link #withWriterPool(PoolType)} ()} was specified, segments will 
be written
  * at the generation that {@code generation.get()} returns when a new 
segment
  * is created by the returned writer.
  */
@@ -106,8 +108,8 @@ public final class DefaultSegmentWriterBuilder {
  * The returned instance is thread safe.
  */
 @NotNull
-public DefaultSegmentWriterBuilder withWriterPool() {
-this.pooled = true;
+public DefaultSegmentWriterBuilder withWriterPool(PoolType writerType) {
+this.poolType = writerType;
 return this;
 }
 
@@ -117,7 +119,7 @@ public final class DefaultSegmentWriterBuilder {
  */
 @NotNull
 public DefaultSegmentWriterBuilder withoutWriterPool() {
-this.pooled = false;
+this.poolType = null;
 return this;
 }
 
@@ -151,7 +153,7 @@ public final class DefaultSegmentWriterBuilder {
 store.getSegmentIdProvider(),
 store.getBlobStore(),
 cacheManager,
-createWriter(store, pooled),
+createWriter(store, poolType),
 store.getBinariesInlineThreshold()
 );
 }
@@ -204,47 +206,27 @@ public final class DefaultSegmentWriterBuilder {
 store.getSegmentIdProvider(),
 store.getBlobStore(),
 cacheManager,
-createWriter(store, pooled),
+createWriter(store, poolType),
 Segment.MEDIUM_LIMIT
 );
 }
 
 @NotNull
-private WriteOperationHandler createWriter(@NotNull FileStore store, 
boolean pooled) {
-if (pooled) {
-return new SegmentBufferWriterPool(
-store.getSegmentIdProvider(),
-store.getReader(),
-name,
-generation

[jackrabbit-oak] 04/04: OAK-9922 Parallel Compaction Address further comments

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-9922
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit bf1c1a093b7554260aa2d54d7a6917b64852fc34
Author: Lucas Weitzendorf 
AuthorDate: Fri Dec 16 13:21:33 2022 +0100

OAK-9922 Parallel Compaction
Address further comments
---
 .../apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java | 4 ++--
 .../org/apache/jackrabbit/oak/segment/SegmentBufferWriterPool.java | 7 +--
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java
 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java
index 5097d40680..b5d7e22b9e 100644
--- 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java
+++ 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java
@@ -56,7 +56,7 @@ public final class DefaultSegmentWriterBuilder {
 private final String name;
 
 @NotNull
-private Supplier generation = 
Suppliers.ofInstance(GCGeneration.NULL);
+private Supplier generation = () -> GCGeneration.NULL;
 
 private PoolType poolType = null;
 
@@ -99,7 +99,7 @@ public final class DefaultSegmentWriterBuilder {
  */
 @NotNull
 public DefaultSegmentWriterBuilder withGeneration(@NotNull GCGeneration 
generation) {
-this.generation = Suppliers.ofInstance(checkNotNull(generation));
+this.generation = () -> checkNotNull(generation);
 return this;
 }
 
diff --git 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPool.java
 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPool.java
index 984f66ae02..b982f94043 100644
--- 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPool.java
+++ 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterPool.java
@@ -173,12 +173,7 @@ public abstract class SegmentBufferWriterPool implements 
WriteOperationHandler {
 @NotNull
 private SegmentBufferWriter getWriter(@NotNull Thread thread, @NotNull 
GCGeneration gcGeneration) {
 SimpleImmutableEntry key = new SimpleImmutableEntry<>(thread, 
gcGeneration);
-SegmentBufferWriter writer = writers.get(key);
-if (writer == null) {
-writer = newWriter(gcGeneration);
-writers.put(key, writer);
-}
-return writer;
+return writers.computeIfAbsent(key, f -> newWriter(gcGeneration));
 }
 }
 

[jackrabbit-oak] branch issues/OAK-9922 created (now bf1c1a093b)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-9922
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at bf1c1a093b OAK-9922 Parallel Compaction Address further comments

This branch includes the following new commits:

 new 4ec09454fe OAK-9922 Parallel Compaction
 new 9402ecb9fe OAK-9922 Parallel Compaction Make SegmentBufferWriterPool 
implementation a configurable option
 new 625fd1c771 OAK-9922 Parallel Compaction Address review comments
 new bf1c1a093b OAK-9922 Parallel Compaction Address further comments

The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[jackrabbit-oak] 01/04: OAK-9922 Parallel Compaction

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-9922
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit 4ec09454fe88515bafd896e4e973e210fdb9cd25
Author: Lucas Weitzendorf 
AuthorDate: Wed Sep 7 13:11:32 2022 +0200

OAK-9922 Parallel Compaction
---
 .../site/markdown/nodestore/segment/overview.md|  27 +-
 .../apache/jackrabbit/oak/run/CompactCommand.java  |  52 +--
 .../oak/segment/aws/tool/AwsCompact.java   |  34 +-
 .../oak/segment/aws/tool/AwsToolUtils.java |  19 +-
 .../oak/segment/azure/tool/AzureCompact.java   |  21 +-
 .../oak/segment/azure/tool/ToolUtils.java  |  26 +-
 oak-segment-tar/pom.xml|   2 -
 .../oak/segment/CheckpointCompactor.java   |  32 +-
 .../jackrabbit/oak/segment/ClassicCompactor.java   |  18 +-
 .../jackrabbit/oak/segment/ParallelCompactor.java  | 373 +
 .../apache/jackrabbit/oak/segment/RecordCache.java | 114 +++
 .../oak/segment/SegmentBufferWriterPool.java   | 178 +++---
 .../oak/segment/compaction/SegmentGCOptions.java   |  47 ++-
 .../segment/file/AbstractCompactionStrategy.java   |  31 +-
 .../jackrabbit/oak/segment/file/FileStore.java |   2 +-
 .../oak/segment/file/GCNodeWriteMonitor.java   |  65 ++--
 .../jackrabbit/oak/segment/file/PriorityCache.java | 196 +++
 .../jackrabbit/oak/segment/tool/Compact.java   |  22 +-
 ...java => AbstractCompactorExternalBlobTest.java} |  30 +-
 ...mpactorTest.java => AbstractCompactorTest.java} |  18 +-
 .../CheckpointCompactorExternalBlobTest.java   | 135 +---
 .../oak/segment/CheckpointCompactorTest.java   | 111 ++
 .../oak/segment/CompactionAndCleanupIT.java|  13 +-
 ...actorTestUtils.java => CompactorTestUtils.java} |  20 +-
 .../segment/ParallelCompactorExternalBlobTest.java |  45 +++
 .../oak/segment/ParallelCompactorTest.java |  45 +++
 .../oak/segment/RecordCacheStatsTest.java  |  13 +-
 .../oak/segment/SegmentBufferWriterPoolTest.java   |  42 +--
 28 files changed, 1053 insertions(+), 678 deletions(-)

diff --git a/oak-doc/src/site/markdown/nodestore/segment/overview.md 
b/oak-doc/src/site/markdown/nodestore/segment/overview.md
index 7a02190b16..53f10542ab 100644
--- a/oak-doc/src/site/markdown/nodestore/segment/overview.md
+++ b/oak-doc/src/site/markdown/nodestore/segment/overview.md
@@ -288,6 +288,22 @@ TarMK GC #2: compacting 
checkpoints/5c45ca7b-5863-4679-a7c5-6056a999a6cd/root.
 TarMK GC #2: compacting root.
 ```
 
+#  How does 
compaction make use of multithreading?
+
+The parallel compactor adds an initial exploration phase to the compaction 
process, which scans and splits the content tree
+into multiple parts to be processed simultaneously. For this to be efficient, 
the tree is only expanded until a certain 
+number of nodes is reached, which is defined relative to the number of threads 
(main thread + compaction workers).
+
+```
+TarMK GC #2: compacting with 8 threads.
+TarMK GC #2: exploring content tree to find subtrees for parallel compaction.
+TarMK GC #2: target node count for expansion is 7000, based on 7 available 
workers.
+TarMK GC #2: Found 3 nodes at depth 1, target is 7000.
+TarMK GC #2: Found 48 nodes at depth 2, target is 7000.
+TarMK GC #2: Found 663 nodes at depth 3, target is 7000.
+TarMK GC #2: Found 66944 nodes at depth 4, target is 7000.
+```
+
 #  How does 
compaction work with concurrent writes?
 
 When compaction runs as part of online garbage collection, it has to work 
concurrently with the rest of the system.
@@ -807,24 +823,25 @@ This option is optional and is disabled by default.
 ###  Compact
 
 ```
-java -jar oak-run.jar compact [--force] [--mmap] [--compactor] SOURCE 
[--target-path DESTINATION] [--persistent-cache-path PERSISTENT_CACHE_PATH] 
[--persistent-cache-size-gb ]
+java -jar oak-run.jar compact [--force] [--mmap] [--compactor] [--threads] 
SOURCE [--target-path DESTINATION] [--persistent-cache-path 
PERSISTENT_CACHE_PATH] [--persistent-cache-size-gb ]
 ```
 
 The `compact` command performs offline compaction of the local/remote Segment 
Store at `SOURCE`. 
 `SOURCE` must be a valid path/uri to an existing Segment Store. Currently, 
Azure Segment Store and AWS Segment Store the supported remote Segment Stores. 
 Please refer to the [Remote Segment Stores](#remote-segment-stores) section 
for details on how to correctly specify connection URIs.
 
-If the optional `--force [Boolean]` argument is set to `true` the tool ignores 
a non 
-matching Segment Store version. *CAUTION*: this will upgrade the Segment Store 
to the 
+If the optional `--force [Boolean]` argument is set to `true` the tool ignores 
a non-matching Segment Store version. *CAUTION*: this will upgrade the Segment 
Store to the 
 latest version, which is incompatible with older versions. *There is no way to 
downgrade 
 an accidentally upgraded Segm

[jackrabbit-oak] 01/03: OAK-9922 - Parallel Compaction Solved conflicts due to guava changes

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit ed8494b5e3a90f0021f3cd6e3fc7112ec9ec6653
Merge: 951d69e57a 00967073c3
Author: dulceanu 
AuthorDate: Thu Jul 27 16:40:00 2023 +0300

OAK-9922 - Parallel Compaction
Solved conflicts due to guava changes

 .DS_Store  | Bin 0 -> 6148 bytes
 .../plugins/index/counter/ApproximateCounter.java  |   9 +-
 .../plugins/index/counter/NodeCounterEditor.java   |   1 +
 .../oak/plugins/index/counter/jmx/NodeCounter.java |   2 +-
 .../index/property/PropertyIndexInfoProvider.java  |   2 +-
 .../strategy/ContentMirrorStoreStrategy.java   |   2 +-
 .../strategy/UniqueEntryStoreStrategy.java |   2 +-
 .../strategy/ContentMirrorStoreStrategyTest.java   |   2 +-
 .../site/markdown/nodestore/segment/overview.md|  27 +-
 .../apache/jackrabbit/oak/run/CompactCommand.java  |  54 +--
 .../oak/segment/aws/tool/AwsCompact.java   |  34 +-
 .../oak/segment/aws/tool/AwsToolUtils.java |  19 +-
 .../oak/segment/azure/tool/AzureCompact.java   |  21 +-
 .../oak/segment/azure/tool/ToolUtils.java  |  26 +-
 .../oak/segment/CheckpointCompactor.java   |  32 +-
 .../jackrabbit/oak/segment/ClassicCompactor.java   |  21 +-
 .../jackrabbit/oak/segment/CompactorUtils.java |  35 ++
 .../oak/segment/DefaultSegmentWriterBuilder.java   |  90 +++--
 .../jackrabbit/oak/segment/ParallelCompactor.java  | 370 +++
 .../apache/jackrabbit/oak/segment/RecordCache.java | 119 +++
 .../oak/segment/SegmentBufferWriterPool.java   | 390 ++---
 .../oak/segment/compaction/SegmentGCOptions.java   |  47 ++-
 .../segment/file/AbstractCompactionStrategy.java   |  31 +-
 .../jackrabbit/oak/segment/file/FileStore.java |   7 +-
 .../oak/segment/file/FileStoreBuilder.java |  14 +-
 .../oak/segment/file/GCNodeWriteMonitor.java   |  66 ++--
 .../jackrabbit/oak/segment/file/PriorityCache.java | 275 +--
 .../jackrabbit/oak/segment/memory/MemoryStore.java |  14 +-
 .../jackrabbit/oak/segment/tool/Compact.java   |  22 +-
 ...java => AbstractCompactorExternalBlobTest.java} |  29 +-
 ...mpactorTest.java => AbstractCompactorTest.java} |  27 +-
 .../CheckpointCompactorExternalBlobTest.java   | 135 +--
 .../oak/segment/CheckpointCompactorTest.java   | 111 +-
 .../oak/segment/CompactionAndCleanupIT.java|  13 +-
 ...actorTestUtils.java => CompactorTestUtils.java} |  20 +-
 .../jackrabbit/oak/segment/NodeRecordTest.java |   4 +-
 .../segment/ParallelCompactorExternalBlobTest.java |  63 
 .../oak/segment/ParallelCompactorTest.java |  63 
 .../oak/segment/RecordCacheStatsTest.java  |  20 +-
 .../jackrabbit/oak/segment/RecordCacheTest.java|  54 ++-
 .../oak/segment/SegmentBufferWriterPoolTest.java   |  67 ++--
 .../segment/file/ConcurrentPriorityCacheTest.java  | 236 +
 .../oak/segment/file/PriorityCacheTest.java|  27 +-
 oak-shaded-guava/.DS_Store | Bin 0 -> 6148 bytes
 .../oak/plugins/index}/ApproximateCounter.java |   2 +-
 .../oak/plugins/index}/ApproximateCounterTest.java |   9 +-
 46 files changed, 1774 insertions(+), 840 deletions(-)

diff --cc .DS_Store
index 00,00..c9f58500aa
new file mode 100644
Binary files differ
diff --cc 
oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
index 212c7a0435,315ac68240..157c863b26
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
@@@ -18,17 -18,15 +18,16 @@@
  package org.apache.jackrabbit.oak.run;
  
  import java.io.File;
 -import com.google.common.base.StandardSystemProperty;
 +
- import org.apache.jackrabbit.guava.common.base.StandardSystemProperty;
  import joptsimple.OptionParser;
  import joptsimple.OptionSet;
  import joptsimple.OptionSpec;
  import org.apache.jackrabbit.oak.run.commons.Command;
+ import org.apache.jackrabbit.oak.segment.aws.tool.AwsCompact;
  import org.apache.jackrabbit.oak.segment.azure.tool.AzureCompact;
- import org.apache.jackrabbit.oak.segment.azure.tool.AzureCompact.Builder;
  import 
org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType;
- import org.apache.jackrabbit.oak.segment.aws.tool.AwsCompact;
  import org.apache.jackrabbit.oak.segment.tool.Compact;
++import org.apache.jackrabbit.guava.common.base.StandardSystemProperty;
  
  class CompactCommand implements Command {
  
diff --cc 
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java
index 94ae636a4b,c461084d29..8e222b9693
--- 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java
+++ 
b/oak-segment-tar/src/main/java/org/apache/jackrab

[jackrabbit-oak] 03/03: Merge branch 'trunk' of github.com:apache/jackrabbit-oak into trunk

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit 1643f1ba56284033042a2f965b599528854f04c7
Merge: c7aa4f0f7b c18d0c5145
Author: dulceanu 
AuthorDate: Thu Jul 27 16:42:47 2023 +0300

Merge branch 'trunk' of github.com:apache/jackrabbit-oak into trunk

 .../jackrabbit/oak/api/jmx/CacheStatsMBean.java| 16 +++---
 oak-auth-external/pom.xml  |  7 ---
 oak-authorization-cug/pom.xml  |  7 ---
 oak-authorization-principalbased/pom.xml   |  7 ---
 .../oak/benchmark/ReplicaCrashResilienceTest.java  |  4 +-
 oak-blob-cloud-azure/pom.xml   |  2 +-
 .../apache/jackrabbit/oak/commons/LongUtils.java   |  4 +-
 .../apache/jackrabbit/oak/commons/Profiler.java|  2 -
 oak-it-osgi/pom.xml|  5 --
 oak-it-osgi/test-bundles.xml   |  1 -
 oak-jcr/pom.xml|  7 ---
 oak-lucene/pom.xml |  7 ---
 oak-parent/pom.xml | 57 +-
 .../oak/run/osgi/JaasConfigSpiTest.groovy  |  2 +-
 .../oak/run/osgi/LuceneSupportTest.groovy  |  2 +-
 .../oak/run/osgi/SegmentNodeStoreConfigTest.groovy |  4 +-
 .../oak/run/osgi/TokenAuthenticationTest.groovy|  2 +-
 .../oak/console/commands/CdCommand.groovy  |  4 +-
 .../oak/console/commands/LsCommand.groovy  |  2 +-
 .../oak/console/commands/LuceneCommand.groovy  |  4 +-
 .../console/commands/PrintDocumentCommand.groovy   |  2 +-
 oak-search-mt/pom.xml  |  1 -
 oak-search/pom.xml |  5 --
 oak-segment-azure/pom.xml  |  2 +-
 oak-segment-tar/pom.xml|  8 ---
 oak-store-composite/pom.xml|  6 +++
 oak-store-document/pom.xml |  8 ---
 .../oak/plugins/document/DocumentNodeStore.java|  4 +-
 .../plugins/document/mongo/MongoDocumentStore.java |  4 +-
 oak-store-spi/pom.xml  | 10 
 30 files changed, 36 insertions(+), 160 deletions(-)

[jackrabbit-oak] branch trunk updated (c18d0c5145 -> 1643f1ba56)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


from c18d0c5145 OAK-10369: oak-segment-azure/oak-blob-cloud-azure leak 
Guava dependency - adjust import range (#1040)
 add 6545b7912c OAK-9922 Parallel Compaction
 add 2abd1e6895 Make SegmentBufferWriterPool implementation a configurable 
option
 add 38b6c78172 Address review comments
 add 00967073c3 Address further comments
 new ed8494b5e3 OAK-9922 - Parallel Compaction Solved conflicts due to 
guava changes
 new c7aa4f0f7b OAK-9922 - Parallel Compaction Merge branch 
'lweitzendorf-breadth-first-scan' into trunk
 new 1643f1ba56 Merge branch 'trunk' of github.com:apache/jackrabbit-oak 
into trunk

The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../resources/img/railroad/div-d.png => .DS_Store  | Bin 8289 -> 6148 bytes
 .../plugins/index/counter/ApproximateCounter.java  |   9 +-
 .../plugins/index/counter/NodeCounterEditor.java   |   1 +
 .../oak/plugins/index/counter/jmx/NodeCounter.java |   2 +-
 .../index/property/PropertyIndexInfoProvider.java  |   2 +-
 .../strategy/ContentMirrorStoreStrategy.java   |   2 +-
 .../strategy/UniqueEntryStoreStrategy.java |   2 +-
 .../strategy/ContentMirrorStoreStrategyTest.java   |   2 +-
 .../site/markdown/nodestore/segment/overview.md|  27 +-
 .../apache/jackrabbit/oak/run/CompactCommand.java  |  54 +--
 .../oak/segment/aws/tool/AwsCompact.java   |  34 +-
 .../oak/segment/aws/tool/AwsToolUtils.java |  19 +-
 .../oak/segment/azure/tool/AzureCompact.java   |  21 +-
 .../oak/segment/azure/tool/ToolUtils.java  |  26 +-
 .../oak/segment/CheckpointCompactor.java   |  32 +-
 .../jackrabbit/oak/segment/ClassicCompactor.java   |  21 +-
 .../jackrabbit/oak/segment/CompactorUtils.java |  21 +-
 .../oak/segment/DefaultSegmentWriterBuilder.java   |  90 +++--
 .../jackrabbit/oak/segment/ParallelCompactor.java  | 370 +++
 .../apache/jackrabbit/oak/segment/RecordCache.java | 119 +++
 .../oak/segment/SegmentBufferWriterPool.java   | 390 ++---
 .../oak/segment/compaction/SegmentGCOptions.java   |  47 ++-
 .../segment/file/AbstractCompactionStrategy.java   |  31 +-
 .../jackrabbit/oak/segment/file/FileStore.java |   7 +-
 .../oak/segment/file/FileStoreBuilder.java |  14 +-
 .../oak/segment/file/GCNodeWriteMonitor.java   |  66 ++--
 .../jackrabbit/oak/segment/file/PriorityCache.java | 275 +--
 .../jackrabbit/oak/segment/memory/MemoryStore.java |  14 +-
 .../jackrabbit/oak/segment/tool/Compact.java   |  22 +-
 ...java => AbstractCompactorExternalBlobTest.java} |  29 +-
 ...mpactorTest.java => AbstractCompactorTest.java} |  27 +-
 .../CheckpointCompactorExternalBlobTest.java   | 135 +--
 .../oak/segment/CheckpointCompactorTest.java   | 111 +-
 .../oak/segment/CompactionAndCleanupIT.java|  13 +-
 ...actorTestUtils.java => CompactorTestUtils.java} |  20 +-
 .../jackrabbit/oak/segment/NodeRecordTest.java |   4 +-
 .../segment/ParallelCompactorExternalBlobTest.java |  63 
 .../oak/segment/ParallelCompactorTest.java |  63 
 .../oak/segment/RecordCacheStatsTest.java  |  20 +-
 .../jackrabbit/oak/segment/RecordCacheTest.java|  54 ++-
 .../oak/segment/SegmentBufferWriterPoolTest.java   |  67 ++--
 .../segment/file/ConcurrentPriorityCacheTest.java  | 236 +
 .../oak/segment/file/PriorityCacheTest.java|  27 +-
 .../div-d.png => oak-shaded-guava/.DS_Store| Bin 8289 -> 6148 bytes
 .../oak/plugins/index}/ApproximateCounter.java |   2 +-
 .../oak/plugins/index}/ApproximateCounterTest.java |   9 +-
 46 files changed, 1752 insertions(+), 848 deletions(-)
 copy oak-doc/src/site/resources/img/railroad/div-d.png => .DS_Store (64%)
 copy 
oak-upgrade/src/main/java/org/apache/jackrabbit/oak/segment/CheckpointAccessor.java
 => 
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/CompactorUtils.java
 (68%)
 create mode 100644 
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ParallelCompactor.java
 copy 
oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/{CheckpointCompactorExternalBlobTest.java
 => AbstractCompactorExternalBlobTest.java} (83%)
 copy 
oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/{CheckpointCompactorTest.java
 => AbstractCompactorTest.java} (83%)
 rename 
oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/{CheckpointCompactorTestUtils.java
 => CompactorTestUtils.java} (88%)
 create mode 100644 
oak-segment-tar/src/test/java/org/apache/jackr

[jackrabbit-oak] 02/03: OAK-9922 - Parallel Compaction Merge branch 'lweitzendorf-breadth-first-scan' into trunk

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit c7aa4f0f7b19567544a633a1e2df6e10fb1b42f9
Merge: 951d69e57a ed8494b5e3
Author: dulceanu 
AuthorDate: Thu Jul 27 16:40:37 2023 +0300

OAK-9922 - Parallel Compaction
Merge branch 'lweitzendorf-breadth-first-scan' into trunk

 .DS_Store  | Bin 0 -> 6148 bytes
 .../plugins/index/counter/ApproximateCounter.java  |   9 +-
 .../plugins/index/counter/NodeCounterEditor.java   |   1 +
 .../oak/plugins/index/counter/jmx/NodeCounter.java |   2 +-
 .../index/property/PropertyIndexInfoProvider.java  |   2 +-
 .../strategy/ContentMirrorStoreStrategy.java   |   2 +-
 .../strategy/UniqueEntryStoreStrategy.java |   2 +-
 .../strategy/ContentMirrorStoreStrategyTest.java   |   2 +-
 .../site/markdown/nodestore/segment/overview.md|  27 +-
 .../apache/jackrabbit/oak/run/CompactCommand.java  |  54 +--
 .../oak/segment/aws/tool/AwsCompact.java   |  34 +-
 .../oak/segment/aws/tool/AwsToolUtils.java |  19 +-
 .../oak/segment/azure/tool/AzureCompact.java   |  21 +-
 .../oak/segment/azure/tool/ToolUtils.java  |  26 +-
 .../oak/segment/CheckpointCompactor.java   |  32 +-
 .../jackrabbit/oak/segment/ClassicCompactor.java   |  21 +-
 .../jackrabbit/oak/segment/CompactorUtils.java |  35 ++
 .../oak/segment/DefaultSegmentWriterBuilder.java   |  90 +++--
 .../jackrabbit/oak/segment/ParallelCompactor.java  | 370 +++
 .../apache/jackrabbit/oak/segment/RecordCache.java | 119 +++
 .../oak/segment/SegmentBufferWriterPool.java   | 390 ++---
 .../oak/segment/compaction/SegmentGCOptions.java   |  47 ++-
 .../segment/file/AbstractCompactionStrategy.java   |  31 +-
 .../jackrabbit/oak/segment/file/FileStore.java |   7 +-
 .../oak/segment/file/FileStoreBuilder.java |  14 +-
 .../oak/segment/file/GCNodeWriteMonitor.java   |  66 ++--
 .../jackrabbit/oak/segment/file/PriorityCache.java | 275 +--
 .../jackrabbit/oak/segment/memory/MemoryStore.java |  14 +-
 .../jackrabbit/oak/segment/tool/Compact.java   |  22 +-
 ...java => AbstractCompactorExternalBlobTest.java} |  29 +-
 ...mpactorTest.java => AbstractCompactorTest.java} |  27 +-
 .../CheckpointCompactorExternalBlobTest.java   | 135 +--
 .../oak/segment/CheckpointCompactorTest.java   | 111 +-
 .../oak/segment/CompactionAndCleanupIT.java|  13 +-
 ...actorTestUtils.java => CompactorTestUtils.java} |  20 +-
 .../jackrabbit/oak/segment/NodeRecordTest.java |   4 +-
 .../segment/ParallelCompactorExternalBlobTest.java |  63 
 .../oak/segment/ParallelCompactorTest.java |  63 
 .../oak/segment/RecordCacheStatsTest.java  |  20 +-
 .../jackrabbit/oak/segment/RecordCacheTest.java|  54 ++-
 .../oak/segment/SegmentBufferWriterPoolTest.java   |  67 ++--
 .../segment/file/ConcurrentPriorityCacheTest.java  | 236 +
 .../oak/segment/file/PriorityCacheTest.java|  27 +-
 oak-shaded-guava/.DS_Store | Bin 0 -> 6148 bytes
 .../oak/plugins/index}/ApproximateCounter.java |   2 +-
 .../oak/plugins/index}/ApproximateCounterTest.java |   9 +-
 46 files changed, 1774 insertions(+), 840 deletions(-)

[jackrabbit-oak] branch trunk updated: OAK-10345 - Add debug log message if larger blob is compared byte-wise. (#1024)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 9e56920b14 OAK-10345 - Add debug log message if larger blob is 
compared byte-wise. (#1024)
9e56920b14 is described below

commit 9e56920b145ecd08369603d24895f4438c817c6f
Author: Axel Hanikel 
AuthorDate: Fri Jul 14 13:27:29 2023 +0200

OAK-10345 - Add debug log message if larger blob is compared byte-wise. 
(#1024)

* OAK-10345 - Add debug log message if larger blob is compared byte-wise.

* OAK-10345 - Add an enable switch to make this as cheap as possible.

* OAK-10345 - Use SystemPropertySupplier for system properties.

-

Co-authored-by: Axel Hanikel 
---
 .../jackrabbit/oak/plugins/memory/AbstractBlob.java | 21 +
 1 file changed, 21 insertions(+)

diff --git 
a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/AbstractBlob.java
 
b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/AbstractBlob.java
index 6415a3d269..59beb49670 100644
--- 
a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/AbstractBlob.java
+++ 
b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/AbstractBlob.java
@@ -26,8 +26,11 @@ import org.apache.jackrabbit.guava.common.hash.Hashing;
 import org.apache.jackrabbit.guava.common.io.ByteSource;
 
 import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.commons.properties.SystemPropertySupplier;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Abstract base class for {@link Blob} implementations.
@@ -36,6 +39,20 @@ import org.jetbrains.annotations.Nullable;
  */
 public abstract class AbstractBlob implements Blob {
 
+private static final Logger LOG = 
LoggerFactory.getLogger(AbstractBlob.class);
+
+private static final boolean DEBUG_BLOB_EQUAL_LOG = SystemPropertySupplier
+.create("oak.abstractblob.equal.log", false)
+.loggingTo(LOG)
+.formatSetMessage( (name, value) -> String.format("%s set to: %s", 
name, value) )
+.get();
+
+private static final long DEBUG_BLOB_EQUAL_LOG_LIMIT = 
SystemPropertySupplier
+.create("oak.abstractblob.equal.log.limit", 100_000_000L)
+.loggingTo(LOG)
+.formatSetMessage( (name, value) -> String.format("%s set to: %s", 
name, value) )
+.get();
+
 private static ByteSource supplier(final Blob blob) {
 return new ByteSource() {
 @Override
@@ -62,6 +79,10 @@ public abstract class AbstractBlob implements Blob {
 return true;
 }
 
+if (DEBUG_BLOB_EQUAL_LOG && al > DEBUG_BLOB_EQUAL_LOG_LIMIT) {
+LOG.debug("Blobs have the same length of {} and we're falling back 
to byte-wise comparison.", al);
+}
+
 try {
 return supplier(a).contentEquals(supplier(b));
 } catch (IOException e) {

[jackrabbit-oak] branch trunk updated (846fcb4e89 -> 353a3f4ce7)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


from 846fcb4e89 OAK-10265 | Refresh the index def after lane revert in 
oak-run out of… (#958)
 add 353a3f4ce7 OAK-10263 Prevent inconsistent state in TarWriter (#957)

No new revisions were added by this update.

Summary of changes:
 .../oak/segment/aws/AwsTarWriterTest.java  |  30 ++-
 oak-segment-azure/pom.xml  |   7 +-
 .../oak/segment/azure/AzurePersistence.java|  28 ++-
 .../segment/azure/AzureSegmentArchiveWriter.java   |  30 ++-
 .../oak/segment/azure/tool/SegmentCopy.java|  13 +-
 .../segment/azure/tool/SegmentStoreMigrator.java   |  74 ++-
 .../jackrabbit/oak/segment/azure/util/Retrier.java |  82 
 .../azure/AzureSegmentArchiveWriterTest.java   | 218 +
 .../oak/segment/azure/AzureTarWriterTest.java  |  38 ++--
 .../oak/segment/azure/util/RetrierTest.java| 147 ++
 .../src/test/resources/logback-test.xml|  11 +-
 .../jackrabbit/oak/segment/file/FileStore.java |   3 +
 .../file/UnrecoverableArchiveException.java|  19 +-
 .../jackrabbit/oak/segment/file/tar/TarWriter.java |  11 +-
 .../jackrabbit/oak/segment/file/FileStoreTest.java | 134 +++--
 .../oak/segment/file/tar/TarWriterTest.java|  75 +--
 .../src/test/resources/logback-test.xml|   1 +
 17 files changed, 773 insertions(+), 148 deletions(-)
 create mode 100644 
oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/Retrier.java
 create mode 100644 
oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentArchiveWriterTest.java
 create mode 100644 
oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/util/RetrierTest.java
 copy {oak-segment-tar => 
oak-segment-azure}/src/test/resources/logback-test.xml (71%)
 copy 
oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/Environment.java
 => 
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/UnrecoverableArchiveException.java
 (67%)

[jackrabbit-oak] branch trunk updated: OAK-10191 - Reusing Azure blob container hangs when primary location is not available (#899)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 42faf5b64e OAK-10191 - Reusing Azure blob container hangs when primary 
location is not available (#899)
42faf5b64e is described below

commit 42faf5b64ead97aeadd7c790af3a07514f09514c
Author: Andrei Dulceanu 
AuthorDate: Fri Apr 14 18:13:40 2023 +0300

OAK-10191 - Reusing Azure blob container hangs when primary location is not 
available (#899)
---
 .../oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java| 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git 
a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
 
b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
index a4e88c3693..a017f7fff0 100644
--- 
a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
+++ 
b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
@@ -204,7 +204,8 @@ public class AzureBlobStoreBackend extends 
AbstractSharedBackend {
 
 CloudBlobContainer azureContainer = getAzureContainer();
 
-if (createBlobContainer && azureContainer.createIfNotExists()) 
{
+if (createBlobContainer && !azureContainer.exists()) {
+azureContainer.create();
 LOG.info("New container created. containerName={}", 
containerName);
 } else {
 LOG.info("Reusing existing container. containerName={}", 
containerName);

[jackrabbit-oak] 01/01: OAK-10191 - Reusing Azure blob container hangs when primary location is not available

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-10191
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit 50540e0c171ea66b4a1932c4b5ec87392e9d6aae
Author: dulceanu 
AuthorDate: Fri Apr 14 17:12:29 2023 +0200

OAK-10191 - Reusing Azure blob container hangs when primary location is not 
available
---
 .../oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java| 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git 
a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
 
b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
index a4e88c3693..a017f7fff0 100644
--- 
a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
+++ 
b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
@@ -204,7 +204,8 @@ public class AzureBlobStoreBackend extends 
AbstractSharedBackend {
 
 CloudBlobContainer azureContainer = getAzureContainer();
 
-if (createBlobContainer && azureContainer.createIfNotExists()) 
{
+if (createBlobContainer && !azureContainer.exists()) {
+azureContainer.create();
 LOG.info("New container created. containerName={}", 
containerName);
 } else {
 LOG.info("Reusing existing container. containerName={}", 
containerName);

[jackrabbit-oak] branch issues/OAK-10191 created (now 50540e0c17)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10191
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at 50540e0c17 OAK-10191 - Reusing Azure blob container hangs when primary 
location is not available

This branch includes the following new commits:

 new 50540e0c17 OAK-10191 - Reusing Azure blob container hangs when primary 
location is not available

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[jackrabbit-oak] 01/01: OAK-10190 - Reusing Azure segment container hangs when primary location is not available

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-10190
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit b0f7d3d1ca14455481ed7e743f737fabff1e39e2
Author: dulceanu 
AuthorDate: Fri Apr 14 17:03:23 2023 +0200

OAK-10190 - Reusing Azure segment container hangs when primary location is 
not available
---
 .../apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
index 9f515bda21..0ccf7fe475 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
@@ -134,8 +134,8 @@ public class AzureSegmentStoreService {
 cloudBlobClient.setDefaultRequestOptions(blobRequestOptions);
 
 CloudBlobContainer container = 
cloudBlobClient.getContainerReference(configuration.containerName());
-if (createContainer) {
-container.createIfNotExists();
+if (createContainer && !container.exists()) {
+container.create();
 }
 String path = normalizePath(configuration.rootPath());
 return new AzurePersistence(container.getDirectoryReference(path));

[jackrabbit-oak] branch issues/OAK-10190 created (now b0f7d3d1ca)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10190
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at b0f7d3d1ca OAK-10190 - Reusing Azure segment container hangs when 
primary location is not available

This branch includes the following new commits:

 new b0f7d3d1ca OAK-10190 - Reusing Azure segment container hangs when 
primary location is not available

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[jackrabbit-oak] 01/01: Reusing containers hangs when primary location is not available

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-10050
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit f340bb3ffa7d401bcc1a327c97859cc209468b51
Author: dulceanu 
AuthorDate: Wed Apr 12 10:28:13 2023 +0200

Reusing containers hangs when primary location is not available
---
 .../cloud/azure/blobstorage/AzureBlobStoreBackend.java | 14 +++---
 .../oak/segment/azure/AzureSegmentStoreService.java|  8 +---
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git 
a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
 
b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
index a4e88c3693..021aad0d24 100644
--- 
a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
+++ 
b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
@@ -135,6 +135,12 @@ public class AzureBlobStoreBackend extends 
AbstractSharedBackend {
 }
 
 protected CloudBlobContainer getAzureContainer() throws DataStoreException 
{
+BlobRequestOptions requestOptions = getRequestOptions();
+
+return Utils.getBlobContainer(connectionString, containerName, 
requestOptions);
+}
+
+private BlobRequestOptions getRequestOptions() {
 BlobRequestOptions requestOptions = new BlobRequestOptions();
 if (null != retryPolicy) {
 requestOptions.setRetryPolicyFactory(retryPolicy);
@@ -146,8 +152,7 @@ public class AzureBlobStoreBackend extends 
AbstractSharedBackend {
 if (enableSecondaryLocation) {
 
requestOptions.setLocationMode(LocationMode.PRIMARY_THEN_SECONDARY);
 }
-
-return Utils.getBlobContainer(connectionString, containerName, 
requestOptions);
+return requestOptions;
 }
 
 @Override
@@ -202,9 +207,12 @@ public class AzureBlobStoreBackend extends 
AbstractSharedBackend {
 
AzureConstants.AZURE_BLOB_ENABLE_SECONDARY_LOCATION_DEFAULT
 );
 
+LOG.info("Using enableSecondaryLocation={}", 
enableSecondaryLocation);
+
 CloudBlobContainer azureContainer = getAzureContainer();
 
-if (createBlobContainer && azureContainer.createIfNotExists()) 
{
+if (createBlobContainer && !azureContainer.exists(null, 
getRequestOptions(), null)) {
+azureContainer.create();
 LOG.info("New container created. containerName={}", 
containerName);
 } else {
 LOG.info("Reusing existing container. containerName={}", 
containerName);
diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
index 9f515bda21..d3234c8488 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
@@ -117,7 +117,7 @@ public class AzureSegmentStoreService {
 }
 
 @NotNull
-private static AzurePersistence createAzurePersistence(
+public static AzurePersistence createAzurePersistence(
 String connectionString,
 Configuration configuration,
 boolean createContainer
@@ -128,14 +128,16 @@ public class AzureSegmentStoreService {
 CloudBlobClient cloudBlobClient = cloud.createCloudBlobClient();
 BlobRequestOptions blobRequestOptions = new BlobRequestOptions();
 
+log.info("Using enableSecondaryLocation={}", 
configuration.enableSecondaryLocation());
+
 if (configuration.enableSecondaryLocation()) {
 
blobRequestOptions.setLocationMode(LocationMode.PRIMARY_THEN_SECONDARY);
 }
 cloudBlobClient.setDefaultRequestOptions(blobRequestOptions);
 
 CloudBlobContainer container = 
cloudBlobClient.getContainerReference(configuration.containerName());
-if (createContainer) {
-container.createIfNotExists();
+if (createContainer && !container.exists(null, blobRequestOptions, 
null)) {
+container.create();
 }
 String path = normalizePath(configuration.rootPath());
 return new AzurePersistence(container.getDirectoryReference(path));

[jackrabbit-oak] branch issues/OAK-10050 created (now f340bb3ffa)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10050
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at f340bb3ffa Reusing containers hangs when primary location is not 
available

This branch includes the following new commits:

 new f340bb3ffa Reusing containers hangs when primary location is not 
available

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[jackrabbit-oak] branch trunk updated: OAK-10050 - Enable access to the secondary Azure blobstore service endpoint in Oak segment node store (#820)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 1446e1b098 OAK-10050 - Enable access to the secondary Azure blobstore 
service endpoint in Oak segment node store (#820)
1446e1b098 is described below

commit 1446e1b09839c2378132fdab971e86105d30fab8
Author: Andrei Dulceanu 
AuthorDate: Mon Mar 27 17:57:28 2023 +0300

OAK-10050 - Enable access to the secondary Azure blobstore service endpoint 
in Oak segment node store (#820)
---
 .../oak/segment/azure/AzureSegmentStoreService.java | 17 +++--
 .../jackrabbit/oak/segment/azure/Configuration.java |  6 ++
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
index 529ae2b74e..9f515bda21 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
@@ -19,7 +19,10 @@
 package org.apache.jackrabbit.oak.segment.azure;
 
 import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.LocationMode;
 import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.blob.BlobRequestOptions;
+import com.microsoft.azure.storage.blob.CloudBlobClient;
 import com.microsoft.azure.storage.blob.CloudBlobContainer;
 import org.apache.commons.lang3.StringUtils;
 import 
org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
@@ -52,6 +55,8 @@ public class AzureSegmentStoreService {
 
 public static final String DEFAULT_ROOT_PATH = "/oak";
 
+public static final boolean DEFAULT_ENABLE_SECONDARY_LOCATION = false;
+
 private ServiceRegistration registration;
 
 @Activate
@@ -103,7 +108,7 @@ public class AzureSegmentStoreService {
 if (!StringUtils.isBlank(configuration.blobEndpoint())) {
 
connectionString.append("BlobEndpoint=").append(configuration.blobEndpoint()).append(';');
 }
-return createAzurePersistence(connectionString.toString(), 
configuration, false); 
+return createAzurePersistence(connectionString.toString(), 
configuration, false);
 }
 
 @NotNull
@@ -120,7 +125,15 @@ public class AzureSegmentStoreService {
 try {
 CloudStorageAccount cloud = 
CloudStorageAccount.parse(connectionString);
 log.info("Connection string: '{}'", cloud);
-CloudBlobContainer container = 
cloud.createCloudBlobClient().getContainerReference(configuration.containerName());
+CloudBlobClient cloudBlobClient = cloud.createCloudBlobClient();
+BlobRequestOptions blobRequestOptions = new BlobRequestOptions();
+
+if (configuration.enableSecondaryLocation()) {
+
blobRequestOptions.setLocationMode(LocationMode.PRIMARY_THEN_SECONDARY);
+}
+cloudBlobClient.setDefaultRequestOptions(blobRequestOptions);
+
+CloudBlobContainer container = 
cloudBlobClient.getContainerReference(configuration.containerName());
 if (createContainer) {
 container.createIfNotExists();
 }
diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/Configuration.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/Configuration.java
index 5923f7881e..2eaf628bc1 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/Configuration.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/Configuration.java
@@ -74,4 +74,10 @@ import static 
org.apache.jackrabbit.oak.segment.azure.Configuration.PID;
 "services in order to create services composed of multiple 
persistence instances. " +
 "E.g. a SplitPersistence composed of a TAR persistence and 
an Azure persistence.")
 String role() default "";
+
+@AttributeDefinition(
+name = "Azure segment store property to enable fallback to the 
secondary location",
+description = "When set to true specifies that requests will be 
attempted in primary, then in secondary region." +
+"Default value is '" + 
AzureSegmentStoreService.DEFAULT_ENABLE_SECONDARY_LOCATION + "'.")
+boolean enableSecondaryLocation() default 
AzureSegmentStoreService.DEFAULT_ENABLE_SECONDARY_LOCATION;
 }
\ No newline at end of file

[jackrabbit-oak] branch trunk updated: OAK-10101 - Improve exception message when retrieving String properties (#840)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new aa492521af OAK-10101 - Improve exception message when retrieving 
String properties (#840)
aa492521af is described below

commit aa492521af6bbe6f3bc3a75e6044d568f42ed5ed
Author: Andrei Dulceanu 
AuthorDate: Thu Feb 2 15:22:34 2023 +0200

OAK-10101 - Improve exception message when retrieving String properties 
(#840)
---
 .../java/org/apache/jackrabbit/oak/segment/data/SegmentDataV12.java| 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/SegmentDataV12.java
 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/SegmentDataV12.java
index ebf2e6bb26..a6e5e34f8b 100644
--- 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/SegmentDataV12.java
+++ 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/SegmentDataV12.java
@@ -189,7 +189,8 @@ class SegmentDataV12 implements SegmentData {
 return new StringData(internalReadRecordId(index + Long.BYTES), 
(int) length);
 }
 
-throw new IllegalStateException("String is too long: " + length);
+throw new IllegalStateException("String is too long: " + length + "; 
possibly trying to read a "
++ "BLOB using getString; can not convert BLOB to String");
 }
 
 private StringData internalReadString(int index, int length) {

[jackrabbit-oak] 01/01: OAK-10101 - Improve exception message when retrieving String properties

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-10101
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit 3db0be054b483110f60419022ee1710f1c27c2c4
Author: dulceanu 
AuthorDate: Thu Feb 2 11:56:35 2023 +0100

OAK-10101 - Improve exception message when retrieving String properties
---
 .../java/org/apache/jackrabbit/oak/segment/data/SegmentDataV12.java| 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/SegmentDataV12.java
 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/SegmentDataV12.java
index ebf2e6bb26..a6e5e34f8b 100644
--- 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/SegmentDataV12.java
+++ 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/SegmentDataV12.java
@@ -189,7 +189,8 @@ class SegmentDataV12 implements SegmentData {
 return new StringData(internalReadRecordId(index + Long.BYTES), 
(int) length);
 }
 
-throw new IllegalStateException("String is too long: " + length);
+throw new IllegalStateException("String is too long: " + length + "; 
possibly trying to read a "
++ "BLOB using getString; can not convert BLOB to String");
 }
 
 private StringData internalReadString(int index, int length) {

[jackrabbit-oak] branch issues/OAK-10101 created (now 3db0be054b)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10101
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at 3db0be054b OAK-10101 - Improve exception message when retrieving 
String properties

This branch includes the following new commits:

 new 3db0be054b OAK-10101 - Improve exception message when retrieving 
String properties

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[jackrabbit-oak] branch 1.22 updated: OAK-10076 - Bump netty dependency from 4.1.68.Final to 4.1.86.Final (#830)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch 1.22
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/1.22 by this push:
 new f5c2c54637 OAK-10076 - Bump netty dependency from 4.1.68.Final to 
4.1.86.Final (#830)
f5c2c54637 is described below

commit f5c2c54637d6b8b83c62dc4ad6c76c830c4c31f1
Author: Andrei Dulceanu 
AuthorDate: Mon Jan 23 18:32:33 2023 +0200

OAK-10076 - Bump netty dependency from 4.1.68.Final to 4.1.86.Final (#830)

* OAK-10076 - Bump netty dependency from 4.1.68.Final to 4.1.86.Final

* OAK-10076 - Bump netty dependency from 4.1.68.Final to 4.1.86.Final
Added the actual dependency bump
---
 oak-segment-tar/pom.xml | 7 ++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/oak-segment-tar/pom.xml b/oak-segment-tar/pom.xml
index d5413c22e1..f7b3fb84c9 100644
--- a/oak-segment-tar/pom.xml
+++ b/oak-segment-tar/pom.xml
@@ -33,7 +33,7 @@
 Oak Segment Tar
 
 
-4.1.68.Final
+4.1.86.Final
 
1.4.2
 
 
@@ -65,6 +65,7 @@
 
com.ning.compress.lzf.util;version="[1.0,2)";resolution:=optional,
 com.oracle.svm.core.annotate;resolution:=optional,
 
io.netty.internal.tcnative;version="[2.0,3)";resolution:=optional,
+
io.netty.channel.unix;version="[4.1,5)";resolution:=optional,
 javax.security.cert;resolution:=optional,
 lzma.sdk;resolution:=optional,
 lzma.sdk.lzma;resolution:=optional,
@@ -77,12 +78,16 @@
 
org.apache.logging.log4j;resolution:=optional;version="[2.6,3)",
 
org.apache.logging.log4j.message;resolution:=optional,
 org.apache.logging.log4j.spi;resolution:=optional,
+org.bouncycastle.asn1.pkcs;resolution:=optional,
 
org.bouncycastle.asn1.x500;version="[1.54,2)";resolution:=optional,
 
org.bouncycastle.cert;version="[1.54,2)";resolution:=optional,
 
org.bouncycastle.cert.jcajce;version="[1.54,2)";resolution:=optional,
 
org.bouncycastle.jce.provider;version="[1.54,2)";resolution:=optional,
+org.bouncycastle.openssl;resolution:=optional,
+
org.bouncycastle.openssl.jcajce;resolution:=optional,
 
org.bouncycastle.operator;version="[1.54,2)";resolution:=optional,
 
org.bouncycastle.operator.jcajce;version="[1.54,2)";resolution:=optional,
+org.bouncycastle.pkcs;resolution:=optional,
 org.conscrypt;resolution:=optional,
 
org.eclipse.jetty.alpn;version="[1,2)";resolution:=optional,
 
org.eclipse.jetty.npn;version="[1,2)";resolution:=optional,

[jackrabbit-oak] branch issues/1.22/OAK-10076 updated (6e2a279c8a -> 3af790ab3c)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/1.22/OAK-10076
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


from 6e2a279c8a OAK-10076 - Bump netty dependency from 4.1.68.Final to 
4.1.86.Final
 add 3af790ab3c OAK-10076 - Bump netty dependency from 4.1.68.Final to 
4.1.86.Final Added the actual dependency bump

No new revisions were added by this update.

Summary of changes:
 oak-segment-tar/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

[jackrabbit-oak] 01/01: OAK-10076 - Bump netty dependency from 4.1.68.Final to 4.1.86.Final

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/1.22/OAK-10076
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit 6e2a279c8a35225829cdc9e76415e6127b9e5fed
Author: dulceanu 
AuthorDate: Mon Jan 23 17:22:16 2023 +0100

OAK-10076 - Bump netty dependency from 4.1.68.Final to 4.1.86.Final
---
 oak-segment-tar/pom.xml | 5 +
 1 file changed, 5 insertions(+)

diff --git a/oak-segment-tar/pom.xml b/oak-segment-tar/pom.xml
index d5413c22e1..f7cb66847a 100644
--- a/oak-segment-tar/pom.xml
+++ b/oak-segment-tar/pom.xml
@@ -65,6 +65,7 @@
 
com.ning.compress.lzf.util;version="[1.0,2)";resolution:=optional,
 com.oracle.svm.core.annotate;resolution:=optional,
 
io.netty.internal.tcnative;version="[2.0,3)";resolution:=optional,
+
io.netty.channel.unix;version="[4.1,5)";resolution:=optional,
 javax.security.cert;resolution:=optional,
 lzma.sdk;resolution:=optional,
 lzma.sdk.lzma;resolution:=optional,
@@ -77,12 +78,16 @@
 
org.apache.logging.log4j;resolution:=optional;version="[2.6,3)",
 
org.apache.logging.log4j.message;resolution:=optional,
 org.apache.logging.log4j.spi;resolution:=optional,
+org.bouncycastle.asn1.pkcs;resolution:=optional,
 
org.bouncycastle.asn1.x500;version="[1.54,2)";resolution:=optional,
 
org.bouncycastle.cert;version="[1.54,2)";resolution:=optional,
 
org.bouncycastle.cert.jcajce;version="[1.54,2)";resolution:=optional,
 
org.bouncycastle.jce.provider;version="[1.54,2)";resolution:=optional,
+org.bouncycastle.openssl;resolution:=optional,
+
org.bouncycastle.openssl.jcajce;resolution:=optional,
 
org.bouncycastle.operator;version="[1.54,2)";resolution:=optional,
 
org.bouncycastle.operator.jcajce;version="[1.54,2)";resolution:=optional,
+org.bouncycastle.pkcs;resolution:=optional,
 org.conscrypt;resolution:=optional,
 
org.eclipse.jetty.alpn;version="[1,2)";resolution:=optional,
 
org.eclipse.jetty.npn;version="[1,2)";resolution:=optional,

[jackrabbit-oak] branch issues/1.22/OAK-10076 created (now 6e2a279c8a)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/1.22/OAK-10076
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at 6e2a279c8a OAK-10076 - Bump netty dependency from 4.1.68.Final to 
4.1.86.Final

This branch includes the following new commits:

 new 6e2a279c8a OAK-10076 - Bump netty dependency from 4.1.68.Final to 
4.1.86.Final

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[jackrabbit-oak] branch trunk updated: OAK-10076 - Bump netty dependency from 4.1.68.Final to 4.1.86.Final (#827)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 3b2671c25f OAK-10076 - Bump netty dependency from 4.1.68.Final to 
4.1.86.Final (#827)
3b2671c25f is described below

commit 3b2671c25feadeaa8fc1d43cee10a270f29ee147
Author: Andrei Dulceanu 
AuthorDate: Mon Jan 23 11:55:25 2023 +0200

OAK-10076 - Bump netty dependency from 4.1.68.Final to 4.1.86.Final (#827)
---
 oak-segment-tar/pom.xml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/oak-segment-tar/pom.xml b/oak-segment-tar/pom.xml
index 212a6b9caa..54e4d4cde8 100644
--- a/oak-segment-tar/pom.xml
+++ b/oak-segment-tar/pom.xml
@@ -33,7 +33,7 @@
 Oak Segment Tar
 
 
-4.1.68.Final
+4.1.86.Final
 
1.4.2
 
 
@@ -66,6 +66,7 @@
 
com.ning.compress.lzf.util;version="[1.0,2)";resolution:=optional,
 com.oracle.svm.core.annotate;resolution:=optional,
 
io.netty.internal.tcnative;version="[2.0,3)";resolution:=optional,
+
io.netty.channel.unix;version="[4.1,5)";resolution:=optional,
 javax.security.cert;resolution:=optional,
 lzma.sdk;resolution:=optional,
 lzma.sdk.lzma;resolution:=optional,

[jackrabbit-oak] 01/01: OAK-10076 - Bump netty dependency from 4.1.68.Final to 4.1.86.Final

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-10076
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit 976e7e037b3634c02ef85be4517952a5d2c0ba3d
Author: dulceanu 
AuthorDate: Fri Jan 20 15:47:12 2023 +0100

OAK-10076 - Bump netty dependency from 4.1.68.Final to 4.1.86.Final
---
 oak-segment-tar/pom.xml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/oak-segment-tar/pom.xml b/oak-segment-tar/pom.xml
index 55aa78cb04..7d8d2c48d7 100644
--- a/oak-segment-tar/pom.xml
+++ b/oak-segment-tar/pom.xml
@@ -33,7 +33,7 @@
 Oak Segment Tar
 
 
-4.1.68.Final
+4.1.86.Final
 
1.4.2
 
 
@@ -66,6 +66,7 @@
 
com.ning.compress.lzf.util;version="[1.0,2)";resolution:=optional,
 com.oracle.svm.core.annotate;resolution:=optional,
 
io.netty.internal.tcnative;version="[2.0,3)";resolution:=optional,
+
io.netty.channel.unix;version="[4.1,5)";resolution:=optional,
 javax.security.cert;resolution:=optional,
 lzma.sdk;resolution:=optional,
 lzma.sdk.lzma;resolution:=optional,

[jackrabbit-oak] branch issues/OAK-10076 created (now 976e7e037b)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10076
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at 976e7e037b OAK-10076 - Bump netty dependency from 4.1.68.Final to 
4.1.86.Final

This branch includes the following new commits:

 new 976e7e037b OAK-10076 - Bump netty dependency from 4.1.68.Final to 
4.1.86.Final

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[jackrabbit-oak] branch issues/OAK-10050 created (now d0b03c1789)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a change to branch issues/OAK-10050
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


  at d0b03c1789 OAK-10050 - Enable access to the secondary Azure blobstore 
service endpoint in Oak segment node store

This branch includes the following new commits:

 new d0b03c1789 OAK-10050 - Enable access to the secondary Azure blobstore 
service endpoint in Oak segment node store

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[jackrabbit-oak] 01/01: OAK-10050 - Enable access to the secondary Azure blobstore service endpoint in Oak segment node store

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch issues/OAK-10050
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit d0b03c1789f32c7118727c038e267598b509da6d
Author: dulceanu 
AuthorDate: Thu Jan 12 11:51:22 2023 +0100

OAK-10050 - Enable access to the secondary Azure blobstore service endpoint 
in Oak segment node store
---
 .../oak/segment/azure/AzureSegmentStoreService.java | 17 +++--
 .../jackrabbit/oak/segment/azure/Configuration.java |  6 ++
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
index 529ae2b74e..9f515bda21 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
@@ -19,7 +19,10 @@
 package org.apache.jackrabbit.oak.segment.azure;
 
 import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.LocationMode;
 import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.blob.BlobRequestOptions;
+import com.microsoft.azure.storage.blob.CloudBlobClient;
 import com.microsoft.azure.storage.blob.CloudBlobContainer;
 import org.apache.commons.lang3.StringUtils;
 import 
org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
@@ -52,6 +55,8 @@ public class AzureSegmentStoreService {
 
 public static final String DEFAULT_ROOT_PATH = "/oak";
 
+public static final boolean DEFAULT_ENABLE_SECONDARY_LOCATION = false;
+
 private ServiceRegistration registration;
 
 @Activate
@@ -103,7 +108,7 @@ public class AzureSegmentStoreService {
 if (!StringUtils.isBlank(configuration.blobEndpoint())) {
 
connectionString.append("BlobEndpoint=").append(configuration.blobEndpoint()).append(';');
 }
-return createAzurePersistence(connectionString.toString(), 
configuration, false); 
+return createAzurePersistence(connectionString.toString(), 
configuration, false);
 }
 
 @NotNull
@@ -120,7 +125,15 @@ public class AzureSegmentStoreService {
 try {
 CloudStorageAccount cloud = 
CloudStorageAccount.parse(connectionString);
 log.info("Connection string: '{}'", cloud);
-CloudBlobContainer container = 
cloud.createCloudBlobClient().getContainerReference(configuration.containerName());
+CloudBlobClient cloudBlobClient = cloud.createCloudBlobClient();
+BlobRequestOptions blobRequestOptions = new BlobRequestOptions();
+
+if (configuration.enableSecondaryLocation()) {
+
blobRequestOptions.setLocationMode(LocationMode.PRIMARY_THEN_SECONDARY);
+}
+cloudBlobClient.setDefaultRequestOptions(blobRequestOptions);
+
+CloudBlobContainer container = 
cloudBlobClient.getContainerReference(configuration.containerName());
 if (createContainer) {
 container.createIfNotExists();
 }
diff --git 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/Configuration.java
 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/Configuration.java
index 5923f7881e..2eaf628bc1 100644
--- 
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/Configuration.java
+++ 
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/Configuration.java
@@ -74,4 +74,10 @@ import static 
org.apache.jackrabbit.oak.segment.azure.Configuration.PID;
 "services in order to create services composed of multiple 
persistence instances. " +
 "E.g. a SplitPersistence composed of a TAR persistence and 
an Azure persistence.")
 String role() default "";
+
+@AttributeDefinition(
+name = "Azure segment store property to enable fallback to the 
secondary location",
+description = "When set to true specifies that requests will be 
attempted in primary, then in secondary region." +
+"Default value is '" + 
AzureSegmentStoreService.DEFAULT_ENABLE_SECONDARY_LOCATION + "'.")
+boolean enableSecondaryLocation() default 
AzureSegmentStoreService.DEFAULT_ENABLE_SECONDARY_LOCATION;
 }
\ No newline at end of file

[jackrabbit-oak] branch 1.22 updated: OAK-9911 - Resource leak in ChunkedBlobStream (#790)

[adulceanu]

This is an automated email from the ASF dual-hosted git repository.

adulceanu pushed a commit to branch 1.22
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/1.22 by this push:
 new 3da99bec4d OAK-9911 - Resource leak in ChunkedBlobStream (#790)
3da99bec4d is described below

commit 3da99bec4d5d4411cd32391cd3f16fe9d8695e92
Author: kunal3112 <49707623+kunal3...@users.noreply.github.com>
AuthorDate: Mon Dec 19 17:47:22 2022 +0530

OAK-9911 - Resource leak in ChunkedBlobStream (#790)

Co-authored-by: Julian Sedding 
---
 .../oak/segment/standby/codec/ChunkedBlobStream.java|  4 ++--
 .../oak/segment/standby/ExternalPrivateStoreIT.java |  8 
 .../jackrabbit/oak/segment/test/TemporaryBlobStore.java |  9 -
 .../jackrabbit/oak/segment/test/TemporaryFileStore.java | 13 +
 4 files changed, 23 insertions(+), 11 deletions(-)

diff --git 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/codec/ChunkedBlobStream.java
 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/codec/ChunkedBlobStream.java
index 4f6a57cd83..6a6c0ccebf 100644
--- 
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/codec/ChunkedBlobStream.java
+++ 
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/codec/ChunkedBlobStream.java
@@ -117,7 +117,7 @@ public class ChunkedBlobStream implements 
ChunkedInput {
 }
 
 boolean release = true;
-ByteBuf decorated = allocator.buffer();
+ByteBuf decorated = null;
 
 try {
 ByteBuf buffer = allocator.buffer();
@@ -131,7 +131,7 @@ public class ChunkedBlobStream implements 
ChunkedInput {
 release = false;
 return decorated;
 } finally {
-if (release) {
+if (decorated != null && release) {
 decorated.release();
 }
 }
diff --git 
a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/ExternalPrivateStoreIT.java
 
b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/ExternalPrivateStoreIT.java
index ff76db587d..69a42c4797 100644
--- 
a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/ExternalPrivateStoreIT.java
+++ 
b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/ExternalPrivateStoreIT.java
@@ -31,13 +31,13 @@ public class ExternalPrivateStoreIT extends 
DataStoreTestBase {
 
 private TemporaryFolder folder = new TemporaryFolder(new File("target"));
 
-private TemporaryBlobStore serverBlobStore = new 
TemporaryBlobStore(folder);
+private TemporaryBlobStore serverBlobStore = new 
TemporaryBlobStore(folder, "server-blob-store");
 
-private TemporaryFileStore serverFileStore = new 
TemporaryFileStore(folder, serverBlobStore, false);
+private TemporaryFileStore serverFileStore = new 
TemporaryFileStore(folder, serverBlobStore, false, "server-file-store");
 
-private TemporaryBlobStore clientBlobStore = new 
TemporaryBlobStore(folder);
+private TemporaryBlobStore clientBlobStore = new 
TemporaryBlobStore(folder, "client-blob-store");
 
-private TemporaryFileStore clientFileStore = new 
TemporaryFileStore(folder, clientBlobStore, true);
+private TemporaryFileStore clientFileStore = new 
TemporaryFileStore(folder, clientBlobStore, true, "client-file-store");
 
 @Rule
 public RuleChain chain = RuleChain.outerRule(folder)
diff --git 
a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java
 
b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java
index 16633d10a6..adce632623 100644
--- 
a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java
+++ 
b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java
@@ -28,17 +28,24 @@ public class TemporaryBlobStore extends ExternalResource {
 
 private final TemporaryFolder folder;
 
+private final String name;
+
 private DataStoreBlobStore store;
 
 public TemporaryBlobStore(TemporaryFolder folder) {
+this(folder, null);
+}
+
+public TemporaryBlobStore(TemporaryFolder folder, String name) {
 this.folder = folder;
+this.name = name;
 }
 
 @Override
 protected void before() throws Throwable {
 FileDataStore fds = new FileDataStore();
 configureDataStore(fds);
-fds.init(folder.newFolder().getAbsolutePath());
+fds.init((name == null ? folder.newFolder() : 
folder.newFolder(name)).getAbsolutePath());
 store = new DataStoreBlobStore(fds);
 }
 
diff --git 
a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java
 
b/oak-