This is an automated email from the ASF dual-hosted git repository.
broustant pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr-sandbox.git
The following commit(s) were added to refs/heads/main by this push:
new e5cce7e Support follower replicas fetching encrypted index files.
(#116)
e5cce7e is described below
commit e5cce7ea14281b6372917d28f2a4f7b09534865d
Author: Bruno Roustant <[email protected]>
AuthorDate: Mon Aug 4 15:41:56 2025 +0200
Support follower replicas fetching encrypted index files. (#116)
---
encryption/build.gradle | 8 +-
.../solr/encryption/EncryptionDirectory.java | 29 +++++-
.../encryption/EncryptionDirectoryFactory.java | 24 +++++
.../solr/encryption/EncryptionRequestHandler.java | 6 +-
.../encryption/EncryptionBackupRepositoryTest.java | 7 +-
.../encryption/EncryptionIndexFetchingTest.java | 107 +++++++++++++++++++++
.../encryption/EncryptionRequestHandlerTest.java | 15 ++-
.../apache/solr/encryption/EncryptionTestUtil.java | 99 +++++++++++++------
8 files changed, 244 insertions(+), 51 deletions(-)
diff --git a/encryption/build.gradle b/encryption/build.gradle
index 3eadcf4..c14c04a 100644
--- a/encryption/build.gradle
+++ b/encryption/build.gradle
@@ -33,8 +33,8 @@ sourceSets {
}
dependencies {
- implementation 'org.apache.solr:solr-core:9.8.0'
- implementation 'org.apache.lucene:lucene-core:9.11.1'
+ implementation 'org.apache.solr:solr-core:9.9.0'
+ implementation 'org.apache.lucene:lucene-core:9.12.2'
implementation 'com.google.code.findbugs:jsr305:3.0.2'
// Optional, used by the KmsKeySupplier example.
@@ -47,8 +47,8 @@ dependencies {
implementation 'commons-io:commons-io:2.18.0'
implementation 'commons-codec:commons-codec:1.18.0'
- testImplementation 'org.apache.solr:solr-test-framework:9.8.0'
- testImplementation 'org.apache.lucene:lucene-test-framework:9.11.1'
+ testImplementation 'org.apache.solr:solr-test-framework:9.9.0'
+ testImplementation 'org.apache.lucene:lucene-test-framework:9.12.2'
}
test {
diff --git
a/encryption/src/main/java/org/apache/solr/encryption/EncryptionDirectory.java
b/encryption/src/main/java/org/apache/solr/encryption/EncryptionDirectory.java
index 74a7e31..fe4cd26 100644
---
a/encryption/src/main/java/org/apache/solr/encryption/EncryptionDirectory.java
+++
b/encryption/src/main/java/org/apache/solr/encryption/EncryptionDirectory.java
@@ -34,6 +34,7 @@ import org.apache.solr.encryption.crypto.DecryptingIndexInput;
import org.apache.solr.encryption.crypto.EncryptingIndexOutput;
import javax.annotation.Nullable;
+import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
@@ -49,6 +50,7 @@ import java.util.stream.Collectors;
import static org.apache.lucene.codecs.CodecUtil.readBEInt;
import static org.apache.lucene.codecs.CodecUtil.writeBEInt;
import static org.apache.solr.encryption.EncryptionUtil.*;
+import static org.apache.solr.encryption.crypto.AesCtrUtil.IV_LENGTH;
/**
* {@link FilterDirectory} that wraps a delegate {@link Directory} to
encrypt/decrypt files on the fly.
@@ -81,6 +83,9 @@ public class EncryptionDirectory extends FilterDirectory {
*/
public static final int ENCRYPTION_MAGIC = 0x2E5BF271; // 777777777 in
decimal
+ // An encrypted file starts with a 4-bytes "magic header", then 4-bytes key
reference, then a 16-bytes random IV.
+ private static final int ENCRYPTION_HEADER_LENGTH = 8 + IV_LENGTH;
+
protected final AesCtrEncrypterFactory encrypterFactory;
protected final KeySupplier keySupplier;
@@ -278,7 +283,13 @@ public class EncryptionDirectory extends FilterDirectory {
// issue because it will be read immediately again when the IndexInput is
returned, to
// check the index header (CodecUtil.checkIndexHeader()).
long filePointer = indexInput.getFilePointer();
- int magic = readBEInt(indexInput);
+ int magic;
+ try {
+ magic = readBEInt(indexInput);
+ } catch (EOFException e) {
+ // The file contains less than 4 bytes (not expected to happen with
Lucene). It is not encrypted.
+ magic = 0;
+ }
if (magic == ENCRYPTION_MAGIC) {
// This file is encrypted.
// Read the key reference that follows.
@@ -327,6 +338,22 @@ public class EncryptionDirectory extends FilterDirectory {
return segmentsWithOldKeyId == null ? Collections.emptyList() :
segmentsWithOldKeyId;
}
+ /**
+ * Returns the logical length of the file. If the file is encrypted, its
logical length ignores the encryption header.
+ *
+ * @param fileName the name of an existing file.
+ * @return the logical length of the file, in bytes.
+ */
+ @Override
+ public long fileLength(String fileName) throws IOException {
+ // Read the first 4 bytes to check the encryption "magic header".
+ try (IndexInput indexInput = in.openInput(fileName, IOContext.READONCE)) {
+ long fileLength = indexInput.length();
+ return fileLength >= 4 && readBEInt(indexInput) == ENCRYPTION_MAGIC ?
+ fileLength - ENCRYPTION_HEADER_LENGTH : fileLength;
+ }
+ }
+
/**
* Keeps the {@link SegmentInfos commit} file name and user data.
*/
diff --git
a/encryption/src/main/java/org/apache/solr/encryption/EncryptionDirectoryFactory.java
b/encryption/src/main/java/org/apache/solr/encryption/EncryptionDirectoryFactory.java
index ab88bef..4232723 100644
---
a/encryption/src/main/java/org/apache/solr/encryption/EncryptionDirectoryFactory.java
+++
b/encryption/src/main/java/org/apache/solr/encryption/EncryptionDirectoryFactory.java
@@ -17,6 +17,7 @@
package org.apache.solr.encryption;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
import org.apache.lucene.store.LockFactory;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
@@ -77,6 +78,21 @@ public class EncryptionDirectoryFactory extends
MMapDirectoryFactory {
private AesCtrEncrypterFactory encrypterFactory;
private InnerFactory innerFactory;
+ public EncryptionDirectoryFactory() {}
+
+ /**
+ * Visible for tests only.
+ */
+ EncryptionDirectoryFactory(
+ KeySupplier keySupplier,
+ AesCtrEncrypterFactory encrypterFactory,
+ InnerFactory innerFactory) {
+ super.init(new NamedList<>());
+ this.keySupplier = keySupplier;
+ this.encrypterFactory = encrypterFactory;
+ this.innerFactory = innerFactory;
+ }
+
@Override
public void init(NamedList<?> args) {
super.init(args);
@@ -144,6 +160,14 @@ public class EncryptionDirectoryFactory extends
MMapDirectoryFactory {
return innerFactory.create(super.create(path, lockFactory, dirContext),
getEncrypterFactory(), getKeySupplier());
}
+ @Override
+ protected Directory filterDirectory(Directory dir, DirContext dirContext) {
+ // Do not unwrap for DirContext.BACKUP, as it is managed by the
EncryptionBackupRepository.
+ // EncryptionBackupRepository needs to both unwrap when copying file
content, and also not
+ // unwrap when checking the checksum of the cleartext content.
+ return dirContext == DirContext.REPLICATION ? FilterDirectory.unwrap(dir)
: dir;
+ }
+
@Override
public void close() throws IOException {
keySupplier.close();
diff --git
a/encryption/src/main/java/org/apache/solr/encryption/EncryptionRequestHandler.java
b/encryption/src/main/java/org/apache/solr/encryption/EncryptionRequestHandler.java
index a4346fe..a124e46 100644
---
a/encryption/src/main/java/org/apache/solr/encryption/EncryptionRequestHandler.java
+++
b/encryption/src/main/java/org/apache/solr/encryption/EncryptionRequestHandler.java
@@ -363,13 +363,13 @@ public class EncryptionRequestHandler extends
RequestHandlerBase {
Collection<Slice> slices = docCollection.getActiveSlices();
Collection<Callable<State>> encryptRequests = new
ArrayList<>(slices.size());
for (Slice slice : slices) {
- Replica replica = slice.getLeader();
- if (replica == null) {
+ Replica leader = slice.getLeader();
+ if (leader == null) {
log.error("No leader found for shard {}", slice.getName());
collectionState = State.ERROR;
continue;
}
- encryptRequests.add(() -> sendEncryptionRequestWithRetry(replica, req,
params, keyId));
+ encryptRequests.add(() -> sendEncryptionRequestWithRetry(leader, req,
params, keyId));
}
try {
List<Future<State>> responses = timeOut == null ?
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionBackupRepositoryTest.java
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionBackupRepositoryTest.java
index 8624aaa..9f36a19 100644
---
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionBackupRepositoryTest.java
+++
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionBackupRepositoryTest.java
@@ -96,14 +96,17 @@ public class EncryptionBackupRepositoryTest extends
AbstractBackupRepositoryTest
AesCtrEncrypterFactory encrypterFactory = random().nextBoolean() ?
CipherAesCtrEncrypter.FACTORY : LightAesCtrEncrypter.FACTORY;
KeySupplier keySupplier = new TestingKeySupplier.Factory().create();
try (Directory fsSourceDir = FSDirectory.open(sourcePath);
- Directory encSourceDir = new TestEncryptionDirectory(fsSourceDir,
encrypterFactory, keySupplier);
- Directory destinationDir =
FSDirectory.open(createTempDir().toAbsolutePath())) {
+ Directory destinationDir =
FSDirectory.open(createTempDir().toAbsolutePath());
+ EncryptionDirectoryFactory encDirFactory = new
EncryptionDirectoryFactory(keySupplier, encrypterFactory,
TestEncryptionDirectory::new);
+ Directory encSourceDir = encDirFactory.get(sourcePath.toString(),
DirectoryFactory.DirContext.BACKUP, null)) {
String fileName = "source-file";
String content = "content";
try (IndexOutput out = encSourceDir.createOutput(fileName,
IOContext.DEFAULT)) {
byte[] bytes = content.getBytes(StandardCharsets.UTF_8);
out.writeBytes(bytes, bytes.length);
CodecUtil.writeFooter(out);
+ } finally {
+ encDirFactory.release(encSourceDir);
}
BackupRepositoryFactory repoFactory = new
BackupRepositoryFactory(plugins);
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionIndexFetchingTest.java
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionIndexFetchingTest.java
new file mode 100644
index 0000000..f37c5ac
--- /dev/null
+++
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionIndexFetchingTest.java
@@ -0,0 +1,107 @@
+package org.apache.solr.encryption;
+
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.MiniSolrCloudCluster;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.util.TimeSource;
+import org.apache.solr.util.TimeOut;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import static
org.apache.solr.encryption.EncryptionDirectoryFactory.PROPERTY_INNER_ENCRYPTION_DIRECTORY_FACTORY;
+import static org.apache.solr.encryption.TestingKeySupplier.KEY_ID_1;
+
+/**
+ * Tests encrypted index fetching, when follower replicas fetch index from the
leader.
+ */
+public class EncryptionIndexFetchingTest extends SolrCloudTestCase {
+
+ private static final String COLLECTION_PREFIX =
EncryptionIndexFetchingTest.class.getSimpleName() + "-collection-";
+
+ private String collectionName;
+ private CloudSolrClient solrClient;
+ private EncryptionTestUtil testUtil;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ System.setProperty(PROPERTY_INNER_ENCRYPTION_DIRECTORY_FACTORY,
EncryptionRequestHandlerTest.MockFactory.class.getName());
+ EncryptionTestUtil.setInstallDirProperty();
+ cluster = new MiniSolrCloudCluster.Builder(2, createTempDir())
+ .addConfig("config", EncryptionTestUtil.getConfigPath("kms"))
+ .configure();
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ cluster.shutdown();
+ }
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ collectionName = COLLECTION_PREFIX + UUID.randomUUID();
+ solrClient = cluster.getSolrClient();
+ CollectionAdminRequest.createCollection(collectionName, null, 1, 1, 0, 1)
+ .process(solrClient);
+ cluster.waitForActiveCollection(collectionName, 1, 2);
+ testUtil = new EncryptionTestUtil(solrClient, collectionName);
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+
CollectionAdminRequest.deleteCollection(collectionName).process(solrClient);
+ super.tearDown();
+ }
+
+ @Test
+ public void testIndexFetchingWithPullReplica() throws Exception {
+ // GIVEN a Solr Cloud cluster composed of 2 nodes, 1 shard, 1 NRT replica
and 1 PULL replica.
+ // GIVEN an encrypted index containing 3 docs.
+ testUtil.encryptAndExpectCompletion(KEY_ID_1);
+ testUtil.indexDocsAndCommit("weather broadcast", "sunny weather", "foggy
weather");
+
+ // WHEN the follower PULL replica is queried.
+ Replica follower = getFollowerReplica();
+ try (Http2SolrClient followerClient = new
Http2SolrClient.Builder(follower.getCoreUrl()).build()) {
+ new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME)
+ .waitFor(null, () -> {
+ try {
+
+ // THEN eventually the PULL replica is able to fetch the
encrypted index
+ // and search it to find the 3 matching docs.
+ QueryResponse response = followerClient.query(new
SolrQuery("weather"));
+ return response.getResults().size() == 3;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ // THEN verify that the index is encrypted on all replicas, including the
follower.
+ EncryptionRequestHandlerTest.forceClearText = true;
+ testUtil.assertCannotReloadCores(false);
+ EncryptionRequestHandlerTest.forceClearText = false;
+ testUtil.reloadCores(false);
+ }
+
+ private Replica getFollowerReplica() {
+ for (Slice slice :
solrClient.getClusterState().getCollection(collectionName).getSlices()) {
+ for (Replica replica : slice.getReplicas()) {
+ if (!replica.isLeader()) {
+ return replica;
+ }
+ }
+ }
+ throw new IllegalStateException("No follower replica found");
+ }
+}
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionRequestHandlerTest.java
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionRequestHandlerTest.java
index e0014a6..32fac67 100644
---
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionRequestHandlerTest.java
+++
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionRequestHandlerTest.java
@@ -52,7 +52,7 @@ public class EncryptionRequestHandlerTest extends
SolrCloudTestCase {
protected static String configDir = "collection1";
- private static volatile boolean forceClearText;
+ static volatile boolean forceClearText;
private static volatile String soleKeyIdAllowed;
private String collectionName;
@@ -80,11 +80,7 @@ public class EncryptionRequestHandlerTest extends
SolrCloudTestCase {
solrClient = cluster.getSolrClient();
CollectionAdminRequest.createCollection(collectionName, 2,
2).process(solrClient);
cluster.waitForActiveCollection(collectionName, 2, 4);
- testUtil = createEncryptionTestUtil(solrClient, collectionName);
- }
-
- protected EncryptionTestUtil createEncryptionTestUtil(CloudSolrClient
solrClient, String collectionName) {
- return new EncryptionTestUtil(solrClient, collectionName);
+ testUtil = new EncryptionTestUtil(solrClient, collectionName);
}
@Override
@@ -272,15 +268,16 @@ public class EncryptionRequestHandlerTest extends
SolrCloudTestCase {
restartSolrServer(1);
waitForState("Timed out waiting for shards to be active",
collectionName,
- SolrCloudTestCase.activeClusterShape(2, 4),
30,
- TimeUnit.SECONDS);
- } catch (InterruptedException | TimeoutException e) {
+ TimeUnit.SECONDS,
+ SolrCloudTestCase.activeClusterShape(2, 4));
+ } catch (InterruptedException | TimeoutException | AssertionError e) {
// Sometimes restarting Solr nodes hangs, or waiting for shards to
become active times out.
// In this case, exit silently the test.
return;
}
+ // Now commit.
testUtil.commit();
testUtil.waitUntilEncryptionIsComplete(KEY_ID_1);
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionTestUtil.java
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionTestUtil.java
index 2cb6aa5..2aab0fa 100644
---
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionTestUtil.java
+++
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionTestUtil.java
@@ -35,10 +35,11 @@ import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.RetryUtil;
import org.apache.solr.encryption.kms.TestingKmsClient;
-import org.junit.Assert;
import java.io.IOException;
import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
@@ -53,6 +54,7 @@ import static
org.apache.solr.encryption.kms.KmsEncryptionRequestHandler.PARAM_E
import static
org.apache.solr.encryption.kms.KmsEncryptionRequestHandler.PARAM_TENANT_ID;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/**
* Utility methods for encryption tests.
@@ -238,39 +240,78 @@ public class EncryptionTestUtil {
}
/**
- * Reloads the leader replica core of the first shard of the collection.
+ * Reloads the replicas of the collection defined in the constructor.
+ * <p>
+ * If {@link #shouldDistributeEncryptRequest()} returns true, then only the
leader replicas are reloaded
+ * (because only them receive the distributed encryption request).
*/
- public void reloadCores() throws Exception {
+ public void reloadCores() {
+ reloadCores(shouldDistributeEncryptRequest());
+ }
+
+ /**
+ * Reloads the replicas of the collection defined in the constructor.
+ *
+ * @param onlyLeaders whether to only reload leader replicas, or all
replicas.
+ */
+ public void reloadCores(boolean onlyLeaders) {
+ forAllReplicas(onlyLeaders, this::reloadCore);
+ }
+
+ /**
+ * Reloads a specific replica.
+ */
+ public void reloadCore(Replica replica) {
try {
- forAllReplicas(shouldDistributeEncryptRequest(), replica -> {
- try {
- CoreAdminRequest req = new CoreAdminRequest();
- req.setCoreName(replica.getCoreName());
- req.setAction(CoreAdminParams.CoreAdminAction.RELOAD);
- try (Http2SolrClient httpSolrClient = new
Http2SolrClient.Builder(replica.getBaseUrl()).build()) {
- httpSolrClient.request(req);
- }
- } catch (SolrServerException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- });
- } catch (SolrException e) {
- throw new CoreReloadException("The index cannot be reloaded. There is
probably an issue with the encryption key ids.", e);
+ CoreAdminRequest req = new CoreAdminRequest();
+ req.setCoreName(replica.getCoreName());
+ req.setAction(CoreAdminParams.CoreAdminAction.RELOAD);
+ try (Http2SolrClient httpSolrClient = new
Http2SolrClient.Builder(replica.getBaseUrl()).build()) {
+ httpSolrClient.request(req);
+ }
+ } catch (SolrServerException | SolrException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "The index
cannot be reloaded. There is probably an issue with the encryption key ids.",
e);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
}
/**
- * Verifies that {@link #reloadCores()} fails.
+ * Verifies that the replicas of the collection defined in the constructor
fail to reload.
+ * <p>
+ * If {@link #shouldDistributeEncryptRequest()} returns true, then only the
leader replicas are reloaded
+ * (because only them receive the distributed encryption request).
*/
- public void assertCannotReloadCores() throws Exception {
- try {
- reloadCores();
- Assert.fail("Core reloaded whereas it was not expected to be possible");
- } catch (CoreReloadException e) {
- // Expected.
+ public void assertCannotReloadCores() {
+ assertCannotReloadCores(shouldDistributeEncryptRequest());
+ }
+
+ /**
+ * Verifies that the replicas of the collection defined in the constructor
fail to reload.
+ *
+ * @param onlyLeaders whether to only reload leader replicas, or all
replicas.
+ */
+ public void assertCannotReloadCores(boolean onlyLeaders) {
+ Map<String, Integer> numReloadedPerShard = new HashMap<>();
+ forAllReplicas(onlyLeaders, replica -> {
+ try {
+ numReloadedPerShard.putIfAbsent(replica.getShard(), 0);
+ reloadCore(replica);
+ numReloadedPerShard.compute(replica.getShard(), (k, v) -> v == null ?
1 : v + 1);
+ } catch (SolrException e) {
+ // Expected.
+ }
+ });
+ // It is tricky to check that index is encrypted. We check that by
reloading the cores, and
+ // forcing the mock EncryptionDirectory to consider clear text, and
attempting to open the index
+ // files. But in our tests, the small index files are not in all shards.
So we check here that
+ // at least one shard has no cores successfully reloaded.
+ for (Map.Entry<String, Integer> entry : numReloadedPerShard.entrySet()) {
+ if (entry.getValue() == 0) {
+ return;
+ }
}
+ fail("Core reloaded whereas it was not expected to be possible");
}
/** Processes the given {@code action} for all replicas of the collection
defined in the constructor. */
@@ -297,12 +338,6 @@ public class EncryptionTestUtil {
}
}
- private static class CoreReloadException extends Exception {
- CoreReloadException(String msg, SolrException cause) {
- super(msg, cause);
- }
- }
-
/** Status of the encryption of potentially multiple cores. */
public static class EncryptionStatus {
