This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch nouveau-purge
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 6dc45453fe7e22193b456199ebb10f8b8595bfea
Author: Robert Newson <rnew...@apache.org>
AuthorDate: Thu May 18 16:28:30 2023 +0100
introduce purge sequence
---
.../couchdb/nouveau/api/DocumentDeleteRequest.java | 12 +++++--
.../org/apache/couchdb/nouveau/api/IndexInfo.java | 12 ++++++-
.../org/apache/couchdb/nouveau/core/Index.java | 37 ++++++++++++++++++----
.../couchdb/nouveau/lucene9/Lucene9Index.java | 10 +++---
.../couchdb/nouveau/resources/IndexResource.java | 9 +++---
.../couchdb/nouveau/lucene9/Lucene9IndexTest.java | 27 ++++++++++++++--
6 files changed, 86 insertions(+), 21 deletions(-)
diff --git
a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/DocumentDeleteRequest.java
b/nouveau/src/main/java/org/apache/couchdb/nouveau/api/DocumentDeleteRequest.java
index 6f22effda..99f6f5d9f 100644
---
a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/DocumentDeleteRequest.java
+++
b/nouveau/src/main/java/org/apache/couchdb/nouveau/api/DocumentDeleteRequest.java
@@ -24,15 +24,18 @@ public class DocumentDeleteRequest {
@Positive
private long seq;
+ private boolean purge;
+
public DocumentDeleteRequest() {
// Jackson deserialization
}
- public DocumentDeleteRequest(long seq) {
+ public DocumentDeleteRequest(long seq, final boolean purge) {
if (seq < 1) {
throw new IllegalArgumentException("seq must be 1 or greater");
}
this.seq = seq;
+ this.purge = purge;
}
@JsonProperty
@@ -40,8 +43,13 @@ public class DocumentDeleteRequest {
return seq;
}
+ @JsonProperty
+ public boolean isPurge() {
+ return purge;
+ }
+
@Override
public String toString() {
- return "DocumentDeleteRequest [seq=" + seq + "]";
+ return "DocumentDeleteRequest [seq=" + seq + ", purge=" + purge + "]";
}
}
diff --git
a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexInfo.java
b/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexInfo.java
index e807c7c70..9958c7780 100644
--- a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexInfo.java
+++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexInfo.java
@@ -24,6 +24,9 @@ public final class IndexInfo {
@PositiveOrZero
private final long updateSeq;
+ @PositiveOrZero
+ private final long purgeSeq;
+
@PositiveOrZero
private final int numDocs;
@@ -32,9 +35,11 @@ public final class IndexInfo {
public IndexInfo(
@JsonProperty("update_seq") final long updateSeq,
+ @JsonProperty("purge_seq") final long purgeSeq,
@JsonProperty("num_docs") final int numDocs,
@JsonProperty("disk_size") final long diskSize) {
this.updateSeq = updateSeq;
+ this.purgeSeq = purgeSeq;
this.numDocs = numDocs;
this.diskSize = diskSize;
}
@@ -51,8 +56,13 @@ public final class IndexInfo {
return updateSeq;
}
+ public long getPurgeSeq() {
+ return purgeSeq;
+ }
+
@Override
public String toString() {
- return "IndexInfo [updateSeq=" + updateSeq + ", numDocs=" + numDocs +
", diskSize=" + diskSize + "]";
+ return "IndexInfo [updateSeq=" + updateSeq + ", purgeSeq=" + purgeSeq
+ ", numDocs=" + numDocs + ", diskSize="
+ + diskSize + "]";
}
}
diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/Index.java
b/nouveau/src/main/java/org/apache/couchdb/nouveau/core/Index.java
index 5b31938e9..6cdd5ef31 100644
--- a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/Index.java
+++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/core/Index.java
@@ -36,13 +36,15 @@ import org.apache.couchdb.nouveau.api.SearchResults;
public abstract class Index implements Closeable {
private long updateSeq;
+ private long purgeSeq;
private boolean deleteOnClose = false;
private long lastCommit = now();
private volatile boolean closed;
private final Semaphore permits = new Semaphore(Integer.MAX_VALUE);
- protected Index(final long updateSeq) {
+ protected Index(final long updateSeq, final long purgeSeq) {
this.updateSeq = updateSeq;
+ this.purgeSeq = purgeSeq;
}
public final boolean tryAcquire() {
@@ -74,7 +76,7 @@ public abstract class Index implements Closeable {
public final IndexInfo info() throws IOException {
final int numDocs = doNumDocs();
final long diskSize = doDiskSize();
- return new IndexInfo(updateSeq, numDocs, diskSize);
+ return new IndexInfo(updateSeq, purgeSeq, numDocs, diskSize);
}
protected abstract int doNumDocs() throws IOException;
@@ -90,9 +92,15 @@ public abstract class Index implements Closeable {
protected abstract void doUpdate(final String docId, final
DocumentUpdateRequest request) throws IOException;
public final synchronized void delete(final String docId, final
DocumentDeleteRequest request) throws IOException {
- assertUpdateSeqIsLower(request.getSeq());
- doDelete(docId, request);
- incrementUpdateSeq(request.getSeq());
+ if (request.isPurge()) {
+ assertPurgeSeqIsLower(request.getSeq());
+ doDelete(docId, request);
+ incrementPurgeSeq(request.getSeq());
+ } else {
+ assertUpdateSeqIsLower(request.getSeq());
+ doDelete(docId, request);
+ incrementUpdateSeq(request.getSeq());
+ }
}
protected abstract void doDelete(final String docId, final
DocumentDeleteRequest request) throws IOException;
@@ -105,10 +113,12 @@ public abstract class Index implements Closeable {
public final boolean commit() throws IOException {
final long updateSeq;
+ final long purgeSeq;
synchronized (this) {
updateSeq = this.updateSeq;
+ purgeSeq = this.purgeSeq;
}
- final boolean result = doCommit(updateSeq);
+ final boolean result = doCommit(updateSeq, purgeSeq);
if (result) {
final long now = now();
synchronized (this) {
@@ -118,7 +128,7 @@ public abstract class Index implements Closeable {
return result;
}
- protected abstract boolean doCommit(final long updateSeq) throws
IOException;
+ protected abstract boolean doCommit(final long updateSeq, final long
purgeSeq) throws IOException;
@Override
public final void close() throws IOException {
@@ -159,6 +169,19 @@ public abstract class Index implements Closeable {
this.updateSeq = updateSeq;
}
+ protected final void assertPurgeSeqIsLower(final long purgeSeq) throws
UpdatesOutOfOrderException {
+ assert Thread.holdsLock(this);
+ if (!(purgeSeq > this.purgeSeq)) {
+ throw new UpdatesOutOfOrderException(this.purgeSeq, purgeSeq);
+ }
+ }
+
+ protected final void incrementPurgeSeq(final long purgeSeq) throws
IOException {
+ assert Thread.holdsLock(this);
+ assertPurgeSeqIsLower(purgeSeq);
+ this.purgeSeq = purgeSeq;
+ }
+
public boolean needsCommit(final long duration, final TimeUnit unit) {
final long commitNeededSince = now() - unit.toNanos(duration);
synchronized (this) {
diff --git
a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java
b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java
index 69e6b1264..ce7f7d7ec 100644
--- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java
+++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java
@@ -25,7 +25,6 @@ import java.nio.charset.CharsetDecoder;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -104,8 +103,9 @@ public class Lucene9Index extends Index {
final Analyzer analyzer,
final IndexWriter writer,
final long updateSeq,
+ final long purgeSeq,
final SearcherManager searcherManager) {
- super(updateSeq);
+ super(updateSeq, purgeSeq);
this.analyzer = Objects.requireNonNull(analyzer);
this.writer = Objects.requireNonNull(writer);
this.searcherManager = Objects.requireNonNull(searcherManager);
@@ -144,12 +144,12 @@ public class Lucene9Index extends Index {
}
@Override
- public boolean doCommit(final long updateSeq) throws IOException {
+ public boolean doCommit(final long updateSeq, final long purgeSeq) throws
IOException {
if (!writer.hasUncommittedChanges()) {
return false;
}
- writer.setLiveCommitData(
- Collections.singletonMap("update_seq",
Long.toString(updateSeq)).entrySet());
+ writer.setLiveCommitData(Map.of("update_seq",
Long.toString(updateSeq), "purge_seq", Long.toString(purgeSeq))
+ .entrySet());
writer.commit();
return true;
}
diff --git
a/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java
b/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java
index 0776df52f..e12ab55fc 100644
---
a/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java
+++
b/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java
@@ -126,19 +126,20 @@ public final class IndexResource {
final IndexWriterConfig config = new IndexWriterConfig(analyzer);
config.setUseCompoundFile(false);
final IndexWriter writer = new IndexWriter(dir, config);
- final long updateSeq = getUpdateSeq(writer);
+ final long updateSeq = getSeq(writer, "update_seq");
+ final long purgeSeq = getSeq(writer, "purge_seq");
final SearcherManager searcherManager = new
SearcherManager(writer, searcherFactory);
- return new Lucene9Index(analyzer, writer, updateSeq,
searcherManager);
+ return new Lucene9Index(analyzer, writer, updateSeq, purgeSeq,
searcherManager);
};
}
- private static long getUpdateSeq(final IndexWriter writer) throws
IOException {
+ private static long getSeq(final IndexWriter writer, final String key)
throws IOException {
final Iterable<Map.Entry<String, String>> commitData =
writer.getLiveCommitData();
if (commitData == null) {
return 0L;
}
for (Map.Entry<String, String> entry : commitData) {
- if (entry.getKey().equals("update_seq")) {
+ if (entry.getKey().equals(key)) {
return Long.parseLong(entry.getValue());
}
}
diff --git
a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexTest.java
b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexTest.java
index 79151a263..5d989804b 100644
---
a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexTest.java
+++
b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexTest.java
@@ -199,11 +199,34 @@ public class Lucene9IndexTest {
IndexInfo info = index.info();
assertThat(info.getNumDocs()).isEqualTo(1);
- index.delete("foo", new DocumentDeleteRequest(3));
+ index.delete("foo", new DocumentDeleteRequest(3, false));
index.commit();
info = index.info();
assertThat(info.getNumDocs()).isEqualTo(0);
+ assertThat(info.getUpdateSeq()).isEqualTo(3);
+ } finally {
+ cleanup(index);
+ }
+ }
+
+ @Test
+ public void testPurge(@TempDir Path path) throws IOException {
+ Index index = setup(path);
+ try {
+ final Collection<Field> fields = List.of(new DoubleField("bar",
12.0, false));
+ index.update("foo", new DocumentUpdateRequest(2, null, fields));
+ index.commit();
+
+ IndexInfo info = index.info();
+ assertThat(info.getNumDocs()).isEqualTo(1);
+
+ index.delete("foo", new DocumentDeleteRequest(3, true));
+ index.commit();
+
+ info = index.info();
+ assertThat(info.getNumDocs()).isEqualTo(0);
+ assertThat(info.getPurgeSeq()).isEqualTo(3);
} finally {
cleanup(index);
}
@@ -217,7 +240,7 @@ public class Lucene9IndexTest {
config.setUseCompoundFile(false);
final IndexWriter writer = new IndexWriter(dir, config);
final SearcherManager searcherManager = new
SearcherManager(writer, null);
- return new Lucene9Index(analyzer, writer, 0L, searcherManager);
+ return new Lucene9Index(analyzer, writer, 0L, 0L, searcherManager);
};
}
}
