This is an automated email from the ASF dual-hosted git repository. daim pushed a commit to branch OAK-10199 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 1f4d7cb1a65920f978e559fa94f321163e3a3c37 Author: Rishabh Kumar <d...@adobe.com> AuthorDate: Wed Aug 9 00:08:12 2023 +0530 OAK-10199 : removed DetailedGC code changes for RDBDocumentStore --- .../plugins/document/VersionGarbageCollector.java | 4 +- .../oak/plugins/document/rdb/RDBDocumentStore.java | 16 +--- .../plugins/document/rdb/RDBDocumentStoreJDBC.java | 16 ++-- .../plugins/document/rdb/RDBVersionGCSupport.java | 104 --------------------- .../oak/plugins/document/VersionGCInitTest.java | 13 +-- 5 files changed, 18 insertions(+), 135 deletions(-) diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java index 8f2c977274..a32ded6c5b 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java @@ -350,7 +350,7 @@ public class VersionGarbageCollector { return "VersionGCStats{" + "ignoredGCDueToCheckPoint=" + ignoredGCDueToCheckPoint + - "ignoredDetailedGCDueToCheckPoint=" + ignoredDetailedGCDueToCheckPoint + + ", ignoredDetailedGCDueToCheckPoint=" + ignoredDetailedGCDueToCheckPoint + ", canceled=" + canceled + ", deletedDocGCCount=" + deletedDocGCCount + " (of which leaf: " + deletedLeafDocGCCount + ")" + ", updateResurrectedGCCount=" + updateResurrectedGCCount + @@ -950,7 +950,7 @@ public class VersionGarbageCollector { updatedDocs = (int) oldDocs.stream().filter(Objects::nonNull).count(); stats.updatedDetailedGCDocsCount += updatedDocs; stats.deletedPropsGCCount += deletedProps; - log.info("Updated [{}] documents, deleted [{}] properties", updatedDocs, deletedProps); + log.debug("Updated [{}] documents, deleted [{}] properties", updatedDocs, deletedProps); // now reset delete metadata updateOpList.clear(); deletedPropsCountMap.clear(); diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java index 25a3aca5e7..82c09e213d 100755 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java @@ -971,8 +971,8 @@ public class RDBDocumentStore implements DocumentStore { public static String VERSIONPROP = "__version"; // set of supported indexed properties - private static final Set<String> INDEXEDPROPERTIES = new HashSet<>(Arrays.asList(MODIFIED, - NodeDocument.HAS_BINARY_FLAG, NodeDocument.DELETED_ONCE, NodeDocument.SD_TYPE, NodeDocument.SD_MAX_REV_TIME_IN_SECS, VERSIONPROP, ID)); + private static final Set<String> INDEXEDPROPERTIES = new HashSet<String>(Arrays.asList(new String[] { MODIFIED, + NodeDocument.HAS_BINARY_FLAG, NodeDocument.DELETED_ONCE, NodeDocument.SD_TYPE, NodeDocument.SD_MAX_REV_TIME_IN_SECS, VERSIONPROP })); // set of required table columns private static final Set<String> REQUIREDCOLUMNS = Collections.unmodifiableSet(new HashSet<String>(Arrays.asList( @@ -1840,7 +1840,7 @@ public class RDBDocumentStore implements DocumentStore { } protected <T extends Document> Iterable<T> queryAsIterable(final Collection<T> collection, String fromKey, String toKey, - final List<String> excludeKeyPatterns, final List<QueryCondition> conditions, final int limit, final List<String> sortBy) { + final List<String> excludeKeyPatterns, final List<QueryCondition> conditions, final int limit, final String sortBy) { final RDBTableMetaData tmd = getTable(collection); Set<String> allowedProps = Sets.intersection(INDEXEDPROPERTIES, tmd.getColumnProperties()); @@ -1853,16 +1853,6 @@ public class RDBDocumentStore implements DocumentStore { } } - if (sortBy != null && !sortBy.isEmpty()) { - for (String key : sortBy) { - if (!allowedProps.contains(key)) { - final String message = "indexed property " + key + " not supported. supported properties are " + allowedProps; - LOG.error(message); - throw new UnsupportedIndexedPropertyException(message); - } - } - } - final String from = collection == Collection.NODES && NodeDocument.MIN_ID_VALUE.equals(fromKey) ? null : fromKey; final String to = collection == Collection.NODES && NodeDocument.MAX_ID_VALUE.equals(toKey) ? null : toKey; diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java index 87d0f4b4dd..5caa65d875 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java @@ -16,11 +16,8 @@ */ package org.apache.jackrabbit.oak.plugins.document.rdb; -import static java.util.List.of; -import static java.util.stream.Collectors.joining; import static org.apache.jackrabbit.guava.common.collect.Iterables.transform; import static org.apache.jackrabbit.guava.common.collect.Sets.newHashSet; -import static org.apache.jackrabbit.oak.plugins.document.Document.ID; import static org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore.CHAR2OCTETRATIO; import static org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore.asBytes; import static org.apache.jackrabbit.oak.plugins.document.rdb.RDBJDBCTools.asDocumentStoreException; @@ -462,7 +459,7 @@ public class RDBDocumentStoreJDBC { + excludeKeyPatterns + ", conditions=" + conditions + ", limit=" + limit) : null); stmt = prepareQuery(connection, tmd, fields, minId, - maxId, excludeKeyPatterns, conditions, limit, of(ID)); + maxId, excludeKeyPatterns, conditions, limit, "ID"); rs = stmt.executeQuery(); while (rs.next() && result.size() < limit) { int field = 1; @@ -557,7 +554,7 @@ public class RDBDocumentStoreJDBC { @NotNull public Iterator<RDBRow> queryAsIterator(RDBConnectionHandler ch, RDBTableMetaData tmd, String minId, String maxId, - List<String> excludeKeyPatterns, List<QueryCondition> conditions, int limit, List<String> sortBy) throws SQLException { + List<String> excludeKeyPatterns, List<QueryCondition> conditions, int limit, String sortBy) throws SQLException { return new ResultSetIterator(ch, tmd, minId, maxId, excludeKeyPatterns, conditions, limit, sortBy); } @@ -576,7 +573,7 @@ public class RDBDocumentStoreJDBC { private long pstart; public ResultSetIterator(RDBConnectionHandler ch, RDBTableMetaData tmd, String minId, String maxId, - List<String> excludeKeyPatterns, List<QueryCondition> conditions, int limit, List<String> sortBy) throws SQLException { + List<String> excludeKeyPatterns, List<QueryCondition> conditions, int limit, String sortBy) throws SQLException { long start = System.currentTimeMillis(); try { this.ch = ch; @@ -698,7 +695,7 @@ public class RDBDocumentStoreJDBC { @NotNull private PreparedStatement prepareQuery(Connection connection, RDBTableMetaData tmd, String columns, String minId, String maxId, - List<String> excludeKeyPatterns, List<QueryCondition> conditions, int limit, List<String> sortBy) throws SQLException { + List<String> excludeKeyPatterns, List<QueryCondition> conditions, int limit, String sortBy) throws SQLException { StringBuilder selectClause = new StringBuilder(); @@ -717,8 +714,8 @@ public class RDBDocumentStoreJDBC { query.append(" where ").append(whereClause); } - if (sortBy != null && !sortBy.isEmpty()) { - query.append(" order by ").append(sortBy.stream().map(INDEXED_PROP_MAPPING::get).collect(joining(", "))); + if (sortBy != null) { + query.append(" order by ID"); } if (limit != Integer.MAX_VALUE) { @@ -969,7 +966,6 @@ public class RDBDocumentStoreJDBC { tmp.put(NodeDocument.SD_TYPE, "SDTYPE"); tmp.put(NodeDocument.SD_MAX_REV_TIME_IN_SECS, "SDMAXREVTIME"); tmp.put(RDBDocumentStore.VERSIONPROP, "VERSION"); - tmp.put(ID, "ID"); INDEXED_PROP_MAPPING = Collections.unmodifiableMap(tmp); } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java index 27c582311b..a463499793 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java @@ -16,23 +16,7 @@ */ package org.apache.jackrabbit.oak.plugins.document.rdb; -import static java.util.Comparator.comparing; -import static java.util.List.of; -import static java.util.Optional.empty; -import static java.util.Optional.ofNullable; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Stream.concat; -import static java.util.stream.StreamSupport.stream; import static org.apache.jackrabbit.guava.common.collect.Iterables.filter; -import static org.apache.jackrabbit.guava.common.collect.Iterables.size; -import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES; -import static org.apache.jackrabbit.oak.plugins.document.Document.ID; -import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MIN_ID_VALUE; -import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MODIFIED_IN_SECS; -import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.getModifiedInSecs; -import static org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore.EMPTY_KEY_PATTERN; -import static org.apache.jackrabbit.oak.plugins.document.util.CloseableIterable.wrap; -import static org.apache.jackrabbit.oak.plugins.document.util.Utils.closeIfCloseable; import java.io.Closeable; import java.io.IOException; @@ -41,15 +25,11 @@ import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; -import java.util.stream.Stream; import org.apache.jackrabbit.oak.commons.properties.SystemPropertySupplier; import org.apache.jackrabbit.oak.plugins.document.Collection; -import org.apache.jackrabbit.oak.plugins.document.Document; import org.apache.jackrabbit.oak.plugins.document.DocumentStoreException; import org.apache.jackrabbit.oak.plugins.document.NodeDocument; import org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType; @@ -60,7 +40,6 @@ import org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore.Unsupport import org.apache.jackrabbit.oak.plugins.document.util.CloseableIterable; import org.apache.jackrabbit.oak.plugins.document.util.Utils; import org.apache.jackrabbit.oak.stats.Clock; -import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -106,67 +85,6 @@ public class RDBVersionGCSupport extends VersionGCSupport { } } - /** - * Returns documents that have a {@link NodeDocument#MODIFIED_IN_SECS} value - * within the given range and are greater than given @{@link NodeDocument#ID}. - * <p> - * The two passed modified timestamps are in milliseconds - * since the epoch and the implementation will convert them to seconds at - * the granularity of the {@link NodeDocument#MODIFIED_IN_SECS} field and - * then perform the comparison. - * - * - * @param fromModified the lower bound modified timestamp (inclusive) - * @param toModified the upper bound modified timestamp (exclusive) - * @param limit the limit of documents to return - * @param fromId the lower bound {@link NodeDocument#ID} - * @return matching documents. - */ - @Override - public Iterable<NodeDocument> getModifiedDocs(final long fromModified, final long toModified, final int limit, - @NotNull final String fromId) { - // (_modified = fromModified && _id > fromId || _modified > fromModified && _modified < toModified) - // TODO : introduce support for OR where clause in RDBDocumentStore - final List<QueryCondition> c1 = of(new QueryCondition(MODIFIED_IN_SECS, "=", getModifiedInSecs(fromModified)), - new QueryCondition(ID, ">", of(fromId))); - - final List<QueryCondition> c2 = of(new QueryCondition(MODIFIED_IN_SECS, "<", getModifiedInSecs(toModified)), - new QueryCondition(MODIFIED_IN_SECS, ">", getModifiedInSecs(fromModified))); - - if (MODE == 1) { - return getNodeDocuments(() -> getIterator(EMPTY_KEY_PATTERN, c1), () -> getIterator(EMPTY_KEY_PATTERN, c2), limit); - } else { - return getNodeDocuments(() -> store.queryAsIterable(NODES, null, null, EMPTY_KEY_PATTERN, c1, limit, of(MODIFIED_IN_SECS, ID)), - () -> store.queryAsIterable(NODES, null, null, EMPTY_KEY_PATTERN, c2, limit, of(MODIFIED_IN_SECS, ID)), - limit); - } - } - - /** - * To fetch {@link NodeDocument} from database - * - * @param supplier1 document supplier on basis of 1st Condition - * @param supplier2 document supplier on basis of 2nd Condition - * @param limit no. of documents to fetch from db - * @return sorted documents supplied by supplier1 & supplier2 - */ - private Iterable<NodeDocument> getNodeDocuments(final Supplier<Iterable<NodeDocument>> supplier1, final Supplier<Iterable<NodeDocument>> supplier2, final int limit) { - - final Iterable<NodeDocument> itr1 = supplier1.get(); - if (size(itr1) >= limit) { - return itr1; - } - - final Iterable<NodeDocument> itr2 = supplier2.get(); - - final Stream<NodeDocument> s1 = stream(itr1.spliterator(), false); - final Stream<NodeDocument> s2 = stream(itr2.spliterator(), false); - return wrap(concat(s1, s2).sorted((o1, o2) -> comparing(NodeDocument::getModified).thenComparing(Document::getId).compare(o1, o2)).limit(limit).collect(toList()), () -> { - closeIfCloseable(itr1); - closeIfCloseable(itr2); - }); - } - @Override protected Iterable<NodeDocument> identifyGarbage(final Set<SplitDocType> gcTypes, final RevisionVector sweepRevs, final long oldestRevTimeStamp) { @@ -321,28 +239,6 @@ public class RDBVersionGCSupport extends VersionGCSupport { } } - /** - * Retrieve the time of the oldest modified document. - * - * @param clock System Clock - * @return the timestamp of the oldest modified document. - */ - @Override - public Optional<NodeDocument> getOldestModifiedDoc(Clock clock) { - - Iterable<NodeDocument> modifiedDocs = null; - try { - modifiedDocs = getModifiedDocs(0L, clock.getTime(), 1, MIN_ID_VALUE); - return modifiedDocs.iterator().hasNext() ? ofNullable(modifiedDocs.iterator().next()) : empty(); - } catch (DocumentStoreException ex) { - LOG.error("getOldestModifiedDoc() <- Error ", ex); - } finally { - closeIfCloseable(modifiedDocs); - } - LOG.info("No Modified Doc has been found, retuning empty"); - return empty(); - } - @Override public long getDeletedOnceCount() { return store.queryCount(Collection.NODES, null, null, RDBDocumentStore.EMPTY_KEY_PATTERN, diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCInitTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCInitTest.java index 0c6b2fccdf..eec3e33b6b 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCInitTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCInitTest.java @@ -31,6 +31,7 @@ import static org.apache.jackrabbit.oak.plugins.document.DetailGCHelper.enableDe import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MIN_ID_VALUE; import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_DETAILED_GC_DOCUMENT_ID_PROP; import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_DETAILED_GC_TIMESTAMP_PROP; +import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_ID; import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getIdFromPath; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -51,12 +52,12 @@ public class VersionGCInitTest { @Test public void lazyInitialize() throws Exception { DocumentStore store = ns.getDocumentStore(); - Document vgc = store.find(SETTINGS, "versionGC"); + Document vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID); assertNull(vgc); ns.getVersionGarbageCollector().gc(1, DAYS); - vgc = store.find(SETTINGS, "versionGC"); + vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID); assertNotNull(vgc); assertEquals(0L, vgc.get(SETTINGS_COLLECTION_DETAILED_GC_TIMESTAMP_PROP)); assertNull(vgc.get(SETTINGS_COLLECTION_DETAILED_GC_DOCUMENT_ID_PROP)); @@ -65,7 +66,7 @@ public class VersionGCInitTest { @Test public void lazyInitializeWithDetailedGC() throws Exception { DocumentStore store = ns.getDocumentStore(); - Document vgc = store.find(SETTINGS, "versionGC"); + Document vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID); assertNull(vgc); enableDetailGC(ns.getVersionGarbageCollector()); @@ -77,7 +78,7 @@ public class VersionGCInitTest { store.createOrUpdate(NODES, op); VersionGCStats stats = ns.getVersionGarbageCollector().gc(1, DAYS); - vgc = store.find(SETTINGS, "versionGC"); + vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID); assertNotNull(vgc); assertEquals(stats.oldestModifiedDocTimeStamp, vgc.get(SETTINGS_COLLECTION_DETAILED_GC_TIMESTAMP_PROP)); assertEquals(stats.oldestModifiedDocId, vgc.get(SETTINGS_COLLECTION_DETAILED_GC_DOCUMENT_ID_PROP)); @@ -87,13 +88,13 @@ public class VersionGCInitTest { @Test public void lazyInitializeWithDetailedGCWithNoData() throws Exception { DocumentStore store = ns.getDocumentStore(); - Document vgc = store.find(SETTINGS, "versionGC"); + Document vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID); assertNull(vgc); enableDetailGC(ns.getVersionGarbageCollector()); VersionGCStats stats = ns.getVersionGarbageCollector().gc(1, DAYS); - vgc = store.find(SETTINGS, "versionGC"); + vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID); assertNotNull(vgc); assertEquals(stats.oldestModifiedDocTimeStamp, vgc.get(SETTINGS_COLLECTION_DETAILED_GC_TIMESTAMP_PROP)); assertEquals(stats.oldestModifiedDocId, vgc.get(SETTINGS_COLLECTION_DETAILED_GC_DOCUMENT_ID_PROP));