Author: thomasm Date: Mon Mar 11 10:40:27 2013 New Revision: 1455089 URL: http://svn.apache.org/r1455089 Log: OAK-619 Lock-free MongoMK implementation (bugfix: re-added nodes contained old properties)
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Node.java jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java?rev=1455089&r1=1455088&r2=1455089&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java (original) +++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java Mon Mar 11 10:40:27 2013 @@ -21,9 +21,11 @@ import java.lang.ref.WeakReference; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeSet; import java.util.concurrent.atomic.AtomicBoolean; @@ -36,6 +38,7 @@ import org.apache.jackrabbit.mk.blobs.Me import org.apache.jackrabbit.mk.json.JsopReader; import org.apache.jackrabbit.mk.json.JsopStream; import org.apache.jackrabbit.mk.json.JsopTokenizer; +import org.apache.jackrabbit.mk.json.JsopWriter; import org.apache.jackrabbit.mongomk.impl.blob.MongoBlobStore; import org.apache.jackrabbit.mongomk.prototype.Node.Children; import org.apache.jackrabbit.oak.commons.PathUtils; @@ -72,6 +75,7 @@ public class MongoMK implements MicroKer * The delay for asynchronous operations (delayed commit propagation and * cache update). */ + // TODO test observation with multiple Oak instances protected static final long ASYNC_DELAY = 1000; /** @@ -251,7 +255,7 @@ public class MongoMK implements MicroKer boolean isRevisionNewer(Revision x, Revision previous) { // TODO currently we only compare the timestamps - return x.compareRevisionTime(previous) >= 0; + return x.compareRevisionTime(previous) > 0; } public Node.Children readChildren(String path, String nodeId, Revision rev, int limit) { @@ -269,8 +273,8 @@ public class MongoMK implements MicroKer List<Map<String, Object>> list = store.query(DocumentStore.Collection.NODES, from, to, limit); c = new Node.Children(path, nodeId, rev); for (Map<String, Object> e : list) { - // Filter out deleted children - if (isDeleted(e, rev)) { + // filter out deleted children + if (getLiveRevision(e, rev) == null) { continue; } // TODO put the whole node in the cache @@ -288,7 +292,9 @@ public class MongoMK implements MicroKer if (map == null) { return null; } - if (isDeleted(map, rev)) { + Revision min = getLiveRevision(map, rev); + if (min == null) { + // deleted return null; } Node n = new Node(path, rev); @@ -305,7 +311,7 @@ public class MongoMK implements MicroKer @SuppressWarnings("unchecked") Map<String, String> valueMap = (Map<String, String>) v; if (valueMap != null) { - String value = getLatestValue(valueMap, rev); + String value = getLatestValue(valueMap, min, rev); String propertyName = Utils.unescapePropertyName(key); n.setProperty(propertyName, value); } @@ -314,12 +320,26 @@ public class MongoMK implements MicroKer return n; } - private String getLatestValue(Map<String, String> valueMap, Revision rev) { + /** + * Get the latest property value that is larger or equal the min revision, + * and smaller or equal the max revision. + * + * @param valueMap the revision-value map + * @param min the minimum revision (null meaning unlimited) + * @param max the maximum revision + * @return the value, or null if not found + */ + private String getLatestValue(Map<String, String> valueMap, Revision min, Revision max) { String value = null; Revision latestRev = null; for (String r : valueMap.keySet()) { Revision propRev = Revision.fromString(r); - if (includeRevision(propRev, rev)) { + if (min != null) { + if (isRevisionNewer(min, propRev)) { + continue; + } + } + if (includeRevision(propRev, max)) { if (latestRev == null || isRevisionNewer(propRev, latestRev)) { latestRev = propRev; value = valueMap.get(r); @@ -366,7 +386,59 @@ public class MongoMK implements MicroKer return ""; } // TODO implement if needed - return "{}"; + if (true) { + return "{}"; + } + if (depth != 0) { + throw new MicroKernelException("Only depth 0 is supported, depth is " + depth); + } + fromRevisionId = stripBranchRevMarker(fromRevisionId); + toRevisionId = stripBranchRevMarker(toRevisionId); + Node from = getNode(path, Revision.fromString(fromRevisionId)); + Node to = getNode(path, Revision.fromString(toRevisionId)); + if (from == null || to == null) { + // TODO implement correct behavior if the node does't/didn't exist + throw new MicroKernelException("Diff is only supported if the node exists in both cases"); + } + JsopWriter w = new JsopStream(); + for (String p : from.getPropertyNames()) { + // changed or removed properties + String fromValue = from.getProperty(p); + String toValue = to.getProperty(p); + if (!fromValue.equals(toValue)) { + w.tag('^').key(p).value(toValue).newline(); + } + } + for (String p : to.getPropertyNames()) { + // added properties + if (from.getProperty(p) == null) { + w.tag('^').key(p).value(to.getProperty(p)).newline(); + } + } + Revision fromRev = Revision.fromString(fromRevisionId); + Revision toRev = Revision.fromString(toRevisionId); + // TODO this does not work well for large child node lists + // use a MongoDB index instead + Children fromChildren = readChildren(path, from.getId(), fromRev, Integer.MAX_VALUE); + Children toChildren = readChildren(path, to.getId(), toRev, Integer.MAX_VALUE); + Set<String> childrenSet = new HashSet<String>(toChildren.children); + for (String n : fromChildren.children) { + if (!childrenSet.contains(n)) { + w.tag('-').key(n).object().endObject().newline(); + } else { + // TODO currently all children seem to diff, + // which is not necessarily the case + // (compare write counters) + w.tag('^').key(n).object().endObject().newline(); + } + } + childrenSet = new HashSet<String>(fromChildren.children); + for (String n : toChildren.children) { + if (!childrenSet.contains(n)) { + w.tag('+').key(n).object().endObject().newline(); + } + } + return w.toString(); } @Override @@ -562,18 +634,38 @@ public class MongoMK implements MicroKer // Remove the node from the cache nodeCache.remove(path + "@" + rev); } - - private boolean isDeleted(Map<String, Object> nodeProps, Revision rev) { + + /** + * Get the latest revision where the node was alive at or before the the + * provided revision. + * + * @param nodeMap the node map + * @param maxRev the maximum revision to return + * @return the earliest revision, or null if the node is deleted at the + * given revision + */ + private Revision getLiveRevision(Map<String, Object> nodeMap, + Revision maxRev) { @SuppressWarnings("unchecked") - Map<String, String> valueMap = (Map<String, String>) nodeProps + Map<String, String> valueMap = (Map<String, String>) nodeMap .get(UpdateOp.DELETED); - if (valueMap != null) { - String value = getLatestValue(valueMap, rev); - if (value == null || "true".equals(value)) { - return true; + Revision firstRev = null; + String value = null; + for (String r : valueMap.keySet()) { + Revision propRev = Revision.fromString(r); + if (isRevisionNewer(propRev, maxRev)) { + continue; } + String v = valueMap.get(r); + if (firstRev == null || isRevisionNewer(propRev, firstRev)) { + firstRev = propRev; + value = v; + } + } + if ("true".equals(value)) { + return null; } - return false; + return firstRev; } private static String stripBranchRevMarker(String revisionId) { Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Node.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Node.java?rev=1455089&r1=1455088&r2=1455089&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Node.java (original) +++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Node.java Mon Mar 11 10:40:27 2013 @@ -18,6 +18,7 @@ package org.apache.jackrabbit.mongomk.pr import java.util.ArrayList; import java.util.Map; +import java.util.Set; import org.apache.jackrabbit.mk.json.JsopWriter; @@ -47,6 +48,10 @@ public class Node { public String getProperty(String propertyName) { return properties.get(propertyName); } + + public Set<String> getPropertyNames() { + return properties.keySet(); + } public void copyTo(Node newNode) { newNode.properties.putAll(properties); Modified: jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java?rev=1455089&r1=1455088&r2=1455089&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java (original) +++ jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java Mon Mar 11 10:40:27 2013 @@ -41,7 +41,7 @@ public class SimpleTest { MongoMK mk = new MongoMK(); mk.dispose(); } - + @Test public void pathToId() { assertEquals("0:/", Utils.getIdFromPath("/")); @@ -87,6 +87,25 @@ public class SimpleTest { assertEquals("Hello", n2.getProperty("name")); mk.dispose(); } + + @Test + public void diff() { + MongoMK mk = createMK(); + String rev0 = mk.getHeadRevision(); + // TODO +// String rev1 = mk.commit("/", "+\"test\":{\"name\": \"Hello\"}", null, null); +// String rev2 = mk.commit("/", "-\"test\"", null, null); +// String rev3 = mk.commit("/", "+\"test\":{\"name\": \"Hallo\"}", null, null); +// String test0 = mk.getNodes("/test", rev0, 0, 0, Integer.MAX_VALUE, null); +// assertNull(null, test0); +// String test1 = mk.getNodes("/test", rev1, 0, 0, Integer.MAX_VALUE, null); +// assertEquals("{\"name\":\"Hello\",\":childNodeCount\":0}", test1); +// String test2 = mk.getNodes("/test", rev2, 0, 0, Integer.MAX_VALUE, null); +// assertNull(null, test2); +// String test3 = mk.getNodes("/test", rev3, 0, 0, Integer.MAX_VALUE, null); +// assertEquals("{\"name\":\"Hallo\",\":childNodeCount\":0}", test3); + mk.dispose(); + } @Test public void reAddDeleted() { @@ -109,7 +128,7 @@ public class SimpleTest { @Test public void reAddDeleted2() { MongoMK mk = createMK(); - String rev = mk.commit("/", "+\"test\":{\"child\": {}}", null, null); + String rev = mk.commit("/", "+\"test\":{\"x\":\"1\",\"child\": {}}", null, null); rev = mk.commit("/", "-\"test\"", rev, null); rev = mk.commit("/", "+\"test\":{}", null, null); String test = mk.getNodes("/test", rev, 0, 0, Integer.MAX_VALUE, null);