Author: mreutegg Date: Tue Aug 25 09:19:23 2015 New Revision: 1697623 URL: http://svn.apache.org/r1697623 Log: OAK-3283: Background read does not close StringSort
Merge revision 1697616 from trunk Modified: jackrabbit/oak/branches/1.2/ (props changed) jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java Propchange: jackrabbit/oak/branches/1.2/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Tue Aug 25 09:19:23 2015 @@ -1,3 +1,3 @@ /jackrabbit/oak/branches/1.0:1665962 -/jackrabbit/oak/trunk:1672350,1672468,1672537,1672603,1672642,1672644,1672834-1672835,1673351,1673410,1673414-1673415,1673436,1673644,1673662-1673664,1673669,1673695,1673713,1673738,1673787,1673791,1674046,1674065,1674075,1674107,1674228,1674780,1674880,1675054-1675055,1675319,1675332,1675354,1675357,1675382,1675555,1675566,1675593,1676198,1676237,1676407,1676458,1676539,1676670,1676693,1676703,1676725,1677579,1677581,1677609,1677611,1677774,1677788,1677797,1677804,1677806,1677939,1677991,1678023,1678095-1678096,1678124,1678171,1678173,1678211,1678323,1678758,1678938,1678954,1679144,1679165,1679191,1679232,1679235,1679503,1679958,1679961,1680170,1680172,1680182,1680222,1680232,1680236,1680461,1680633,1680643,1680747,1680805-1680806,1680903,1681282,1681767,1681918,1681955,1682042,1682218,1682235,1682437,1682494,1682555,1682855,1682904,1683059,1683089,1683213,1683249,1683259,1683278,1683323,1683687,1683700,1684174-1684175,1684186,1684376,1684442,1684561,1684570,1684601,1684618,1684820 ,1684868,1685023,1685075,1685370,1685552,1685589-1685590,1685840,1685964,1685977,1685989,1685999,1686023,1686032,1686097,1686162,1686229,1686234,1686253,1686414,1686780,1686854,1686857,1686971,1687053-1687055,1687175,1687196,1687198,1687220,1687239-1687240,1687301,1687441,1687553,1688089-1688090,1688172,1688179,1688349,1688421,1688436,1688453,1688616,1688622,1688634,1688636,1688817,1689003-1689004,1689008,1689577,1689581,1689623,1689810,1689828,1689831,1689833,1689903,1690017,1690043,1690047,1690057,1690247,1690249,1690634-1690637,1690650,1690669,1690674,1690885,1690941,1691139,1691151,1691159,1691167,1691183,1691188,1691210,1691280,1691307,1691331-1691333,1691345,1691384-1691385,1691401,1691509,1692133-1692134,1692156,1692250,1692274,1692363,1692382,1692478,1692955,1693002,1693030,1693209,1693421,1693525-1693526,1694007,1694393-1694394,1695050,1695122,1695280,1695299,1695457,1695482,1695507,1695521,1695540,1696194,1696242,1696285,1696578,1696759,1696916,1697373,1697410,1697582,1697 589 +/jackrabbit/oak/trunk:1672350,1672468,1672537,1672603,1672642,1672644,1672834-1672835,1673351,1673410,1673414-1673415,1673436,1673644,1673662-1673664,1673669,1673695,1673713,1673738,1673787,1673791,1674046,1674065,1674075,1674107,1674228,1674780,1674880,1675054-1675055,1675319,1675332,1675354,1675357,1675382,1675555,1675566,1675593,1676198,1676237,1676407,1676458,1676539,1676670,1676693,1676703,1676725,1677579,1677581,1677609,1677611,1677774,1677788,1677797,1677804,1677806,1677939,1677991,1678023,1678095-1678096,1678124,1678171,1678173,1678211,1678323,1678758,1678938,1678954,1679144,1679165,1679191,1679232,1679235,1679503,1679958,1679961,1680170,1680172,1680182,1680222,1680232,1680236,1680461,1680633,1680643,1680747,1680805-1680806,1680903,1681282,1681767,1681918,1681955,1682042,1682218,1682235,1682437,1682494,1682555,1682855,1682904,1683059,1683089,1683213,1683249,1683259,1683278,1683323,1683687,1683700,1684174-1684175,1684186,1684376,1684442,1684561,1684570,1684601,1684618,1684820 ,1684868,1685023,1685075,1685370,1685552,1685589-1685590,1685840,1685964,1685977,1685989,1685999,1686023,1686032,1686097,1686162,1686229,1686234,1686253,1686414,1686780,1686854,1686857,1686971,1687053-1687055,1687175,1687196,1687198,1687220,1687239-1687240,1687301,1687441,1687553,1688089-1688090,1688172,1688179,1688349,1688421,1688436,1688453,1688616,1688622,1688634,1688636,1688817,1689003-1689004,1689008,1689577,1689581,1689623,1689810,1689828,1689831,1689833,1689903,1690017,1690043,1690047,1690057,1690247,1690249,1690634-1690637,1690650,1690669,1690674,1690885,1690941,1691139,1691151,1691159,1691167,1691183,1691188,1691210,1691280,1691307,1691331-1691333,1691345,1691384-1691385,1691401,1691509,1692133-1692134,1692156,1692250,1692274,1692363,1692382,1692478,1692955,1693002,1693030,1693209,1693421,1693525-1693526,1694007,1694393-1694394,1695050,1695122,1695280,1695299,1695457,1695482,1695507,1695521,1695540,1696194,1696242,1696285,1696578,1696759,1696916,1697373,1697410,1697582,1697 589,1697616 /jackrabbit/trunk:1345480 Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java?rev=1697623&r1=1697622&r2=1697623&view=diff ============================================================================== --- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (original) +++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java Tue Aug 25 09:19:23 2015 @@ -76,6 +76,7 @@ import com.google.common.collect.Sets; import com.google.common.util.concurrent.UncheckedExecutionException; import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.commons.IOUtils; import org.apache.jackrabbit.oak.commons.jmx.AnnotatedStandardMBean; import org.apache.jackrabbit.oak.commons.json.JsopReader; import org.apache.jackrabbit.oak.commons.json.JsopTokenizer; @@ -1762,118 +1763,123 @@ public final class DocumentNodeStore Revision otherSeen = Revision.newRevision(0); StringSort externalSort = JournalEntry.newSorter(); - - Map<Revision, Revision> externalChanges = Maps.newHashMap(); - for (Map.Entry<Integer, Revision> e : lastRevMap.entrySet()) { - int machineId = e.getKey(); - if (machineId == clusterId) { - // ignore own lastRev - continue; - } - Revision r = e.getValue(); - Revision last = lastKnownRevision.get(machineId); - if (last == null || r.compareRevisionTime(last) > 0) { - lastKnownRevision.put(machineId, r); - // OAK-2345 - // only consider as external change if - // - the revision changed for the machineId - // or - // - the revision is within the time frame we remember revisions - if (last != null - || r.getTimestamp() > revisionPurgeMillis()) { - externalChanges.put(r, otherSeen); - } - // collect external changes - if (last != null && externalSort != null) { - // add changes for this particular clusterId to the externalSort - try { - fillExternalChanges(externalSort, last, r, store); - } catch (IOException e1) { - LOG.error("backgroundRead: Exception while reading external changes from journal: "+e1, e1); - externalSort = null; + + try { + Map<Revision, Revision> externalChanges = Maps.newHashMap(); + for (Map.Entry<Integer, Revision> e : lastRevMap.entrySet()) { + int machineId = e.getKey(); + if (machineId == clusterId) { + // ignore own lastRev + continue; + } + Revision r = e.getValue(); + Revision last = lastKnownRevision.get(machineId); + if (last == null || r.compareRevisionTime(last) > 0) { + lastKnownRevision.put(machineId, r); + // OAK-2345 + // only consider as external change if + // - the revision changed for the machineId + // or + // - the revision is within the time frame we remember revisions + if (last != null + || r.getTimestamp() > revisionPurgeMillis()) { + externalChanges.put(r, otherSeen); + } + // collect external changes + if (last != null && externalSort != null) { + // add changes for this particular clusterId to the externalSort + try { + fillExternalChanges(externalSort, last, r, store); + } catch (IOException e1) { + LOG.error("backgroundRead: Exception while reading external changes from journal: " + e1, e1); + IOUtils.closeQuietly(externalSort); + externalSort = null; + } } } } - } - stats.readHead = clock.getTime() - time; - time = clock.getTime(); + stats.readHead = clock.getTime() - time; + time = clock.getTime(); - if (!externalChanges.isEmpty()) { - // invalidate caches - if (externalSort == null) { - // if no externalSort available, then invalidate the classic way: everything - stats.cacheStats = store.invalidateCache(); - docChildrenCache.invalidateAll(); - } else { - try { - externalSort.sort(); - stats.cacheStats = store.invalidateCache(pathToId(externalSort)); - // OAK-3002: only invalidate affected items (using journal) - long origSize = docChildrenCache.size(); - if (origSize == 0) { - // if docChildrenCache is empty, don't bother - // calling invalidateAll either way - // (esp calling invalidateAll(Iterable) will - // potentially iterate over all keys even though - // there's nothing to be deleted) - LOG.trace("backgroundRead: docChildrenCache nothing to invalidate"); - } else { - // however, if the docChildrenCache is not empty, - // use the invalidateAll(Iterable) variant, - // passing it a Iterable<StringValue>, as that's - // what is contained in the cache - docChildrenCache.invalidateAll(asStringValueIterable(externalSort)); - long newSize = docChildrenCache.size(); - LOG.trace("backgroundRead: docChildrenCache invalidation result: orig: {}, new: {} ", origSize, newSize); - } - } catch (Exception ioe) { - LOG.error("backgroundRead: got IOException during external sorting/cache invalidation (as a result, invalidating entire cache): "+ioe, ioe); + if (!externalChanges.isEmpty()) { + // invalidate caches + if (externalSort == null) { + // if no externalSort available, then invalidate the classic way: everything stats.cacheStats = store.invalidateCache(); docChildrenCache.invalidateAll(); + } else { + try { + externalSort.sort(); + stats.cacheStats = store.invalidateCache(pathToId(externalSort)); + // OAK-3002: only invalidate affected items (using journal) + long origSize = docChildrenCache.size(); + if (origSize == 0) { + // if docChildrenCache is empty, don't bother + // calling invalidateAll either way + // (esp calling invalidateAll(Iterable) will + // potentially iterate over all keys even though + // there's nothing to be deleted) + LOG.trace("backgroundRead: docChildrenCache nothing to invalidate"); + } else { + // however, if the docChildrenCache is not empty, + // use the invalidateAll(Iterable) variant, + // passing it a Iterable<StringValue>, as that's + // what is contained in the cache + docChildrenCache.invalidateAll(asStringValueIterable(externalSort)); + long newSize = docChildrenCache.size(); + LOG.trace("backgroundRead: docChildrenCache invalidation result: orig: {}, new: {} ", origSize, newSize); + } + } catch (Exception ioe) { + LOG.error("backgroundRead: got IOException during external sorting/cache invalidation (as a result, invalidating entire cache): "+ioe, ioe); + stats.cacheStats = store.invalidateCache(); + docChildrenCache.invalidateAll(); + } } - } - stats.cacheInvalidationTime = clock.getTime() - time; - time = clock.getTime(); + stats.cacheInvalidationTime = clock.getTime() - time; + time = clock.getTime(); - // make sure update to revision comparator is atomic - // and no local commit is in progress - backgroundOperationLock.writeLock().lock(); - try { - stats.lock = clock.getTime() - time; + // make sure update to revision comparator is atomic + // and no local commit is in progress + backgroundOperationLock.writeLock().lock(); + try { + stats.lock = clock.getTime() - time; - // the latest revisions of the current cluster node - // happened before the latest revisions of other cluster nodes - revisionComparator.add(newRevision(), headSeen); - // then we saw other revisions - for (Map.Entry<Revision, Revision> e : externalChanges.entrySet()) { - revisionComparator.add(e.getKey(), e.getValue()); - } + // the latest revisions of the current cluster node + // happened before the latest revisions of other cluster nodes + revisionComparator.add(newRevision(), headSeen); + // then we saw other revisions + for (Map.Entry<Revision, Revision> e : externalChanges.entrySet()) { + revisionComparator.add(e.getKey(), e.getValue()); + } - Revision oldHead = headRevision; - // the new head revision is after other revisions - setHeadRevision(newRevision()); - if (dispatchChange) { - time = clock.getTime(); - if (externalSort != null) { - // then there were external changes and reading them - // was successful -> apply them to the diff cache - try { - JournalEntry.applyTo(externalSort, diffCache, oldHead, headRevision); - } catch (Exception e1) { - LOG.error("backgroundRead: Exception while processing external changes from journal: "+e1, e1); + Revision oldHead = headRevision; + // the new head revision is after other revisions + setHeadRevision(newRevision()); + if (dispatchChange) { + time = clock.getTime(); + if (externalSort != null) { + // then there were external changes and reading them + // was successful -> apply them to the diff cache + try { + JournalEntry.applyTo(externalSort, diffCache, oldHead, headRevision); + } catch (Exception e1) { + LOG.error("backgroundRead: Exception while processing external changes from journal: "+e1, e1); + } } - } - stats.populateDiffCache = clock.getTime() - time; - time = clock.getTime(); + stats.populateDiffCache = clock.getTime() - time; + time = clock.getTime(); - dispatcher.contentChanged(getRoot().fromExternalChange(), null); + dispatcher.contentChanged(getRoot().fromExternalChange(), null); + } + } finally { + backgroundOperationLock.writeLock().unlock(); } - } finally { - backgroundOperationLock.writeLock().unlock(); + stats.dispatchChanges = clock.getTime() - time; + time = clock.getTime(); } - stats.dispatchChanges = clock.getTime() - time; - time = clock.getTime(); + } finally { + IOUtils.closeQuietly(externalSort); } revisionComparator.purge(revisionPurgeMillis()); stats.purge = clock.getTime() - time;