[
https://issues.apache.org/jira/browse/OAK-8006?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16755098#comment-16755098
]
Andrei Dulceanu commented on OAK-8006:
--------------------------------------
[~frm], thanks for providing the sketch for a test case. In
[^OAK-8006-test.patch] I managed to create a test based on it, which actually
exhibits the desired behaviour. This is the good news. The bad news is that the
first patch proposed wasn't effective. Now the problem re-surfaced higher in
the call chain:
{noformat}
17:01:30.578 INFO [standby-run-1] StandbyClientSyncExecution.java:193 Copying
data segment dabdbb4c-4915-4466-a68a-02fdd9c286e7 from primary
17:01:30.590 INFO [standby-run-1] StandbyClientSyncExecution.java:193 Copying
data segment 009edfc9-1771-43f6-a18a-a26eb7bac3ad from primary
17:01:30.613 ERROR [standby-run-1] StandbyClientSync.java:183 Failed
synchronizing state.
org.apache.jackrabbit.oak.segment.SegmentNotFoundException: Segment
009edfc9-1771-43f6-a18a-a26eb7bac3ad not found
at
org.apache.jackrabbit.oak.segment.file.AbstractFileStore.readSegmentUncached(AbstractFileStore.java:284)
at
org.apache.jackrabbit.oak.segment.file.FileStore.lambda$14(FileStore.java:498)
at
org.apache.jackrabbit.oak.segment.SegmentCache$EmptyCache.getSegment(SegmentCache.java:229)
at
org.apache.jackrabbit.oak.segment.file.FileStore.readSegment(FileStore.java:498)
at
org.apache.jackrabbit.oak.segment.SegmentId.getSegment(SegmentId.java:153)
at org.apache.jackrabbit.oak.segment.Record.getSegment(Record.java:70)
at
org.apache.jackrabbit.oak.segment.ListRecord.getEntries(ListRecord.java:99)
at
org.apache.jackrabbit.oak.segment.ListRecord.getEntries(ListRecord.java:92)
at
org.apache.jackrabbit.oak.segment.SegmentStream.read(SegmentStream.java:165)
at com.google.common.io.ByteStreams.read(ByteStreams.java:828)
at com.google.common.io.ByteStreams.readFully(ByteStreams.java:695)
at com.google.common.io.ByteStreams.readFully(ByteStreams.java:676)
at
org.apache.jackrabbit.oak.segment.SegmentStream.getString(SegmentStream.java:104)
at
org.apache.jackrabbit.oak.segment.Segment.readString(Segment.java:467)
at
org.apache.jackrabbit.oak.segment.SegmentBlob.readLongBlobId(SegmentBlob.java:210)
at
org.apache.jackrabbit.oak.segment.SegmentBlob.readBlobId(SegmentBlob.java:163)
at
org.apache.jackrabbit.oak.segment.file.AbstractFileStore$3.consume(AbstractFileStore.java:262)
at
org.apache.jackrabbit.oak.segment.Segment.forEachRecord(Segment.java:601)
at
org.apache.jackrabbit.oak.segment.file.AbstractFileStore.readBinaryReferences(AbstractFileStore.java:257)
at
org.apache.jackrabbit.oak.segment.file.FileStore.writeSegment(FileStore.java:534)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.copySegmentFromPrimary(StandbyClientSyncExecution.java:225)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.copySegmentHierarchyFromPrimary(StandbyClientSyncExecution.java:194)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.compareAgainstBaseState(StandbyClientSyncExecution.java:101)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.execute(StandbyClientSyncExecution.java:76)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync.run(StandbyClientSync.java:165)
at
org.apache.jackrabbit.oak.segment.standby.StandbySegmentBlobTestIT.testSyncWithLongBlobId(StandbySegmentBlobTestIT.java:104)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at
org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:89)
at
org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:41)
at
org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:541)
at
org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:763)
at
org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:463)
at
org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:209)
{noformat}
> SegmentBlob#readLongBlobId might cause SegmentNotFoundException on standby
> --------------------------------------------------------------------------
>
> Key: OAK-8006
> URL: https://issues.apache.org/jira/browse/OAK-8006
> Project: Jackrabbit Oak
> Issue Type: Bug
> Components: segment-tar, tarmk-standby
> Affects Versions: 1.6.0
> Reporter: Andrei Dulceanu
> Assignee: Andrei Dulceanu
> Priority: Major
> Labels: cold-standby
> Fix For: 1.12, 1.10.1, 1.8.12
>
> Attachments: OAK-8006-test.patch, OAK-8006.patch
>
>
> When persisting a segment transferred from master, among others, the cold
> standby needs to read the binary references from the segment. While this
> usually doesn't involve any additional reads from any other segments, there
> is a special case concerning binary IDs larger than 4092 bytes. These can
> live in other segments (which got transferred prior to the current segment
> and are already on the standby), but it might also be the case that the
> binary ID is stored in the same segment. If this happens, the call to
> {{blobId.getSegment()}}[0], triggers a new read of the current, un-persisted
> segment . Thus, a {{SegmentNotFoundException}} is thrown:
> {noformat}
> 22.01.2019 09:35:59.345 *ERROR* [standby-run-1]
> org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync Failed
> synchronizing state.
> org.apache.jackrabbit.oak.segment.SegmentNotFoundException: Segment
> d40a9da6-06a2-4dc0-ab91-5554a33c02b0 not found
> at
> org.apache.jackrabbit.oak.segment.file.AbstractFileStore.readSegmentUncached(AbstractFileStore.java:284)
> [org.apache.jackrabbit.oak-segment-tar:1.10.0]
> at
> org.apache.jackrabbit.oak.segment.file.FileStore.lambda$readSegment$10(FileStore.java:498)
> [org.apache.jackrabbit.oak-segment-tar:1.10.0]
> at
> org.apache.jackrabbit.oak.segment.SegmentCache$NonEmptyCache.lambda$getSegment$0(SegmentCache.java:163)
> [org.apache.jackrabbit.oak-segment-tar:1.10.0]
> at
> com.google.common.cache.LocalCache$LocalManualCache$1.load(LocalCache.java:4724)
> [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002]
> at
> com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3522)
> [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002]
> at
> com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2315)
> [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002]
> at
> com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2278)
> [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002]
> at
> com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2193)
> [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002]
> at com.google.common.cache.LocalCache.get(LocalCache.java:3932)
> [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002]
> at
> com.google.common.cache.LocalCache$LocalManualCache.get(LocalCache.java:4721)
> [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002]
> at
> org.apache.jackrabbit.oak.segment.SegmentCache$NonEmptyCache.getSegment(SegmentCache.java:160)
> [org.apache.jackrabbit.oak-segment-tar:1.10.0]
> at
> org.apache.jackrabbit.oak.segment.file.FileStore.readSegment(FileStore.java:498)
> [org.apache.jackrabbit.oak-segment-tar:1.10.0]
> at
> org.apache.jackrabbit.oak.segment.SegmentId.getSegment(SegmentId.java:153)
> [org.apache.jackrabbit.oak-segment-tar:1.10.0]
> at
> org.apache.jackrabbit.oak.segment.RecordId.getSegment(RecordId.java:98)
> [org.apache.jackrabbit.oak-segment-tar:1.10.0]
> at
> org.apache.jackrabbit.oak.segment.SegmentBlob.readLongBlobId(SegmentBlob.java:206)
> [org.apache.jackrabbit.oak-segment-tar:1.10.0]
> at
> org.apache.jackrabbit.oak.segment.SegmentBlob.readBlobId(SegmentBlob.java:163)
> [org.apache.jackrabbit.oak-segment-tar:1.10.0]
> at
> org.apache.jackrabbit.oak.segment.file.AbstractFileStore$3.consume(AbstractFileStore.java:262)
> [org.apache.jackrabbit.oak-segment-tar:1.10.0]
> at
> org.apache.jackrabbit.oak.segment.Segment.forEachRecord(Segment.java:601)
> [org.apache.jackrabbit.oak-segment-tar:1.10.0]
> at
> org.apache.jackrabbit.oak.segment.file.AbstractFileStore.readBinaryReferences(AbstractFileStore.java:257)
> [org.apache.jackrabbit.oak-segment-tar:1.10.0]
> at
> org.apache.jackrabbit.oak.segment.file.FileStore.writeSegment(FileStore.java:533)
> [org.apache.jackrabbit.oak-segment-tar:1.10.0]
> at
> org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.copySegmentFromPrimary(StandbyClientSyncExecution.java:225)
> [org.apache.jackrabbit.oak-segment-tar:1.10.0]
> at
> org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.copySegmentHierarchyFromPrimary(StandbyClientSyncExecution.java:194)
> [org.apache.jackrabbit.oak-segment-tar:1.10.0]
> at
> org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.compareAgainstBaseState(StandbyClientSyncExecution.java:101)
> [org.apache.jackrabbit.oak-segment-tar:1.10.0]
> at
> org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.execute(StandbyClientSyncExecution.java:76)
> [org.apache.jackrabbit.oak-segment-tar:1.10.0]
> at
> org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync.run(StandbyClientSync.java:165)
> [org.apache.jackrabbit.oak-segment-tar:1.10.0]
> at
> org.apache.sling.commons.scheduler.impl.QuartzJobExecutor.execute(QuartzJobExecutor.java:347)
> [org.apache.sling.commons.scheduler:2.7.2]
> at org.quartz.core.JobRunShell.run(JobRunShell.java:202)
> [org.apache.sling.commons.scheduler:2.7.2]
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> at java.base/java.lang.Thread.run(Thread.java:834){noformat}
>
> [0]
> https://github.com/apache/jackrabbit-oak/blob/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java#L205
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
