[ https://issues.apache.org/jira/browse/OAK-4969?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Timothee Maret updated OAK-4969: -------------------------------- Description: 1. In a setup composed of two instances (primary, standby) configured with a custom blob store (File blob store). 2. On the primary instance, set/update a BINARY property of an existing resource with > 2MB binary. 3. Observe that the standby instance does not fetch the binary and instead, enters a loop detecting the missing binary upon comparing node states. For example, the following stack trace would be printed every 5 seconds on the standby (the polling time is 5sec). {code} 19.10.2016 16:22:38.035 *DEBUG* [nioEventLoopGroup-1005-1] org.apache.jackrabbit.oak.segment.standby.codec.ResponseDecoder Decoding 'get head' response 19.10.2016 16:22:38.038 *DEBUG* [sling-default-81-Registered Service.607] org.apache.jackrabbit.oak.segment.standby.client.StandbyClient Channel closed 19.10.2016 16:22:40.241 *DEBUG* [sling-default-81-Registered Service.607] org.apache.jackrabbit.oak.segment.standby.client.StandbyClient Group shut down 19.10.2016 16:22:40.242 *ERROR* [sling-default-81-Registered Service.607] org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync Failed synchronizing state. java.lang.RuntimeException: Error occurred while obtaining InputStream for blobId [4dfc748c91d518c9221031ec6115fd7ac04fe27b#100000] at org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob.getNewStream(BlobStoreBlob.java:49) at org.apache.jackrabbit.oak.segment.SegmentBlob.getNewStream(SegmentBlob.java:252) at org.apache.jackrabbit.oak.segment.SegmentBlob.getNewStream(SegmentBlob.java:87) at org.apache.jackrabbit.oak.plugins.memory.AbstractBlob$1.getInput(AbstractBlob.java:45) at org.apache.jackrabbit.oak.plugins.memory.AbstractBlob$1.getInput(AbstractBlob.java:42) at com.google.common.io.ByteStreams$3.openStream(ByteStreams.java:907) at com.google.common.io.ByteSource.contentEquals(ByteSource.java:301) at com.google.common.io.ByteStreams.equal(ByteStreams.java:661) at org.apache.jackrabbit.oak.plugins.memory.AbstractBlob.equal(AbstractBlob.java:68) at org.apache.jackrabbit.oak.segment.SegmentBlob.equals(SegmentBlob.java:193) at com.google.common.base.Objects.equal(Objects.java:55) at org.apache.jackrabbit.oak.plugins.memory.AbstractPropertyState.equal(AbstractPropertyState.java:53) at org.apache.jackrabbit.oak.segment.SegmentPropertyState.equals(SegmentPropertyState.java:249) at org.apache.jackrabbit.oak.segment.SegmentNodeState.compareProperties(SegmentNodeState.java:622) at org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:516) at org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216) at org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186) at org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:415) at org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609) at org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216) at org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186) at org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:457) at org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609) at org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216) at org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186) at org.apache.jackrabbit.oak.segment.MapRecord$2.childNodeChanged(MapRecord.java:401) at org.apache.jackrabbit.oak.segment.MapRecord$3.childNodeChanged(MapRecord.java:442) at org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:485) at org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:433) at org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:391) at org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609) at org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216) at org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186) at org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:485) at org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609) at org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.compareAgainstBaseState(StandbyClientSyncExecution.java:91) at org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.execute(StandbyClientSyncExecution.java:74) at org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync.run(StandbyClientSync.java:143) at org.apache.sling.commons.scheduler.impl.QuartzJobExecutor.execute(QuartzJobExecutor.java:118) at org.quartz.core.JobRunShell.run(JobRunShell.java:202) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.io.IOException: org.apache.jackrabbit.core.data.DataStoreException: Record 4dfc748c91d518c9221031ec6115fd7ac04fe27b does not exist at org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getStream(DataStoreBlobStore.java:560) at org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getInputStream(DataStoreBlobStore.java:330) at org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob.getNewStream(BlobStoreBlob.java:47) ... 42 common frames omitted Caused by: org.apache.jackrabbit.core.data.DataStoreException: Record 4dfc748c91d518c9221031ec6115fd7ac04fe27b does not exist at org.apache.jackrabbit.core.data.AbstractDataStore.getRecord(AbstractDataStore.java:53) at org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getDataRecord(DataStoreBlobStore.java:569) at org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getStream(DataStoreBlobStore.java:554) ... 44 common frames omitted {code} was: 1. In a setup composed of two instances (primary, standdy) configured with a custom blob store (File blob store). 2. On the primary instance, set/update a BINARY property of an existing resource with > 2MB binary. 3. Observe that the standby instance does not fetch the binary and instead, enters a loop detecting the missing binary upon comparing node states. For example, the following stack trace would be printed every 5 seconds on the standby (the polling time is 5sec). {code} 19.10.2016 16:22:38.035 *DEBUG* [nioEventLoopGroup-1005-1] org.apache.jackrabbit.oak.segment.standby.codec.ResponseDecoder Decoding 'get head' response 19.10.2016 16:22:38.038 *DEBUG* [sling-default-81-Registered Service.607] org.apache.jackrabbit.oak.segment.standby.client.StandbyClient Channel closed 19.10.2016 16:22:40.241 *DEBUG* [sling-default-81-Registered Service.607] org.apache.jackrabbit.oak.segment.standby.client.StandbyClient Group shut down 19.10.2016 16:22:40.242 *ERROR* [sling-default-81-Registered Service.607] org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync Failed synchronizing state. java.lang.RuntimeException: Error occurred while obtaining InputStream for blobId [4dfc748c91d518c9221031ec6115fd7ac04fe27b#100000] at org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob.getNewStream(BlobStoreBlob.java:49) at org.apache.jackrabbit.oak.segment.SegmentBlob.getNewStream(SegmentBlob.java:252) at org.apache.jackrabbit.oak.segment.SegmentBlob.getNewStream(SegmentBlob.java:87) at org.apache.jackrabbit.oak.plugins.memory.AbstractBlob$1.getInput(AbstractBlob.java:45) at org.apache.jackrabbit.oak.plugins.memory.AbstractBlob$1.getInput(AbstractBlob.java:42) at com.google.common.io.ByteStreams$3.openStream(ByteStreams.java:907) at com.google.common.io.ByteSource.contentEquals(ByteSource.java:301) at com.google.common.io.ByteStreams.equal(ByteStreams.java:661) at org.apache.jackrabbit.oak.plugins.memory.AbstractBlob.equal(AbstractBlob.java:68) at org.apache.jackrabbit.oak.segment.SegmentBlob.equals(SegmentBlob.java:193) at com.google.common.base.Objects.equal(Objects.java:55) at org.apache.jackrabbit.oak.plugins.memory.AbstractPropertyState.equal(AbstractPropertyState.java:53) at org.apache.jackrabbit.oak.segment.SegmentPropertyState.equals(SegmentPropertyState.java:249) at org.apache.jackrabbit.oak.segment.SegmentNodeState.compareProperties(SegmentNodeState.java:622) at org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:516) at org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216) at org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186) at org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:415) at org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609) at org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216) at org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186) at org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:457) at org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609) at org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216) at org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186) at org.apache.jackrabbit.oak.segment.MapRecord$2.childNodeChanged(MapRecord.java:401) at org.apache.jackrabbit.oak.segment.MapRecord$3.childNodeChanged(MapRecord.java:442) at org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:485) at org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:433) at org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:391) at org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609) at org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216) at org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186) at org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:485) at org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609) at org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.compareAgainstBaseState(StandbyClientSyncExecution.java:91) at org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.execute(StandbyClientSyncExecution.java:74) at org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync.run(StandbyClientSync.java:143) at org.apache.sling.commons.scheduler.impl.QuartzJobExecutor.execute(QuartzJobExecutor.java:118) at org.quartz.core.JobRunShell.run(JobRunShell.java:202) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.io.IOException: org.apache.jackrabbit.core.data.DataStoreException: Record 4dfc748c91d518c9221031ec6115fd7ac04fe27b does not exist at org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getStream(DataStoreBlobStore.java:560) at org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getInputStream(DataStoreBlobStore.java:330) at org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob.getNewStream(BlobStoreBlob.java:47) ... 42 common frames omitted Caused by: org.apache.jackrabbit.core.data.DataStoreException: Record 4dfc748c91d518c9221031ec6115fd7ac04fe27b does not exist at org.apache.jackrabbit.core.data.AbstractDataStore.getRecord(AbstractDataStore.java:53) at org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getDataRecord(DataStoreBlobStore.java:569) at org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getStream(DataStoreBlobStore.java:554) ... 44 common frames omitted {code} > ColdStandby does not fetch missing blobs > ---------------------------------------- > > Key: OAK-4969 > URL: https://issues.apache.org/jira/browse/OAK-4969 > Project: Jackrabbit Oak > Issue Type: Bug > Components: segment-tar > Affects Versions: Segment Tar 0.0.10 > Reporter: Timothee Maret > Fix For: Segment Tar 0.0.16 > > > 1. In a setup composed of two instances (primary, standby) configured with a > custom blob store (File blob store). > 2. On the primary instance, set/update a BINARY property of an existing > resource with > 2MB binary. > 3. Observe that the standby instance does not fetch the binary and instead, > enters a loop detecting the missing binary upon comparing node states. > For example, the following stack trace would be printed every 5 seconds on > the standby (the polling time is 5sec). > {code} > 19.10.2016 16:22:38.035 *DEBUG* [nioEventLoopGroup-1005-1] > org.apache.jackrabbit.oak.segment.standby.codec.ResponseDecoder Decoding 'get > head' response > 19.10.2016 16:22:38.038 *DEBUG* [sling-default-81-Registered Service.607] > org.apache.jackrabbit.oak.segment.standby.client.StandbyClient Channel closed > 19.10.2016 16:22:40.241 *DEBUG* [sling-default-81-Registered Service.607] > org.apache.jackrabbit.oak.segment.standby.client.StandbyClient Group shut down > 19.10.2016 16:22:40.242 *ERROR* [sling-default-81-Registered Service.607] > org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync Failed > synchronizing state. > java.lang.RuntimeException: Error occurred while obtaining InputStream for > blobId [4dfc748c91d518c9221031ec6115fd7ac04fe27b#100000] > at > org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob.getNewStream(BlobStoreBlob.java:49) > at > org.apache.jackrabbit.oak.segment.SegmentBlob.getNewStream(SegmentBlob.java:252) > at > org.apache.jackrabbit.oak.segment.SegmentBlob.getNewStream(SegmentBlob.java:87) > at > org.apache.jackrabbit.oak.plugins.memory.AbstractBlob$1.getInput(AbstractBlob.java:45) > at > org.apache.jackrabbit.oak.plugins.memory.AbstractBlob$1.getInput(AbstractBlob.java:42) > at com.google.common.io.ByteStreams$3.openStream(ByteStreams.java:907) > at com.google.common.io.ByteSource.contentEquals(ByteSource.java:301) > at com.google.common.io.ByteStreams.equal(ByteStreams.java:661) > at > org.apache.jackrabbit.oak.plugins.memory.AbstractBlob.equal(AbstractBlob.java:68) > at > org.apache.jackrabbit.oak.segment.SegmentBlob.equals(SegmentBlob.java:193) > at com.google.common.base.Objects.equal(Objects.java:55) > at > org.apache.jackrabbit.oak.plugins.memory.AbstractPropertyState.equal(AbstractPropertyState.java:53) > at > org.apache.jackrabbit.oak.segment.SegmentPropertyState.equals(SegmentPropertyState.java:249) > at > org.apache.jackrabbit.oak.segment.SegmentNodeState.compareProperties(SegmentNodeState.java:622) > at > org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:516) > at > org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216) > at > org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186) > at > org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:415) > at > org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609) > at > org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216) > at > org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186) > at > org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:457) > at > org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609) > at > org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216) > at > org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186) > at > org.apache.jackrabbit.oak.segment.MapRecord$2.childNodeChanged(MapRecord.java:401) > at > org.apache.jackrabbit.oak.segment.MapRecord$3.childNodeChanged(MapRecord.java:442) > at > org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:485) > at > org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:433) > at > org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:391) > at > org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609) > at > org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216) > at > org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186) > at > org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:485) > at > org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609) > at > org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.compareAgainstBaseState(StandbyClientSyncExecution.java:91) > at > org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.execute(StandbyClientSyncExecution.java:74) > at > org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync.run(StandbyClientSync.java:143) > at > org.apache.sling.commons.scheduler.impl.QuartzJobExecutor.execute(QuartzJobExecutor.java:118) > at org.quartz.core.JobRunShell.run(JobRunShell.java:202) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > Caused by: java.io.IOException: > org.apache.jackrabbit.core.data.DataStoreException: Record > 4dfc748c91d518c9221031ec6115fd7ac04fe27b does not exist > at > org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getStream(DataStoreBlobStore.java:560) > at > org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getInputStream(DataStoreBlobStore.java:330) > at > org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob.getNewStream(BlobStoreBlob.java:47) > ... 42 common frames omitted > Caused by: org.apache.jackrabbit.core.data.DataStoreException: Record > 4dfc748c91d518c9221031ec6115fd7ac04fe27b does not exist > at > org.apache.jackrabbit.core.data.AbstractDataStore.getRecord(AbstractDataStore.java:53) > at > org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getDataRecord(DataStoreBlobStore.java:569) > at > org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getStream(DataStoreBlobStore.java:554) > ... 44 common frames omitted > {code} -- This message was sent by Atlassian JIRA (v6.3.4#6332)