Author: frm
Date: Wed Jan 23 10:27:08 2019
New Revision: 1851902

URL: http://svn.apache.org/viewvc?rev=1851902&view=rev
Log:
OAK-6749 - Don't trigger a binary download request for in-memory Blob IDs

Added:
    
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobFetchTimeoutException.java
      - copied unchanged from r1851533, 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobFetchTimeoutException.java
    
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobProcessor.java
      - copied unchanged from r1851533, 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobProcessor.java
    
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobTypeUnknownException.java
      - copied unchanged from r1851533, 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobTypeUnknownException.java
    
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobWriteException.java
      - copied unchanged from r1851533, 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobWriteException.java
    
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java
      - copied, changed from r1851535, 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java
    
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/
      - copied from r1851534, 
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/
Modified:
    jackrabbit/oak/branches/1.8/   (props changed)
    
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyDiff.java
    
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java
    
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java

Propchange: jackrabbit/oak/branches/1.8/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jan 23 10:27:08 2019
@@ -1,3 +1,3 @@
 /jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1820660-1820661,1820729,1820734,1820859,1820861,1820878,1820888,1820947,1821027,1821130,1821140-1821141,1821178,1821237,1821240,1821249,1821258,1821325,1821358,1821361-1821362,1821370,1821375,1821393,1821477,1821487,1821516,1821617,1821663,1821665,1821668,1821681,1821847,1821975-1821983,1822121,1822201,1822207,1822527,1822642,1822723,1822808,1822850,1822934,1823135,1823163,1823169,1823172,1823655,1823669,1824196,1824198,1824253,1824255,1824896,1824962,1825065,1825362,1825381,1825442,1825448,1825466,1825470-1825471,1825475,1825523,1825525,1825561,1825619-1825621,1825651,1825654,1825992,1826079,1826090,1826096,1826216,1826237,1826338,1826516,1826532,1826551,1826560,1826638,1826640,1826730,1826833,1826932,1826957,1827423,1827472,1827486,1827816,1827977,1828349,1828439,1828502,1828529,1828948,1829527,1829534,1829546,1829569,1829587,1829665,1829854,1829864,1829978,1829985,1829987,1829998,1830019,1830048,1830160,1830171,1830197,1830209,1830239,1830347,1830748,1830911
 
,1830923,1831157-1831158,1831163,1831190,1831374,1831560,1831689,1832258,1832376,1832379,1832535,1833308,1833347,1833833,1834112,1834117,1834287,1834291,1834302,1834326,1834328,1834336,1834428,1834468,1834483,1834610,1834648-1834649,1834681,1834823,1834857-1834858,1835060,1835518,1835521,1835635,1835642,1835780,1835819,1836082,1836121,1836167-1836168,1836170-1836187,1836189-1836196,1836206,1836487,1836493,1837057,1837274,1837296,1837326,1837475,1837503,1837547,1837569,1837600,1837657,1837718,1837998,1838076,1838637,1839549,1839570,1839637,1839746,1840019,1840024,1840031,1840226,1840455,1840462,1840574,1841314,1841352,1842089,1842677,1843175,1843222,1843231,1843398,1843618,1843652,1843911,1844325,1844549,1844625,1844627,1844642,1844728,1844775,1844932,1845135,1845336,1845405,1845415,1845730-1845731,1845863,1845865,1846057,1846617,1848073,1848181-1848182,1848191,1848217,1848822-1848823,1850837
+/jackrabbit/oak/trunk:1820660-1820661,1820729,1820734,1820859,1820861,1820878,1820888,1820947,1821027,1821130,1821140-1821141,1821178,1821237,1821240,1821249,1821258,1821325,1821358,1821361-1821362,1821370,1821375,1821393,1821477,1821487,1821516,1821617,1821663,1821665,1821668,1821681,1821847,1821975-1821983,1822121,1822201,1822207,1822527,1822642,1822723,1822808,1822850,1822934,1823135,1823163,1823169,1823172,1823655,1823669,1824196,1824198,1824253,1824255,1824896,1824962,1825065,1825362,1825381,1825442,1825448,1825466,1825470-1825471,1825475,1825523,1825525,1825561,1825619-1825621,1825651,1825654,1825992,1826079,1826090,1826096,1826216,1826237,1826338,1826516,1826532,1826551,1826560,1826638,1826640,1826730,1826833,1826932,1826957,1827423,1827472,1827486,1827816,1827977,1828349,1828439,1828502,1828529,1828948,1829527,1829534,1829546,1829569,1829587,1829665,1829854,1829864,1829978,1829985,1829987,1829998,1830019,1830048,1830160,1830171,1830197,1830209,1830239,1830347,1830748,1830911
 
,1830923,1831157-1831158,1831163,1831190,1831374,1831560,1831689,1832258,1832376,1832379,1832535,1833308,1833347,1833833,1834112,1834117,1834287,1834291,1834302,1834326,1834328,1834336,1834428,1834468,1834483,1834610,1834648-1834649,1834681,1834823,1834857-1834858,1835060,1835518,1835521,1835635,1835642,1835780,1835819,1836082,1836121,1836167-1836168,1836170-1836187,1836189-1836196,1836206,1836487,1836493,1837057,1837274,1837296,1837326,1837475,1837503,1837547,1837569,1837600,1837657,1837718,1837998,1838076,1838637,1839549,1839570,1839637,1839746,1840019,1840024,1840031,1840226,1840455,1840462,1840574,1841314,1841352,1842089,1842677,1843175,1843222,1843231,1843398,1843618,1843652,1843911,1844325,1844549,1844625,1844627,1844642,1844728,1844775,1844932,1845135,1845336,1845405,1845415,1845730-1845731,1845863,1845865,1846057,1846617,1848073,1848181-1848182,1848191,1848217,1848822-1848823,1850837,1851533-1851535,1851619
 /jackrabbit/trunk:1345480

Copied: 
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java
 (from r1851535, 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java)
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java?p2=jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java&p1=jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java&r1=1851535&r2=1851902&rev=1851902&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java
 (original)
+++ 
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java
 Wed Jan 23 10:27:08 2019
@@ -19,6 +19,8 @@
 
 package org.apache.jackrabbit.oak.segment.standby.client;
 
+import static org.apache.jackrabbit.oak.commons.IOUtils.closeQuietly;
+
 import java.io.IOException;
 import java.io.InputStream;
 
@@ -59,7 +61,59 @@ class RemoteBlobProcessor implements Blo
     }
 
     private boolean shouldFetchBinary(SegmentBlob blob) {
-        return blob.isExternal() && blob.getReference() == null && 
blob.getBlobId() != null;
+
+        // Shortcut: If the Blob ID is null, this is an inline binary and we
+        // don't have to fetch it.
+
+        String blobId = blob.getBlobId();
+
+        if (blobId == null) {
+            return false;
+        }
+
+        // Shortcut: If the Blob Store is able to retrieve a non-null reference
+        // to the Blob, we can be sure that the Blob is already stored locally.
+        // We don't have to download it.
+
+        String reference;
+
+        try {
+            reference = blob.getReference();
+        } catch (Exception e) {
+            reference = null;
+        }
+
+        if (reference != null) {
+            return false;
+        }
+
+        // Worst case: A null reference to the Blob might just mean that the
+        // Blob Store doesn't support references. The Blob might still be 
stored
+        // locally. We have to retrieve an InputStream for the Blob, and
+        // perform a tentative read in order to overcome a possible lazy
+        // implementation of the returned InputStream.
+
+        InputStream data;
+
+        try {
+            data = blobStore.getInputStream(blobId);
+        } catch (Exception e) {
+            return true;
+        }
+
+        if (data == null) {
+            return true;
+        }
+
+        try {
+            data.read();
+        } catch (Exception e) {
+            return true;
+        } finally {
+            closeQuietly(data);
+        }
+
+        return false;
     }
 
     private void fetchAndStoreBlob(String blobId) throws InterruptedException {

Modified: 
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyDiff.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyDiff.java?rev=1851902&r1=1851901&r2=1851902&view=diff
==============================================================================
--- 
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyDiff.java
 (original)
+++ 
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyDiff.java
 Wed Jan 23 10:27:08 2019
@@ -31,7 +31,6 @@ import org.apache.jackrabbit.oak.api.Blo
 import org.apache.jackrabbit.oak.api.PropertyState;
 import org.apache.jackrabbit.oak.api.Type;
 import org.apache.jackrabbit.oak.segment.CancelableDiff;
-import org.apache.jackrabbit.oak.segment.SegmentBlob;
 import org.apache.jackrabbit.oak.segment.SegmentNodeState;
 import org.apache.jackrabbit.oak.segment.file.FileStore;
 import org.apache.jackrabbit.oak.spi.blob.BlobStore;
@@ -51,24 +50,30 @@ class StandbyDiff implements NodeStateDi
 
     private final StandbyClient client;
 
-    private final boolean hasDataStore;
-
     private final String path;
 
     private final Supplier<Boolean> running;
 
+    private final BlobProcessor blobProcessor;
+
     StandbyDiff(NodeBuilder builder, FileStore store, StandbyClient client, 
Supplier<Boolean> running) {
         this(builder, store, client, "/", running);
     }
 
-    private StandbyDiff(NodeBuilder builder, FileStore store, StandbyClient 
client, String path,
-            Supplier<Boolean> running) {
+    private static BlobProcessor newBinaryFetcher(BlobStore blobStore, 
StandbyClient client) {
+        if (blobStore == null) {
+            return (blob) -> {};
+        }
+        return new RemoteBlobProcessor(blobStore, client::getBlob);
+    }
+
+    private StandbyDiff(NodeBuilder builder, FileStore store, StandbyClient 
client, String path, Supplier<Boolean> running) {
         this.builder = builder;
         this.store = store;
-        this.hasDataStore = store.getBlobStore() != null;
         this.client = client;
         this.path = path;
         this.running = running;
+        this.blobProcessor = newBinaryFetcher(store.getBlobStore(), client);
     }
 
     @Override
@@ -129,7 +134,7 @@ class StandbyDiff implements NodeStateDi
                 return (SegmentNodeState) after;
             }
 
-            if (!hasDataStore) {
+            if (store.getBlobStore() == null) {
                 return (SegmentNodeState) after;
             }
 
@@ -137,7 +142,7 @@ class StandbyDiff implements NodeStateDi
             // traversal to verify binaries
 
             for (PropertyState propertyState : after.getProperties()) {
-                fetchBinary(propertyState);
+                processBinary(propertyState);
             }
 
             boolean success = after.compareAgainstBaseState(before, new 
CancelableDiff(this, newCanceledSupplier()));
@@ -162,59 +167,44 @@ class StandbyDiff implements NodeStateDi
         };
     }
 
-    private PropertyState fetchBinary(PropertyState property) {
+    private PropertyState processBinary(PropertyState property) {
         Type<?> type = property.getType();
 
         if (type == BINARY) {
-            fetchBinary(property.getValue(Type.BINARY), property.getName());
+            processBinary(property.getValue(Type.BINARY), property.getName());
         } else if (type == BINARIES) {
             for (Blob blob : property.getValue(BINARIES)) {
-                fetchBinary(blob, property.getName());
+                processBinary(blob, property.getName());
             }
         }
 
         return property;
     }
 
-    private void fetchBinary(Blob b, String pName) {
-        if (b instanceof SegmentBlob) {
-            fetchBinary((SegmentBlob) b, pName);
-        } else {
-            log.warn("Unknown Blob {} at {}, ignoring", 
b.getClass().getName(), path + "#" + pName);
-        }
-    }
-
-    private void fetchBinary(SegmentBlob sb, String pName) {
-        if (sb.isExternal() && hasDataStore && sb.getReference() == null) {
-            String blobId = sb.getBlobId();
-
-            if (blobId == null) {
-                return;
-            }
-
-            try {
-                fetchAndStoreBlob(blobId, pName);
-            } catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
-            }
-        }
-    }
-
-    private void fetchAndStoreBlob(String blobId, String pName) throws 
InterruptedException {
-        InputStream in = client.getBlob(blobId);
-
-        if (in == null) {
-            throw new IllegalStateException("Unable to load remote blob " + 
blobId + " at " + path + "#" + pName
-                    + " in " + client.getReadTimeoutMs() + "ms. Please 
increase the timeout and try again.");
-        }
-
+    private void processBinary(Blob b, String propertyName) {
         try {
-            BlobStore blobStore = store.getBlobStore();
-            assert blobStore != null : "Blob store must not be null";
-            blobStore.writeBlob(in);
-            in.close();
-        } catch (IOException f) {
-            throw new IllegalStateException("Unable to persist blob " + blobId 
+ " at " + path + "#" + pName, f);
+            blobProcessor.processBinary(b);
+        } catch (BlobFetchTimeoutException e) {
+            String message = String.format(
+                "Unable to load remote blob %s at %s#%s in %dms. Please 
increase the timeout and try again.",
+                e.getBlobId(),
+                path,
+                propertyName,
+                client.getReadTimeoutMs()
+            );
+            throw new IllegalStateException(message, e);
+        } catch (BlobWriteException e) {
+            String message = String.format(
+                "Unable to persist blob %s at %s#%s",
+                e.getBlobId(),
+                path,
+                propertyName
+            );
+            throw new IllegalStateException(message, e);
+        } catch (BlobTypeUnknownException e) {
+            log.warn("Unknown Blob {} at {}, ignoring", 
b.getClass().getName(), path + "#" + propertyName);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
         }
     }
 

Modified: 
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java?rev=1851902&r1=1851534&r2=1851902&view=diff
==============================================================================
--- 
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java
 (original)
+++ 
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java
 Wed Jan 23 10:27:08 2019
@@ -33,7 +33,6 @@ import org.apache.jackrabbit.oak.spi.com
 import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
 import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
 import org.junit.Assert;
-import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.RuleChain;
@@ -85,7 +84,6 @@ public class RemoteBlobProcessorTest {
      * downloaded.
      */
     @Test
-    @Ignore("OAK-6749")
     public void inMemoryBinaryShouldNotBeDownloaded() throws Exception {
         SegmentNodeStore store = 
SegmentNodeStoreBuilders.builder(fileStore.fileStore()).build();
 

Modified: 
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java?rev=1851902&r1=1851901&r2=1851902&view=diff
==============================================================================
--- 
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java
 (original)
+++ 
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java
 Wed Jan 23 10:27:08 2019
@@ -37,11 +37,15 @@ public class TemporaryBlobStore extends
     @Override
     protected void before() throws Throwable {
         FileDataStore fds = new FileDataStore();
-        fds.setMinRecordLength(4092);
+        configureDataStore(fds);
         fds.init(folder.newFolder().getAbsolutePath());
         store = new DataStoreBlobStore(fds);
     }
 
+    protected void configureDataStore(FileDataStore dataStore) {
+        dataStore.setMinRecordLength(4092);
+    }
+
     @Override
     protected void after() {
         try {


Reply via email to