This is an automated email from the ASF dual-hosted git repository. alexpl pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push: new a77ebdd IGNITE-12827 Fix skipped detach flag when unmarshalling collections and object arrays - Fixes #7679. a77ebdd is described below commit a77ebdd101a4d191ea60becc2af12e88e214af71 Author: Ivan Daschinskiy <ivanda...@gmail.com> AuthorDate: Fri May 8 17:57:45 2020 +0300 IGNITE-12827 Fix skipped detach flag when unmarshalling collections and object arrays - Fixes #7679. Signed-off-by: Aleksey Plekhanov <plehanov.a...@gmail.com> --- .../ignite/internal/binary/BinaryReaderExImpl.java | 12 +-- .../apache/ignite/internal/binary/BinaryUtils.java | 27 +++--- .../internal/binary/BinaryMarshallerSelfTest.java | 96 ++++++++++++++++++++++ 3 files changed, 116 insertions(+), 19 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java index f81ee0b..dc84328 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java @@ -1359,7 +1359,7 @@ public class BinaryReaderExImpl implements BinaryReader, BinaryRawReaderEx, Bina @Nullable @Override public Object[] readObjectArray() throws BinaryObjectException { switch (checkFlag(OBJ_ARR)) { case NORMAL: - return BinaryUtils.doReadObjectArray(in, ctx, ldr, this, true); + return BinaryUtils.doReadObjectArray(in, ctx, ldr, this, false, true); case HANDLE: return readHandleField(); @@ -1532,7 +1532,7 @@ public class BinaryReaderExImpl implements BinaryReader, BinaryRawReaderEx, Bina throws BinaryObjectException { switch (checkFlag(COL)) { case NORMAL: - return (Collection)BinaryUtils.doReadCollection(in, ctx, ldr, this, true, factory); + return (Collection)BinaryUtils.doReadCollection(in, ctx, ldr, this, false, true, factory); case HANDLE: { int handlePos = BinaryUtils.positionForHandle(in) - in.readInt(); @@ -1610,7 +1610,7 @@ public class BinaryReaderExImpl implements BinaryReader, BinaryRawReaderEx, Bina private Map readMap0(@Nullable BinaryMapFactory factory) throws BinaryObjectException { switch (checkFlag(MAP)) { case NORMAL: - return (Map)BinaryUtils.doReadMap(in, ctx, ldr, this, true, factory); + return (Map)BinaryUtils.doReadMap(in, ctx, ldr, this, false, true, factory); case HANDLE: { int handlePos = BinaryUtils.positionForHandle(in) - in.readInt(); @@ -1908,17 +1908,17 @@ public class BinaryReaderExImpl implements BinaryReader, BinaryRawReaderEx, Bina break; case OBJ_ARR: - obj = BinaryUtils.doReadObjectArray(in, ctx, ldr, this, true); + obj = BinaryUtils.doReadObjectArray(in, ctx, ldr, this, false, true); break; case COL: - obj = BinaryUtils.doReadCollection(in, ctx, ldr, this, true, null); + obj = BinaryUtils.doReadCollection(in, ctx, ldr, this, false, true, null); break; case MAP: - obj = BinaryUtils.doReadMap(in, ctx, ldr, this, true, null); + obj = BinaryUtils.doReadMap(in, ctx, ldr, this, false, true, null); break; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java index 2e33bf5..8325615 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java @@ -1840,7 +1840,7 @@ public class BinaryUtils { in.position(handlePos); - obj = unmarshal(in, ctx, ldr, handles); + obj = unmarshal(in, ctx, ldr, handles, detach); in.position(retPos); } @@ -1961,13 +1961,13 @@ public class BinaryUtils { return doReadTimeArray(in); case GridBinaryMarshaller.OBJ_ARR: - return doReadObjectArray(in, ctx, ldr, handles, false); + return doReadObjectArray(in, ctx, ldr, handles, detach, false); case GridBinaryMarshaller.COL: - return doReadCollection(in, ctx, ldr, handles, false, null); + return doReadCollection(in, ctx, ldr, handles, detach, false, null); case GridBinaryMarshaller.MAP: - return doReadMap(in, ctx, ldr, handles, false, null); + return doReadMap(in, ctx, ldr, handles, detach, false, null); case GridBinaryMarshaller.BINARY_OBJ: return doReadBinaryObject(in, ctx, detach); @@ -2000,12 +2000,13 @@ public class BinaryUtils { * @param ctx Binary context. * @param ldr Class loader. * @param handles Holder for handles. + * @param detach Detach flag. * @param deserialize Deep flag. * @return Value. * @throws BinaryObjectException In case of error. */ public static Object[] doReadObjectArray(BinaryInputStream in, BinaryContext ctx, ClassLoader ldr, - BinaryReaderHandlesHolder handles, boolean deserialize) throws BinaryObjectException { + BinaryReaderHandlesHolder handles, boolean detach, boolean deserialize) throws BinaryObjectException { int hPos = positionForHandle(in); Class compType = doReadClass(in, ctx, ldr, deserialize); @@ -2017,7 +2018,7 @@ public class BinaryUtils { handles.setHandle(arr, hPos); for (int i = 0; i < len; i++) - arr[i] = deserializeOrUnmarshal(in, ctx, ldr, handles, deserialize); + arr[i] = deserializeOrUnmarshal(in, ctx, ldr, handles, detach, deserialize); return arr; } @@ -2030,7 +2031,7 @@ public class BinaryUtils { */ @SuppressWarnings("unchecked") public static Collection<?> doReadCollection(BinaryInputStream in, BinaryContext ctx, ClassLoader ldr, - BinaryReaderHandlesHolder handles, boolean deserialize, BinaryCollectionFactory factory) + BinaryReaderHandlesHolder handles, boolean detach, boolean deserialize, BinaryCollectionFactory factory) throws BinaryObjectException { int hPos = positionForHandle(in); @@ -2089,7 +2090,7 @@ public class BinaryUtils { handles.setHandle(col, hPos); for (int i = 0; i < size; i++) - col.add(deserializeOrUnmarshal(in, ctx, ldr, handles, deserialize)); + col.add(deserializeOrUnmarshal(in, ctx, ldr, handles, detach, deserialize)); return colType == GridBinaryMarshaller.SINGLETON_LIST ? U.convertToSingletonList(col) : col; } @@ -2102,7 +2103,7 @@ public class BinaryUtils { */ @SuppressWarnings("unchecked") public static Map<?, ?> doReadMap(BinaryInputStream in, BinaryContext ctx, ClassLoader ldr, - BinaryReaderHandlesHolder handles, boolean deserialize, BinaryMapFactory factory) + BinaryReaderHandlesHolder handles, boolean detach, boolean deserialize, BinaryMapFactory factory) throws BinaryObjectException { int hPos = positionForHandle(in); @@ -2141,8 +2142,8 @@ public class BinaryUtils { handles.setHandle(map, hPos); for (int i = 0; i < size; i++) { - Object key = deserializeOrUnmarshal(in, ctx, ldr, handles, deserialize); - Object val = deserializeOrUnmarshal(in, ctx, ldr, handles, deserialize); + Object key = deserializeOrUnmarshal(in, ctx, ldr, handles, detach, deserialize); + Object val = deserializeOrUnmarshal(in, ctx, ldr, handles, detach, deserialize); map.put(key, val); } @@ -2157,8 +2158,8 @@ public class BinaryUtils { * @return Result. */ private static Object deserializeOrUnmarshal(BinaryInputStream in, BinaryContext ctx, ClassLoader ldr, - BinaryReaderHandlesHolder handles, boolean deserialize) { - return deserialize ? doReadObject(in, ctx, ldr, handles) : unmarshal(in, ctx, ldr, handles); + BinaryReaderHandlesHolder handles, boolean detach, boolean deserialize) { + return deserialize ? doReadObject(in, ctx, ldr, handles) : unmarshal(in, ctx, ldr, handles, detach); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java index 80736ea..d5464d8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java @@ -53,6 +53,8 @@ import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentSkipListSet; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteSystemProperties; @@ -77,13 +79,17 @@ import org.apache.ignite.binary.Binarylizable; import org.apache.ignite.configuration.BinaryConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.binary.builder.BinaryObjectBuilderImpl; +import org.apache.ignite.internal.binary.streams.BinaryHeapInputStream; +import org.apache.ignite.internal.binary.streams.BinaryHeapOutputStream; import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager; import org.apache.ignite.internal.managers.systemview.GridSystemViewManager; import org.apache.ignite.internal.processors.cache.CacheObjectContext; import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.internal.util.lang.GridMapEntry; +import org.apache.ignite.internal.util.lang.IgniteThrowableConsumer; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteUuid; @@ -2947,6 +2953,96 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { * @throws Exception If failed. */ @Test + public void testReadDetachedMap() throws Exception { + Map<Key, Value> map = IntStream.range(0, 1000).mapToObj(i -> new T2<>(new Key(i), new Value(i))) + .collect(Collectors.toMap(T2::getKey, T2::getValue)); + + testReadDetachObjectProperly(map, obj -> { + Map<BinaryObject, BinaryObject> desMap = (Map<BinaryObject, BinaryObject>)obj; + + assertEquals(map.size(), desMap.size()); + + desMap.forEach((k, v) -> { + Key key = new Key(k.field("key")); + Value val = new Value(v.field("val")); + + assertTrue(map.containsKey(key)); + assertEquals(val, map.get(key)); + }); + }); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testReadDetachedCollection() throws Exception { + Collection<Value> col = IntStream.range(0, 1000).mapToObj(Value::new).collect(Collectors.toSet()); + + testReadDetachObjectProperly(col, obj -> { + Collection<BinaryObject> desCol = (Collection<BinaryObject>)obj; + + assertEquals(col.size(), desCol.size()); + + desCol.forEach(v -> { + Value val = new Value(v.field("val")); + + assertTrue(col.contains(val)); + }); + }); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testReadDetachedArray() throws Exception { + Value[] arr = IntStream.range(0, 1000).mapToObj(Value::new).toArray(Value[]::new); + + testReadDetachObjectProperly(arr, obj -> { + Object[] desArr = (Object[])obj; + + assertEquals(arr.length, desArr.length); + + for (int i = 0; i < arr.length; i++) { + BinaryObject val = (BinaryObject)desArr[i]; + + assertEquals(arr[i], new Value(val.field("val"))); + } + }); + } + + /** + * Perform action on binary object after unmarshalling from offheap data, when offheap memory chunk cleared. + * + * @param obj Object to marshal-unmarshal. + * @param action Action to perform on object. + * @throws Exception If failed. + */ + private void testReadDetachObjectProperly(Object obj, IgniteThrowableConsumer<Object> action) throws Exception { + BinaryMarshaller marsh = binaryMarshaller(); + + BinaryHeapOutputStream os = new BinaryHeapOutputStream(1024); + + BinaryWriterExImpl writer = marsh.binaryMarshaller().writer(os); + + writer.writeObject(obj); + + BinaryHeapInputStream is = new BinaryHeapInputStream(os.array()); + + BinaryReaderExImpl reader = marsh.binaryMarshaller().reader(is); + + Object bObj = reader.readObjectDetached(); + + Arrays.fill(os.array(), (byte)0); + + action.accept(bObj); + } + + /** + * @throws Exception If failed. + */ + @Test public void testMarshallingThroughJdk() throws Exception { BinaryMarshaller marsh = binaryMarshaller();