IGNITE-5716 .NET: Fix 2-byte field offset handling
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/82e5f8a6 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/82e5f8a6 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/82e5f8a6 Branch: refs/heads/master Commit: 82e5f8a6553323e793c01c54e24dda6d47188ce6 Parents: eb37f53 Author: Pavel Tupitsyn <ptupit...@apache.org> Authored: Fri Jul 7 19:28:54 2017 +0300 Committer: Pavel Tupitsyn <ptupit...@apache.org> Committed: Fri Jul 7 19:28:54 2017 +0300 ---------------------------------------------------------------------- .../Apache.Ignite.Core.Tests.csproj | 1 + .../Binary/BinaryFooterTest.cs | 146 +++++++++++++++++++ .../Binary/BinarySelfTest.cs | 32 ---- .../Impl/Binary/BinaryObject.cs | 2 +- .../Impl/Binary/BinaryObjectBuilder.cs | 2 +- .../Impl/Binary/BinaryObjectSchemaField.cs | 3 + .../Impl/Binary/BinaryObjectSchemaSerializer.cs | 93 +++++++++--- .../Impl/Binary/BinaryReader.cs | 49 +------ 8 files changed, 228 insertions(+), 100 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/82e5f8a6/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj index 09eac70..d91f0e6 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj @@ -72,6 +72,7 @@ <Compile Include="Binary\BinaryDateTimeTest.cs" /> <Compile Include="Binary\BinaryEqualityComparerTest.cs" /> <Compile Include="Binary\BinaryBuilderSelfTestDynamicRegistration.cs" /> + <Compile Include="Binary\BinaryFooterTest.cs" /> <Compile Include="Binary\BinaryNameMapperTest.cs" /> <Compile Include="Binary\BinaryReaderWriterTest.cs" /> <Compile Include="Binary\BinarySelfTestSimpleName.cs" /> http://git-wip-us.apache.org/repos/asf/ignite/blob/82e5f8a6/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs new file mode 100644 index 0000000..36f2f65 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Tests.Binary +{ + using System; + using System.Linq; + using Apache.Ignite.Core.Binary; + using Apache.Ignite.Core.Impl; + using Apache.Ignite.Core.Impl.Binary; + using NUnit.Framework; + + /// <summary> + /// Tests binary footer integrity (field offsets). + /// Writes objects of various sizes to test schema compaction + /// (where field offsets can be stored as 1, 2 or 4 bytes). + /// </summary> + public class BinaryFooterTest + { + /// <summary> + /// Tears down the test. + /// </summary> + [TearDown] + public void TearDown() + { + Ignition.StopAll(true); + } + + /// <summary> + /// Tests full footers in offline mode. + /// </summary> + [Test] + public void TestFullFooterOffline() + { + // Register type to avoid unregistered type name in binary object. + // Use new marshaller to read and write to avoid schema caching. + + TestOffsets(() => new Marshaller(new BinaryConfiguration(typeof(OffsetTest)) + { + // Compact footers do not work in offline mode. + CompactFooter = false + })); + } + + /// <summary> + /// Tests the full footer online. + /// </summary> + [Test] + public void TestFullFooterOnline() + { + var ignite = Ignition.Start(new IgniteConfiguration(TestUtils.GetTestConfiguration()) + { + BinaryConfiguration = new BinaryConfiguration + { + CompactFooter = false + } + }); + + TestOffsets(() => ((Ignite) ignite).Marshaller); + } + + /// <summary> + /// Tests the full footer online. + /// </summary> + [Test] + public void TestCompactFooterOnline() + { + var ignite = Ignition.Start(TestUtils.GetTestConfiguration()); + + TestOffsets(() => ((Ignite) ignite).Marshaller); + } + + /// <summary> + /// Tests the offsets. + /// </summary> + private static void TestOffsets(Func<Marshaller> getMarsh) + { + // Corner cases are byte/sbyte/short/ushort max values. + foreach (var i in new[] {1, sbyte.MaxValue, byte.MaxValue, short.MaxValue, ushort.MaxValue}) + { + foreach (var j in new[] {-1, 0, 1}) + { + var arrSize = i + j; + + var dt = new OffsetTest + { + Arr = Enumerable.Range(0, arrSize).Select(x => (byte) x).ToArray(), + Int = arrSize + }; + + var bytes = getMarsh().Marshal(dt); + + var res = getMarsh().Unmarshal<OffsetTest>(bytes); + var binRes = getMarsh().Unmarshal<IBinaryObject>(bytes, BinaryMode.ForceBinary); + var binFieldRes = new OffsetTest + { + Arr = binRes.GetField<byte[]>("arr"), + Int = binRes.GetField<int>("int") + }; + + foreach (var r in new[] {res, binRes.Deserialize<OffsetTest>(), binFieldRes}) + { + Assert.AreEqual(dt.Arr, r.Arr); + Assert.AreEqual(dt.Int, r.Int); + } + } + } + } + + /// <summary> + /// Offset test. + /// </summary> + private class OffsetTest : IBinarizable + { + public byte[] Arr; // Array to enforce field offset. + public int Int; // Value at offset. + + public void WriteBinary(IBinaryWriter writer) + { + writer.WriteByteArray("arr", Arr); + writer.WriteInt("int", Int); + } + + public void ReadBinary(IBinaryReader reader) + { + // Read in different order to enforce full schema scan. + Int = reader.ReadInt("int"); + Arr = reader.ReadByteArray("arr"); + } + } + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/82e5f8a6/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs index 2b22d5a..e24dca0 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs @@ -1534,38 +1534,6 @@ namespace Apache.Ignite.Core.Tests.Binary Assert.AreEqual(nDateArr, obj2.NDateArr); } - /// <summary> - /// Writes objects of various sizes to test schema compaction - /// (where field offsets can be stored as 1, 2 or 4 bytes). - /// </summary> - [Test] - public void TestCompactSchema() - { - var marsh = new Marshaller(new BinaryConfiguration - { - TypeConfigurations = new List<BinaryTypeConfiguration> - { - new BinaryTypeConfiguration(typeof (SpecialArray)), - new BinaryTypeConfiguration(typeof (SpecialArrayMarshalAware)) - } - }); - - var dt = new SpecialArrayMarshalAware(); - - foreach (var i in new[] {1, 5, 10, 13, 14, 15, 100, 200, 1000, 5000, 15000, 30000}) - { - dt.NGuidArr = Enumerable.Range(1, i).Select(x => (Guid?) Guid.NewGuid()).ToArray(); - dt.NDateArr = Enumerable.Range(1, i).Select(x => (DateTime?) DateTime.Now.AddDays(x)).ToArray(); - - var bytes = marsh.Marshal(dt); - - var res = marsh.Unmarshal<SpecialArrayMarshalAware>(bytes); - - CollectionAssert.AreEquivalent(dt.NGuidArr, res.NGuidArr); - CollectionAssert.AreEquivalent(dt.NDateArr, res.NDateArr); - } - } - [Test] public void TestBinaryConfigurationValidation() { http://git-wip-us.apache.org/repos/asf/ignite/blob/82e5f8a6/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObject.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObject.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObject.cs index 8c5cee6..370233f 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObject.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObject.cs @@ -239,7 +239,7 @@ namespace Apache.Ignite.Core.Impl.Binary { var hdr = BinaryObjectHeader.Read(stream, _offset); - _fields = BinaryObjectSchemaSerializer.ReadSchema(stream, _offset, hdr, desc.Schema,_marsh) + _fields = BinaryObjectSchemaSerializer.ReadSchema(stream, _offset, hdr, desc.Schema, _marsh.Ignite) .ToDictionary() ?? EmptyFields; } } http://git-wip-us.apache.org/repos/asf/ignite/blob/82e5f8a6/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs index 91fe12a..c310b3a 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs @@ -617,7 +617,7 @@ namespace Apache.Ignite.Core.Impl.Binary { // New object, write in full form. var inSchema = BinaryObjectSchemaSerializer.ReadSchema(inStream, inStartPos, inHeader, - _desc.Schema, _binary.Marshaller); + _desc.Schema, _binary.Marshaller.Ignite); var outSchema = BinaryObjectSchemaHolder.Current; var schemaIdx = outSchema.PushSchema(); http://git-wip-us.apache.org/repos/asf/ignite/blob/82e5f8a6/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaField.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaField.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaField.cs index 3c5339a..be6278a 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaField.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaField.cs @@ -17,6 +17,7 @@ namespace Apache.Ignite.Core.Impl.Binary { + using System.Diagnostics; using System.Runtime.InteropServices; /// <summary> @@ -41,6 +42,8 @@ namespace Apache.Ignite.Core.Impl.Binary /// <param name="offset">The offset.</param> public BinaryObjectSchemaField(int id, int offset) { + Debug.Assert(offset >= 0); + Id = id; Offset = offset; } http://git-wip-us.apache.org/repos/asf/ignite/blob/82e5f8a6/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaSerializer.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaSerializer.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaSerializer.cs index e2f9ea7..1d699c2 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaSerializer.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaSerializer.cs @@ -54,18 +54,17 @@ namespace Apache.Ignite.Core.Impl.Binary /// <param name="position">The position.</param> /// <param name="hdr">The header.</param> /// <param name="schema">The schema.</param> - /// <param name="marsh">The marshaller.</param> + /// <param name="ignite">The ignite.</param> /// <returns> /// Schema. /// </returns> public static BinaryObjectSchemaField[] ReadSchema(IBinaryStream stream, int position, BinaryObjectHeader hdr, - BinaryObjectSchema schema, Marshaller marsh) + BinaryObjectSchema schema, Ignite ignite) { Debug.Assert(stream != null); Debug.Assert(schema != null); - Debug.Assert(marsh != null); - return ReadSchema(stream, position, hdr, () => GetFieldIds(hdr, schema, marsh)); + return ReadSchema(stream, position, hdr, () => GetFieldIds(hdr, schema, ignite)); } /// <summary> @@ -78,8 +77,8 @@ namespace Apache.Ignite.Core.Impl.Binary /// <returns> /// Schema. /// </returns> - public static BinaryObjectSchemaField[] ReadSchema(IBinaryStream stream, int position, BinaryObjectHeader hdr, - Func<int[]> fieldIdsFunc) + public static unsafe BinaryObjectSchemaField[] ReadSchema(IBinaryStream stream, int position, + BinaryObjectHeader hdr, Func<int[]> fieldIdsFunc) { Debug.Assert(stream != null); Debug.Assert(fieldIdsFunc != null); @@ -110,7 +109,7 @@ namespace Apache.Ignite.Core.Impl.Binary else if (offsetSize == 2) { for (var i = 0; i < schemaSize; i++) - res[i] = new BinaryObjectSchemaField(fieldIds[i], stream.ReadShort()); + res[i] = new BinaryObjectSchemaField(fieldIds[i], (ushort) stream.ReadShort()); } else { @@ -128,12 +127,22 @@ namespace Apache.Ignite.Core.Impl.Binary else if (offsetSize == 2) { for (var i = 0; i < schemaSize; i++) - res[i] = new BinaryObjectSchemaField(stream.ReadInt(), stream.ReadShort()); + res[i] = new BinaryObjectSchemaField(stream.ReadInt(), (ushort) stream.ReadShort()); } else { - for (var i = 0; i < schemaSize; i++) - res[i] = new BinaryObjectSchemaField(stream.ReadInt(), stream.ReadInt()); + if (BitConverter.IsLittleEndian) + { + fixed (BinaryObjectSchemaField* ptr = &res[0]) + { + stream.Read((byte*) ptr, schemaSize * BinaryObjectSchemaField.Size); + } + } + else + { + for (var i = 0; i < schemaSize; i++) + res[i] = new BinaryObjectSchemaField(stream.ReadInt(), stream.ReadInt()); + } } } @@ -220,7 +229,7 @@ namespace Apache.Ignite.Core.Impl.Binary { fixed (BinaryObjectSchemaField* ptr = &fields[offset]) { - stream.Write((byte*)ptr, count / BinaryObjectSchemaField.Size); + stream.Write((byte*)ptr, count * BinaryObjectSchemaField.Size); } } else @@ -243,22 +252,66 @@ namespace Apache.Ignite.Core.Impl.Binary /// <summary> /// Gets the field ids. /// </summary> - private static int[] GetFieldIds(BinaryObjectHeader hdr, BinaryObjectSchema schema, Marshaller marsh) + private static int[] GetFieldIds(BinaryObjectHeader hdr, Ignite ignite) { - var fieldIds = schema.Get(hdr.SchemaId); + Debug.Assert(hdr.TypeId != BinaryUtils.TypeUnregistered); + + int[] fieldIds = null; + + if (ignite != null) + { + fieldIds = ignite.BinaryProcessor.GetSchema(hdr.TypeId, hdr.SchemaId); + } if (fieldIds == null) { - Debug.Assert(hdr.TypeId != BinaryUtils.TypeUnregistered); + throw new BinaryObjectException("Cannot find schema for object with compact footer [" + + "typeId=" + hdr.TypeId + ", schemaId=" + hdr.SchemaId + ']'); + } + return fieldIds; + } - if (marsh.Ignite != null) - fieldIds = marsh.Ignite.BinaryProcessor.GetSchema(hdr.TypeId, hdr.SchemaId); + /// <summary> + /// Reads the schema, maintains stream position. + /// </summary> + public static int[] GetFieldIds(BinaryObjectHeader hdr, Ignite ignite, IBinaryStream stream, int objectPos) + { + Debug.Assert(stream != null); - if (fieldIds == null) - throw new BinaryObjectException("Cannot find schema for object with compact footer [" + - "typeId=" + hdr.TypeId + ", schemaId=" + hdr.SchemaId + ']'); + if (hdr.IsCompactFooter) + { + // Get schema from Java + return GetFieldIds(hdr, ignite); } - return fieldIds; + + var pos = stream.Position; + + stream.Seek(objectPos + hdr.SchemaOffset, SeekOrigin.Begin); + + var count = hdr.SchemaFieldCount; + + var offsetSize = hdr.SchemaFieldOffsetSize; + + var res = new int[count]; + + for (var i = 0; i < count; i++) + { + res[i] = stream.ReadInt(); + stream.Seek(offsetSize, SeekOrigin.Current); // Skip offsets. + } + + stream.Seek(pos, SeekOrigin.Begin); + + return res; + } + + + /// <summary> + /// Gets the field ids. + /// </summary> + private static int[] GetFieldIds(BinaryObjectHeader hdr, BinaryObjectSchema schema, Ignite ignite) + { + return schema.Get(hdr.SchemaId) ?? GetFieldIds(hdr, ignite); } } } http://git-wip-us.apache.org/repos/asf/ignite/blob/82e5f8a6/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReader.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReader.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReader.cs index e792dce..76237c4 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReader.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReader.cs @@ -20,7 +20,6 @@ namespace Apache.Ignite.Core.Impl.Binary using System; using System.Collections; using System.Collections.Generic; - using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; using Apache.Ignite.Core.Binary; @@ -783,7 +782,8 @@ namespace Apache.Ignite.Core.Impl.Binary if (_frame.Schema == null) { - _frame.Schema = ReadSchema(desc.TypeId); + _frame.Schema = + BinaryObjectSchemaSerializer.GetFieldIds(_frame.Hdr, Marshaller.Ignite, Stream, _frame.Pos); desc.Schema.Add(_frame.Hdr.SchemaId, _frame.Schema); } @@ -795,49 +795,6 @@ namespace Apache.Ignite.Core.Impl.Binary } /// <summary> - /// Reads the schema. - /// </summary> - private int[] ReadSchema(int typeId) - { - if (_frame.Hdr.IsCompactFooter) - { - // Get schema from Java - var ignite = Marshaller.Ignite; - - Debug.Assert(typeId != BinaryUtils.TypeUnregistered); - - var schema = ignite == null - ? null - : ignite.BinaryProcessor.GetSchema(_frame.Hdr.TypeId, _frame.Hdr.SchemaId); - - if (schema == null) - throw new BinaryObjectException("Cannot find schema for object with compact footer [" + - "typeId=" + typeId + ", schemaId=" + _frame.Hdr.SchemaId + ']'); - - return schema; - } - - var pos = Stream.Position; - - Stream.Seek(_frame.Pos + _frame.Hdr.SchemaOffset, SeekOrigin.Begin); - - var count = _frame.Hdr.SchemaFieldCount; - - var offsetSize = _frame.Hdr.SchemaFieldOffsetSize; - - var res = new int[count]; - - for (int i = 0; i < count; i++) - { - res[i] = Stream.ReadInt(); - Stream.Seek(offsetSize, SeekOrigin.Current); - } - - Stream.Seek(pos, SeekOrigin.Begin); - - return res; - } - /// <summary> /// Reads the handle object. /// </summary> private T ReadHandleObject<T>(int pos, Type typeOverride) @@ -942,7 +899,7 @@ namespace Apache.Ignite.Core.Impl.Binary int pos; - if (!_frame.SchemaMap.TryGetValue(fieldId, out pos)) + if (_frame.SchemaMap == null || !_frame.SchemaMap.TryGetValue(fieldId, out pos)) return false; Stream.Seek(pos + _frame.Pos, SeekOrigin.Begin);
