Repository: ignite Updated Branches: refs/heads/ignite-1282 49c495b9c -> 4e053e45d
IGNITE-1420: Optimized fields calculation and metadata handling. Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4e053e45 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4e053e45 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4e053e45 Branch: refs/heads/ignite-1282 Commit: 4e053e45dceab7b700ef0b83498219018c0d33e9 Parents: 49c495b Author: vozerov-gridgain <voze...@gridgain.com> Authored: Wed Oct 7 15:00:46 2015 +0300 Committer: vozerov-gridgain <voze...@gridgain.com> Committed: Wed Oct 7 15:00:46 2015 +0300 ---------------------------------------------------------------------- .../Portable/PortableWriteBenchmark.cs | 2 +- .../Apache.Ignite.Core.Tests.csproj | 1 + .../Portable/PortableStructureTest.cs | 261 +++++++++++++++ .../Apache.Ignite.Core.csproj | 4 + .../Impl/Portable/IPortableTypeDescriptor.cs | 16 + .../Impl/Portable/PortableFullTypeDescriptor.cs | 22 ++ .../Impl/Portable/PortableMarshaller.cs | 1 + .../Portable/PortableSurrogateTypeDescriptor.cs | 24 +- .../Impl/Portable/PortableUtils.cs | 4 + .../Impl/Portable/PortableWriterImpl.cs | 88 ++++- .../Portable/Structure/PortableStructure.cs | 333 +++++++++++++++++++ .../Structure/PortableStructureEntry.cs | 129 +++++++ .../Structure/PortableStructureJumpTable.cs | 118 +++++++ .../Structure/PortableStructureUpdate.cs | 84 +++++ 14 files changed, 1071 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/4e053e45/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Portable/PortableWriteBenchmark.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Portable/PortableWriteBenchmark.cs b/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Portable/PortableWriteBenchmark.cs index 9fcfa46..5638195 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Portable/PortableWriteBenchmark.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Portable/PortableWriteBenchmark.cs @@ -46,7 +46,7 @@ namespace Apache.Ignite.Benchmarks.Portable { TypeConfigurations = new List<PortableTypeConfiguration> { - new PortableTypeConfiguration(typeof (Address)) {MetadataEnabled = false} + new PortableTypeConfiguration(typeof (Address)) {MetadataEnabled = true} } }); } http://git-wip-us.apache.org/repos/asf/ignite/blob/4e053e45/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj index 90f3481..7cbe784 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj @@ -111,6 +111,7 @@ <Compile Include="MarshallerTest.cs" /> <Compile Include="MessagingTest.cs" /> <Compile Include="PortableConfigurationTest.cs" /> + <Compile Include="Portable\PortableStructureTest.cs" /> <Compile Include="SerializationTest.cs" /> <Compile Include="IgniteStartStopTest.cs" /> <Compile Include="TestUtils.cs" /> http://git-wip-us.apache.org/repos/asf/ignite/blob/4e053e45/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Portable/PortableStructureTest.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Portable/PortableStructureTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Portable/PortableStructureTest.cs new file mode 100644 index 0000000..1212ea0 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Portable/PortableStructureTest.cs @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Tests.Portable +{ + using System; + using System.Collections.Generic; + using Apache.Ignite.Core.Impl; + using Apache.Ignite.Core.Impl.Portable; + using Apache.Ignite.Core.Impl.Portable.Structure; + using Apache.Ignite.Core.Portable; + using NUnit.Framework; + + /// <summary> + /// Contains tests for portable type structure. + /// </summary> + [TestFixture] + public class PortableStructureTest + { + /** Repeat count. */ + public static readonly int RepeatCnt = 10; + + /** Objects per mode. */ + public static readonly int ObjectsPerMode = 5; + + /// <summary> + /// Test object write with different structures. + /// </summary> + [Test] + public void TestStructure() + { + for (int i = 1; i <= RepeatCnt; i++) + { + Console.WriteLine(">>> Iteration started: " + i); + + // 1. Generate and shuffle objects. + IList<BranchedType> objs = new List<BranchedType>(); + + for (int j = 0; j < 6 * ObjectsPerMode; j++) + objs.Add(new BranchedType((j%6) + 1)); + + objs = IgniteUtils.Shuffle(objs); + + // 2. Create new marshaller. + PortableTypeConfiguration typeCfg = new PortableTypeConfiguration(typeof(BranchedType)); + + PortableConfiguration cfg = new PortableConfiguration + { + TypeConfigurations = new List<PortableTypeConfiguration> { typeCfg } + }; + + PortableMarshaller marsh = new PortableMarshaller(cfg); + + // 3. Marshal all data and ensure deserialized object is fine. + foreach (BranchedType obj in objs) + { + Console.WriteLine(">>> Write object [mode=" + obj.mode + ']'); + + byte[] data = marsh.Marshal(obj); + + BranchedType other = marsh.Unmarshal<BranchedType>(data); + + Assert.IsTrue(obj.Equals(other)); + } + + Console.WriteLine(); + + // 4. Ensure that all fields are recorded. + IPortableTypeDescriptor desc = marsh.Descriptor(typeof (BranchedType)); + + PortableStructure typeStruct = desc.TypeStructure; + + IDictionary<string, byte> fields = typeStruct.FieldTypes; + + Assert.IsTrue(fields.Count == 8); + + Assert.IsTrue(fields.ContainsKey("mode")); + Assert.IsTrue(fields.ContainsKey("f2")); + Assert.IsTrue(fields.ContainsKey("f3")); + Assert.IsTrue(fields.ContainsKey("f4")); + Assert.IsTrue(fields.ContainsKey("f5")); + Assert.IsTrue(fields.ContainsKey("f6")); + Assert.IsTrue(fields.ContainsKey("f7")); + Assert.IsTrue(fields.ContainsKey("f8")); + } + } + } + + public class BranchedType : IPortableMarshalAware + { + public int mode; + public int f2; + public int f3; + public int f4; + public int f5; + public int f6; + public int f7; + public int f8; + + public BranchedType(int mode) + { + this.mode = mode; + + switch (mode) + { + case 1: + f2 = 2; + + break; + + case 2: + f2 = 2; + f3 = 3; + f4 = 4; + + break; + + case 3: + f2 = 2; + f3 = 3; + f5 = 5; + + break; + + case 4: + f2 = 2; + f3 = 3; + f5 = 5; + f6 = 6; + + break; + + case 5: + f2 = 2; + f3 = 3; + f7 = 7; + + break; + + case 6: + f8 = 8; + + break; + } + } + + public void WritePortable(IPortableWriter writer) + { + writer.WriteInt("mode", mode); + + switch (mode) + { + case 1: + writer.WriteInt("f2", f2); + + break; + + case 2: + writer.WriteInt("f2", f2); + writer.WriteInt("f3", f3); + writer.WriteInt("f4", f4); + + break; + + case 3: + writer.WriteInt("f2", f2); + writer.WriteInt("f3", f3); + writer.WriteInt("f5", f5); + + break; + + case 4: + writer.WriteInt("f2", f2); + writer.WriteInt("f3", f3); + writer.WriteInt("f5", f5); + writer.WriteInt("f6", f6); + + break; + + case 5: + writer.WriteInt("f2", f2); + writer.WriteInt("f3", f3); + writer.WriteInt("f7", f7); + + break; + + case 6: + writer.WriteInt("f8", f8); + + break; + } + } + + public void ReadPortable(IPortableReader reader) + { + mode = reader.ReadInt("mode"); + + switch (mode) + { + case 1: + f2 = reader.ReadInt("f2"); + + break; + + case 2: + f2 = reader.ReadInt("f2"); + f3 = reader.ReadInt("f3"); + f4 = reader.ReadInt("f4"); + + break; + + case 3: + f2 = reader.ReadInt("f2"); + f3 = reader.ReadInt("f3"); + f5 = reader.ReadInt("f5"); + + break; + + case 4: + f2 = reader.ReadInt("f2"); + f3 = reader.ReadInt("f3"); + f5 = reader.ReadInt("f5"); + f6 = reader.ReadInt("f6"); + + break; + + case 5: + f2 = reader.ReadInt("f2"); + f3 = reader.ReadInt("f3"); + f7 = reader.ReadInt("f7"); + + break; + + case 6: + f8 = reader.ReadInt("f8"); + + break; + } + } + + public bool Equals(BranchedType other) + { + return mode == other.mode && f2 == other.f2 && f3 == other.f3 && f4 == other.f4 && f5 == other.f5 && + f6 == other.f6 && f7 == other.f7 && f8 == other.f8; + } + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/4e053e45/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj index 855dda8..848ce49 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj @@ -275,6 +275,10 @@ <Compile Include="Impl\Portable\PortableUtils.cs" /> <Compile Include="Impl\Portable\PortableWriterImpl.cs" /> <Compile Include="Impl\Portable\SerializableObjectHolder.cs" /> + <Compile Include="Impl\Portable\Structure\PortableStructure.cs" /> + <Compile Include="Impl\Portable\Structure\PortableStructureEntry.cs" /> + <Compile Include="Impl\Portable\Structure\PortableStructureJumpTable.cs" /> + <Compile Include="Impl\Portable\Structure\PortableStructureUpdate.cs" /> <Compile Include="Impl\Portable\TypeResolver.cs" /> <Compile Include="Impl\Resource\IResourceInjector.cs" /> <Compile Include="Impl\Resource\ResourceFieldInjector.cs" /> http://git-wip-us.apache.org/repos/asf/ignite/blob/4e053e45/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/IPortableTypeDescriptor.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/IPortableTypeDescriptor.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/IPortableTypeDescriptor.cs index 62597d5..81b6c8d 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/IPortableTypeDescriptor.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/IPortableTypeDescriptor.cs @@ -18,6 +18,9 @@ namespace Apache.Ignite.Core.Impl.Portable { using System; + using System.Collections.Generic; + + using Apache.Ignite.Core.Impl.Portable.Structure; using Apache.Ignite.Core.Portable; /// <summary> @@ -104,5 +107,18 @@ namespace Apache.Ignite.Core.Impl.Portable { get; } + + /// <summary> + /// Type structure. + /// </summary> + PortableStructure TypeStructure { get; } + + /// <summary> + /// Update type structure. + /// </summary> + /// <param name="exp">Expected type structure.</param> + /// <param name="pathIdx">Path index.</param> + /// <param name="updates">Recorded updates.</param> + void UpdateStructure(PortableStructure exp, int pathIdx, IList<PortableStructureUpdate> updates); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/4e053e45/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableFullTypeDescriptor.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableFullTypeDescriptor.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableFullTypeDescriptor.cs index 79b860f..ecad8ba 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableFullTypeDescriptor.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableFullTypeDescriptor.cs @@ -18,6 +18,9 @@ namespace Apache.Ignite.Core.Impl.Portable { using System; + using System.Collections.Generic; + + using Apache.Ignite.Core.Impl.Portable.Structure; using Apache.Ignite.Core.Portable; /// <summary> @@ -55,6 +58,9 @@ namespace Apache.Ignite.Core.Impl.Portable /** Affinity field key name. */ private readonly string _affKeyFieldName; + /** Type structure. */ + private volatile PortableStructure _typeStruct = PortableStructure.CreateEmpty(); + /// <summary> /// Constructor. /// </summary> @@ -171,5 +177,21 @@ namespace Apache.Ignite.Core.Impl.Portable { get { return _affKeyFieldName; } } + + /** <inheritDoc /> */ + public PortableStructure TypeStructure + { + get { return _typeStruct; } + } + + /** <inheritDoc /> */ + public void UpdateStructure(PortableStructure exp, int pathIdx, + IList<PortableStructureUpdate> updates) + { + lock (this) + { + _typeStruct = _typeStruct.Merge(exp, pathIdx, updates); + } + } } } http://git-wip-us.apache.org/repos/asf/ignite/blob/4e053e45/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableMarshaller.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableMarshaller.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableMarshaller.cs index c7a0b7b..c7262e1 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableMarshaller.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableMarshaller.cs @@ -309,6 +309,7 @@ namespace Apache.Ignite.Core.Impl.Portable return new PortableHashsetMetadataHandler(ids, newType); } + return null; } http://git-wip-us.apache.org/repos/asf/ignite/blob/4e053e45/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableSurrogateTypeDescriptor.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableSurrogateTypeDescriptor.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableSurrogateTypeDescriptor.cs index 9842c46..8dc18b6 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableSurrogateTypeDescriptor.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableSurrogateTypeDescriptor.cs @@ -18,10 +18,14 @@ namespace Apache.Ignite.Core.Impl.Portable { using System; + using System.Collections.Generic; + + using Apache.Ignite.Core.Impl.Portable.Structure; using Apache.Ignite.Core.Portable; /// <summary> - /// Surrogate type descriptor. Used in cases when type if identified by name and is not provided in configuration. + /// Surrogate type descriptor. Used in cases when type if identified by name and + /// is not provided in configuration. /// </summary> internal class PortableSurrogateTypeDescriptor : IPortableTypeDescriptor { @@ -34,6 +38,9 @@ namespace Apache.Ignite.Core.Impl.Portable /** Type name. */ private readonly string _name; + /** Type structure. */ + private volatile PortableStructure _typeStruct = PortableStructure.CreateEmpty(); + /// <summary> /// Constructor. /// </summary> @@ -117,5 +124,20 @@ namespace Apache.Ignite.Core.Impl.Portable { get { return null; } } + + /** <inheritDoc /> */ + public PortableStructure TypeStructure + { + get { return _typeStruct; } + } + + /** <inheritDoc /> */ + public void UpdateStructure(PortableStructure exp, int pathIdx, IList<PortableStructureUpdate> updates) + { + lock (this) + { + _typeStruct = _typeStruct.Merge(exp, pathIdx, updates); + } + } } } http://git-wip-us.apache.org/repos/asf/ignite/blob/4e053e45/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableUtils.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableUtils.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableUtils.cs index 2344db2..fb0b195 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableUtils.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableUtils.cs @@ -1799,6 +1799,10 @@ namespace Apache.Ignite.Core.Impl.Portable if (id == 0) id = StringHashCode(fieldName); + if (id == 0) + throw new PortableException("Field ID is zero (please provide ID mapper or change field name) " + + "[typeId=" + typeId + ", fieldName=" + fieldName + ", idMapper=" + idMapper + ']'); + return id; } http://git-wip-us.apache.org/repos/asf/ignite/blob/4e053e45/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableWriterImpl.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableWriterImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableWriterImpl.cs index 69523c9..0495b68 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableWriterImpl.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/PortableWriterImpl.cs @@ -24,6 +24,7 @@ namespace Apache.Ignite.Core.Impl.Portable using Apache.Ignite.Core.Impl.Portable.IO; using Apache.Ignite.Core.Impl.Portable.Metadata; + using Apache.Ignite.Core.Impl.Portable.Structure; using Apache.Ignite.Core.Portable; using PU = PortableUtils; @@ -56,12 +57,21 @@ namespace Apache.Ignite.Core.Impl.Portable /** Current mapper. */ private IPortableIdMapper _curMapper; - - /** Current metadata handler. */ - private IPortableMetadataHandler _curMetaHnd; /** Current raw position. */ private long _curRawPos; + + /** Current type structure. */ + private PortableStructure _curStruct; + + /** Current type structure path index. */ + private int _curStructPath; + + /** Current type structure action index. */ + private int _curStructAction; + + /** Current type structure updates. */ + private List<PortableStructureUpdate> _curStructUpdates; /** Whether we are currently detaching an object. */ private bool _detaching; @@ -1301,16 +1311,24 @@ namespace Apache.Ignite.Core.Impl.Portable int oldTypeId = _curTypeId; IPortableNameMapper oldConverter = _curConverter; IPortableIdMapper oldMapper = _curMapper; - IPortableMetadataHandler oldMetaHnd = _curMetaHnd; long oldRawPos = _curRawPos; + + PortableStructure oldStruct = _curStruct; + int oldStructPath = _curStructPath; + int oldStructAction = _curStructAction; + var oldStructUpdates = _curStructUpdates; // Push new frame. _curTypeId = desc.TypeId; _curConverter = desc.NameConverter; _curMapper = desc.Mapper; - _curMetaHnd = desc.MetadataEnabled ? _marsh.MetadataHandler(desc) : null; _curRawPos = 0; + _curStruct = desc.TypeStructure; + _curStructPath = 0; + _curStructAction = 0; + _curStructUpdates = null; + // Write object fields. desc.Serializer.WritePortable(obj, this); @@ -1324,21 +1342,35 @@ namespace Apache.Ignite.Core.Impl.Portable else _stream.WriteInt(pos + 14, len); - // 13. Collect metadata. - if (_curMetaHnd != null) + // Apply structure updates if any. + if (_curStructUpdates != null) { - IDictionary<string, int> meta = _curMetaHnd.OnObjectWriteFinished(); + desc.UpdateStructure(_curStruct, _curStructPath, _curStructUpdates); - if (meta != null) - SaveMetadata(_curTypeId, desc.TypeName, desc.AffinityKeyFieldName, meta); + IPortableMetadataHandler metaHnd = _marsh.MetadataHandler(desc); + + if (metaHnd != null) + { + foreach (var u in _curStructUpdates) + metaHnd.OnFieldWrite(u.FieldId, u.FieldName, u.FieldType); + + IDictionary<string, int> meta = metaHnd.OnObjectWriteFinished(); + + if (meta != null) + SaveMetadata(_curTypeId, desc.TypeName, desc.AffinityKeyFieldName, meta); + } } // Restore old frame. _curTypeId = oldTypeId; _curConverter = oldConverter; _curMapper = oldMapper; - _curMetaHnd = oldMetaHnd; _curRawPos = oldRawPos; + + _curStruct = oldStruct; + _curStructPath = oldStructPath; + _curStructAction = oldStructAction; + _curStructUpdates = oldStructUpdates; } else { @@ -1595,12 +1627,40 @@ namespace Apache.Ignite.Core.Impl.Portable if (_curRawPos != 0) throw new PortableException("Cannot write named fields after raw data is written."); - int fieldId = PU.FieldId(_curTypeId, fieldName, _curConverter, _curMapper); + int action = _curStructAction++; + + int fieldId; + + if (_curStructUpdates == null) + { + fieldId = _curStruct.GetFieldId(fieldName, fieldTypeId, ref _curStructPath, action); + + if (fieldId == 0) + fieldId = GetNewFieldId(fieldName, fieldTypeId, action); + } + else + fieldId = GetNewFieldId(fieldName, fieldTypeId, action); _stream.WriteInt(fieldId); + } + + /// <summary> + /// Get ID for the new field and save structure update. + /// </summary> + /// <param name="fieldName">Field name.</param> + /// <param name="fieldTypeId">Field type ID.</param> + /// <param name="action">Action index.</param> + /// <returns>Field ID.</returns> + private int GetNewFieldId(string fieldName, byte fieldTypeId, int action) + { + int fieldId = PU.FieldId(_curTypeId, fieldName, _curConverter, _curMapper); + + if (_curStructUpdates == null) + _curStructUpdates = new List<PortableStructureUpdate>(); + + _curStructUpdates.Add(new PortableStructureUpdate(fieldName, fieldId, fieldTypeId, action)); - if (_curMetaHnd != null) - _curMetaHnd.OnFieldWrite(fieldId, fieldName, fieldTypeId); + return fieldId; } /// <summary> http://git-wip-us.apache.org/repos/asf/ignite/blob/4e053e45/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/Structure/PortableStructure.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/Structure/PortableStructure.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/Structure/PortableStructure.cs new file mode 100644 index 0000000..2d47755 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/Structure/PortableStructure.cs @@ -0,0 +1,333 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Impl.Portable.Structure +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + + using Apache.Ignite.Core.Portable; + + /// <summary> + /// Portable type structure. Cache field IDs and metadata to improve marshalling performance. + /// Every object write contains a set of field writes. Every unique ordered set of written fields + /// produce write "path". We cache these paths allowing for very fast traverse over object structure + /// without expensive map lookups and field ID calculations. + /// </summary> + internal class PortableStructure + { + /// <summary> + /// Create empty type structure. + /// </summary> + /// <returns>Empty type structure.</returns> + public static PortableStructure CreateEmpty() + { + return new PortableStructure(new[] { new PortableStructureEntry[0] }, + new PortableStructureJumpTable[1], new Dictionary<string, byte>()); + } + + /** Entries. */ + private readonly PortableStructureEntry[][] _paths; + + /** Jumps. */ + private readonly PortableStructureJumpTable[] _jumps; + + /** Field types. */ + private readonly IDictionary<string, byte> _fieldTypes; + + /// <summary> + /// Constructor. + /// </summary> + /// <param name="paths">Paths.</param> + /// <param name="jumps">Jumps.</param> + /// <param name="fieldTypes">Field types.</param> + private PortableStructure(PortableStructureEntry[][] paths, + PortableStructureJumpTable[] jumps, IDictionary<string, byte> fieldTypes) + { + _paths = paths; + _jumps = jumps; + _fieldTypes = fieldTypes; + } + + /// <summary> + /// Gets field ID if possible. + /// </summary> + /// <param name="fieldName">Field name.</param> + /// <param name="fieldType">Field type.</param> + /// <param name="pathIdx">Path index, changes during jumps.</param> + /// <param name="actionIdx">Action index.</param> + /// <returns>Field ID or zero in case there are no matching path.</returns> + public int GetFieldId(string fieldName, byte fieldType, ref int pathIdx, int actionIdx) + { + Debug.Assert(pathIdx <= _paths.Length); + + // Get path. + PortableStructureEntry[] path = _paths[pathIdx]; + + if (actionIdx < path.Length) + { + // Get entry matching the action index. + PortableStructureEntry entry = path[actionIdx]; + + if (entry.IsExpected(fieldName, fieldType)) + // Entry matches our expectations, return. + return entry.Id; + else if (entry.IsJumpTable) + { + // Entry is a pointer to a jump table. + Debug.Assert(entry.Id < _jumps.Length); + + PortableStructureJumpTable jmpTbl = _jumps[entry.Id]; + + int pathIdx0 = jmpTbl.GetPathIndex(fieldName); + + if (pathIdx0 < 0) + return 0; + + Debug.Assert(pathIdx0 < _paths.Length); + + entry = _paths[pathIdx0][actionIdx]; + + entry.ValidateType(fieldType); + + pathIdx = pathIdx0; + + return entry.Id; + } + } + + // Failed to find anything because this is a new field. + return 0; + } + + /// <summary> + /// Merge updates into a new type structure. + /// </summary> + /// <param name="exp">Expected type structure to apply updates to </param> + /// <param name="pathIdx">Path index.</param> + /// <param name="updates">Updates.</param> + /// <returns>New type structure with updates.</returns> + public PortableStructure Merge(PortableStructure exp, int pathIdx, + IList<PortableStructureUpdate> updates) + { + if (updates.Count == 0) + return this; + + // Algorithm ensures that updates are applied to the same type structure, + // where they were initially observed. This allow us to keep structure + // internals simpler and more efficient. On the other hand, this imposes + // some performance hit because in case of concurrent update, recorded + // changes will be discarded and recorded again during the next write + // on the same path. This should occur only during application warmup. + + // Note that field types are merged anyway to avoid metadata clashes. + PortableStructure res = MergeFieldTypes(updates); + + if (ReferenceEquals(exp, this)) + { + PortableStructureUpdate firstUpdate = updates[0]; + + if (firstUpdate.Index == 0) + { + // Special case: the very first structure update. Simply attach all updates. + Debug.Assert(_paths.Length == 1); + Debug.Assert(_paths[0].Length == 0); + Debug.Assert(pathIdx == 0); + + var newPaths = CopyPaths(updates.Count, 0); + + ApplyUpdatesToPath(newPaths[0], updates); + + res = new PortableStructure(newPaths, _jumps, res._fieldTypes); + } + else + { + // Get entry where updates should start. + PortableStructureEntry[] path = _paths[pathIdx]; + + PortableStructureEntry startEntry = default(PortableStructureEntry); + + if (firstUpdate.Index < path.Length) + startEntry = path[firstUpdate.Index]; + + if (startEntry.IsEmpty) + { + // We are on the empty/non-existent entry. Continue the path without branching. + var newPaths = CopyPaths(firstUpdate.Index + updates.Count, 0); + + ApplyUpdatesToPath(newPaths[pathIdx], updates); + + res = new PortableStructure(newPaths, _jumps, res._fieldTypes); + } + else if (startEntry.IsJumpTable) + { + // We are on the jump table. Add a new path and record it in the jump table. + + // 1. Prepare new structures. + var newPaths = CopyPaths(firstUpdate.Index + updates.Count, 1); + var newJumps = CopyJumps(0); + + // New path will be the last one. + int newPathIdx = newPaths.Length - 1; + + // Apply updates to the new path. + ApplyUpdatesToPath(newPaths[newPathIdx], updates); + + // Add the jump to the table. + newJumps[startEntry.Id] = + newJumps[startEntry.Id].CopyAndAdd(firstUpdate.FieldName, newPathIdx); + + res = new PortableStructure(newPaths, newJumps, res._fieldTypes); + } + else + { + // We are on existing entry. Need to create a new jump table here and two new paths. + + // 1. Prepaare new structures. + var newPaths = CopyPaths(firstUpdate.Index + updates.Count, 2); + var newJumps = CopyJumps(1); + + // Old path will be moved here. + int oldPathIdx = newPaths.Length - 2; + + // New path will reside here. + int newPathIdx = newPaths.Length - 1; + + // Create new jump table. + int newJumpIdx = newJumps.Length - 1; + + newJumps[newJumpIdx] = new PortableStructureJumpTable(startEntry.Name, oldPathIdx, + firstUpdate.FieldName, newPathIdx); + + // Re-create old path in two steps: move old path to the new place, then clean the old path. + for (int i = firstUpdate.Index; i < path.Length; i++) + { + newPaths[oldPathIdx][i] = newPaths[pathIdx][i]; + + if (i == firstUpdate.Index) + // Inject jump table ... + newPaths[pathIdx][i] = new PortableStructureEntry(newJumpIdx); + else + // ... or just reset. + newPaths[pathIdx][i] = new PortableStructureEntry(); + } + + // Apply updates to the new path. + ApplyUpdatesToPath(newPaths[newPaths.Length - 1], updates); + + res = new PortableStructure(newPaths, newJumps, res._fieldTypes); + } + + } + } + + return res; + } + + /// <summary> + /// Copy and possibly expand paths. + /// </summary> + /// <param name="minLen">Minimum length.</param> + /// <param name="additionalPaths">Amount of additional paths required.</param> + /// <returns>Result.</returns> + private PortableStructureEntry[][] CopyPaths(int minLen, int additionalPaths) + { + var newPaths = new PortableStructureEntry[_paths.Length + additionalPaths][]; + + int newPathLen = Math.Max(_paths[0].Length, minLen); + + for (int i = 0; i < newPaths.Length; i++) + { + newPaths[i] = new PortableStructureEntry[newPathLen]; + + if (i < _paths.Length) + Array.Copy(_paths[i], newPaths[i], _paths[i].Length); + } + + return newPaths; + } + + /// <summary> + /// Copy and possibly expand jump tables. + /// </summary> + /// <param name="additionalJumps">Amount of additional jumps required.</param> + /// <returns>Result.</returns> + private PortableStructureJumpTable[] CopyJumps(int additionalJumps) + { + var newJumps = new PortableStructureJumpTable[_jumps.Length + additionalJumps]; + + // The very first jump is always null so that we can distinguish between jump table + // and empty value in PortableStructureEntry. + for (int i = 1; i < _jumps.Length; i++) + newJumps[i] = _jumps[i].Copy(); + + return newJumps; + } + + /// <summary> + /// Apply updates to path. + /// </summary> + /// <param name="path">Path.</param> + /// <param name="updates">Updates.</param> + private static void ApplyUpdatesToPath(IList<PortableStructureEntry> path, + IEnumerable<PortableStructureUpdate> updates) + { + foreach (var u in updates) + path[u.Index] = new PortableStructureEntry(u.FieldName, u.FieldId, u.FieldType); + } + + /// <summary> + /// Merge field types. + /// </summary> + /// <param name="updates">Updates.</param> + /// <returns>Type structure with applied updates.</returns> + private PortableStructure MergeFieldTypes(IList<PortableStructureUpdate> updates) + { + IDictionary<string, byte> newFieldTypes = new Dictionary<string, byte>(_fieldTypes); + + foreach (PortableStructureUpdate update in updates) + { + byte expType; + + if (_fieldTypes.TryGetValue(update.FieldName, out expType)) + { + // This is an old field. + if (expType != update.FieldType) + { + throw new PortableException("Field type mismatch detected [fieldName=" + update.FieldName + + ", expectedType=" + expType + ", actualType=" + update.FieldType + ']'); + } + } + else + // This is a new field. + newFieldTypes[update.FieldName] = update.FieldType; + } + + return newFieldTypes.Count == _fieldTypes.Count ? + this : new PortableStructure(_paths, _jumps, newFieldTypes); + } + + /// <summary> + /// Recorded field types. + /// </summary> + internal IDictionary<string, byte> FieldTypes + { + get { return _fieldTypes; } + } + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/4e053e45/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/Structure/PortableStructureEntry.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/Structure/PortableStructureEntry.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/Structure/PortableStructureEntry.cs new file mode 100644 index 0000000..9476635 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/Structure/PortableStructureEntry.cs @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Impl.Portable.Structure +{ + using System.Diagnostics; + + using Apache.Ignite.Core.Portable; + + /// <summary> + /// Portable type structure entry. Might be either a normal field, a reference to jump table, or an empty entry. + /// </summary> + internal struct PortableStructureEntry + { + /** Field name. */ + private readonly string _name; + + /** Field ID. */ + private readonly int _id; + + /** Field type. */ + private readonly byte _type; + + /// <summary> + /// Constructor for jump table entry. + /// </summary> + /// <param name="jumpTblIdx">Jump table index.</param> + public PortableStructureEntry(int jumpTblIdx) + { + Debug.Assert(jumpTblIdx > 0); + + _name = null; + _id = jumpTblIdx; + _type = 0; + } + + /// <summary> + /// Constructor for field entry. + /// </summary> + /// <param name="name">Field name.</param> + /// <param name="id">Field ID.</param> + /// <param name="type">Field type.</param> + public PortableStructureEntry(string name, int id, byte type) + { + Debug.Assert(name != null); + + _name = name; + _id = id; + _type = type; + } + + /// <summary> + /// Check whether current field entry matches passed arguments. + /// </summary> + /// <param name="name">Field name.</param> + /// <param name="type">Field type.</param> + /// <returns>True if expected.</returns> + public bool IsExpected(string name, byte type) + { + // Perform reference equality check first because field name is a literal in most cases. + if (!ReferenceEquals(_name, name) && !name.Equals(_name)) + return false; + + ValidateType(type); + + return true; + } + + /// <summary> + /// Validate field type. + /// </summary> + /// <param name="type">Expected type.</param> + public void ValidateType(byte type) + { + if (_type != type) + { + throw new PortableException("Field type mismatch detected [fieldName=" + _name + + ", expectedType=" + _type + ", actualType=" + type + ']'); + } + } + + /// <summary> + /// Whether this is an empty entry. + /// </summary> + /// <returns></returns> + public bool IsEmpty + { + get { return _id == 0; } + } + + /// <summary> + /// Whether this is a jump table. + /// </summary> + public bool IsJumpTable + { + get { return _name == null && _id >= 0; } + } + + /// <summary> + /// Field name. + /// </summary> + public string Name + { + get { return _name; } + } + + /// <summary> + /// Field ID. + /// </summary> + public int Id + { + get { return _id; } + } + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/4e053e45/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/Structure/PortableStructureJumpTable.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/Structure/PortableStructureJumpTable.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/Structure/PortableStructureJumpTable.cs new file mode 100644 index 0000000..7f8b373 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/Structure/PortableStructureJumpTable.cs @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Impl.Portable.Structure +{ + using System; + using System.Diagnostics; + + /// <summary> + /// Jump table. + /// </summary> + internal class PortableStructureJumpTable + { + /** Names. */ + private readonly string[] _names; + + /** Path indexes. */ + private readonly int[] _pathIdxs; + + /// <summary> + /// Create minimal jump table with two entries. + /// </summary> + /// <param name="firstName">First name.</param> + /// <param name="firstPathIdx">First path index.</param> + /// <param name="secondName">Second name.</param> + /// <param name="secondPathIdx">Second path index.</param> + public PortableStructureJumpTable(string firstName, int firstPathIdx, + string secondName, int secondPathIdx) + { + _names = new[] { firstName, secondName }; + _pathIdxs = new[] { firstPathIdx, secondPathIdx }; + } + + /// <summary> + /// Constructor. + /// </summary> + /// <param name="names">Field names.</param> + /// <param name="pathIdxs">Path indexes.</param> + private PortableStructureJumpTable(string[] names, int[] pathIdxs) + { + Debug.Assert(names.Length > 1); + Debug.Assert(names.Length == pathIdxs.Length); + + _names = names; + _pathIdxs = pathIdxs; + } + + /// <summary> + /// Get path index for the given field. + /// </summary> + /// <param name="fieldName">Field name.</param> + /// <returns>Path index.</returns> + public int GetPathIndex(string fieldName) + { + Debug.Assert(fieldName != null); + + // Optimistically assume that field name is a literal. + for (var i = 0; i < _names.Length; i++) + { + if (ReferenceEquals(fieldName, _names[i])) + return _pathIdxs[i]; + } + + // Fallback to slow-path with normal string comparison. + for (var i = 0; i < _names.Length; i++) + { + if (fieldName.Equals(_names[i])) + return _pathIdxs[i]; + } + + // No path found for the field. + return -1; + } + + /// <summary> + /// Copy jump table. + /// </summary> + /// <returns>New jump table.</returns> + public PortableStructureJumpTable Copy() + { + return new PortableStructureJumpTable(_names, _pathIdxs); + } + + /// <summary> + /// Copy jump table with additional jump. + /// </summary> + /// <param name="name">Field name.</param> + /// <param name="pathIdx">Path index.</param> + /// <returns>New jump table.</returns> + public PortableStructureJumpTable CopyAndAdd(string name, int pathIdx) + { + var newNames = new string[_names.Length + 1]; + var newPathIdxs = new int[_pathIdxs.Length + 1]; + + Array.Copy(_names, newNames, _names.Length); + Array.Copy(_pathIdxs, newPathIdxs, _pathIdxs.Length); + + newNames[newNames.Length - 1] = name; + newPathIdxs[newPathIdxs.Length - 1] = pathIdx; + + return new PortableStructureJumpTable(newNames, newPathIdxs); + } + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/4e053e45/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/Structure/PortableStructureUpdate.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/Structure/PortableStructureUpdate.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/Structure/PortableStructureUpdate.cs new file mode 100644 index 0000000..fa239db --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Portable/Structure/PortableStructureUpdate.cs @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Impl.Portable.Structure +{ + /// <summary> + /// Portable type structure update descriptor. + /// </summary> + internal class PortableStructureUpdate + { + /** Field name. */ + private readonly string _fieldName; + + /** Field ID. */ + private readonly int _fieldId; + + /** Field type. */ + private readonly byte _fieldType; + + /** Field index. */ + private readonly int _idx; + + /// <summary> + /// Constructor. + /// </summary> + /// <param name="fieldName">Field name.</param> + /// <param name="fieldId">Field ID.</param> + /// <param name="fieldType">Field type.</param> + /// <param name="idx">Index.</param> + public PortableStructureUpdate(string fieldName, int fieldId, byte fieldType, int idx) + { + _fieldName = fieldName; + _fieldId = fieldId; + _fieldType = fieldType; + _idx = idx; + } + + /// <summary> + /// Field name. + /// </summary> + public string FieldName + { + get { return _fieldName; } + } + + /// <summary> + /// Field ID. + /// </summary> + public int FieldId + { + get { return _fieldId; } + } + + /// <summary> + /// Field type. + /// </summary> + public byte FieldType + { + get { return _fieldType; } + } + + /// <summary> + /// Index. + /// </summary> + public int Index + { + get { return _idx; } + } + } +}
