IGNITE-1816: Implemented compact footers.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0b4a8f83 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0b4a8f83 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0b4a8f83 Branch: refs/heads/ignite-1282 Commit: 0b4a8f831fe2d183e6e831c90da0d3fc86ac2ed0 Parents: 66c84ea Author: vozerov-gridgain <voze...@gridgain.com> Authored: Wed Nov 18 14:54:38 2015 +0300 Committer: vozerov-gridgain <voze...@gridgain.com> Committed: Wed Nov 18 14:54:38 2015 +0300 ---------------------------------------------------------------------- .../apache/ignite/internal/IgniteKernal.java | 6 + .../ignite/internal/IgniteNodeAttributes.java | 3 + .../portable/BinaryCachingMetadataHandler.java | 70 + .../internal/portable/BinaryMetadata.java | 16 +- .../portable/BinaryMetadataCollector.java | 49 +- .../internal/portable/BinaryObjectEx.java | 2 +- .../internal/portable/BinaryObjectImpl.java | 14 +- .../portable/BinaryObjectOffheapImpl.java | 14 +- .../internal/portable/BinaryReaderExImpl.java | 132 +- .../internal/portable/BinaryTypeImpl.java | 7 +- .../internal/portable/BinaryWriterExImpl.java | 161 +- .../portable/PortableClassDescriptor.java | 102 +- .../internal/portable/PortableContext.java | 66 +- .../internal/portable/PortableSchema.java | 296 +- .../ignite/internal/portable/PortableUtils.java | 457 ++- .../builder/BinaryObjectBuilderImpl.java | 130 +- .../portable/builder/PortableBuilderReader.java | 21 +- .../portable/CacheObjectBinaryProcessor.java | 2 +- .../CacheObjectBinaryProcessorImpl.java | 162 +- .../platform/PlatformContextImpl.java | 10 +- .../cpp/PlatformCppConfigurationClosure.java | 9 +- .../PlatformDotNetConfigurationClosure.java | 9 +- .../ignite/internal/util/IgniteUtils.java | 25 + .../marshaller/portable/PortableMarshaller.java | 38 +- .../ignite/spi/discovery/tcp/ServerImpl.java | 51 +- .../portable/BinaryFieldsAbstractSelfTest.java | 13 +- .../BinaryFooterOffsetsAbstractSelfTest.java | 199 + .../BinaryFooterOffsetsHeapSelfTest.java | 32 + .../BinaryFooterOffsetsOffheapSelfTest.java | 61 + .../portable/BinaryMarshallerSelfTest.java | 3795 ++++++++++++++++++ .../BinaryObjectBuilderAdditionalSelfTest.java | 1291 ++++++ .../portable/BinaryObjectBuilderSelfTest.java | 1066 +++++ ...idBinaryObjectBuilderAdditionalSelfTest.java | 1280 ------ .../GridBinaryObjectBuilderSelfTest.java | 1053 ----- ...idPortableMarshallerCtxDisabledSelfTest.java | 2 +- .../GridPortableMarshallerSelfTest.java | 3760 ----------------- .../PortableCompactOffsetsAbstractSelfTest.java | 190 - .../PortableCompactOffsetsHeapSelfTest.java | 32 - .../PortableCompactOffsetsOffheapSelfTest.java | 61 - .../BinaryFieldsHeapNonCompactSelfTest.java | 34 + .../BinaryFieldsOffheapNonCompactSelfTest.java | 30 + ...naryFooterOffsetsHeapNonCompactSelfTest.java | 30 + ...yFooterOffsetsOffheapNonCompactSelfTest.java | 30 + .../BinaryMarshallerNonCompactSelfTest.java | 30 + ...jectBuilderAdditionalNonCompactSelfTest.java | 30 + .../BinaryObjectBuilderNonCompactSelfTest.java | 30 + .../IgnitePortableObjectsTestSuite.java | 36 +- .../core-test/src/binary_reader_writer_test.cpp | 64 +- .../include/ignite/impl/binary/binary_common.h | 22 +- .../ignite/impl/binary/binary_reader_impl.h | 58 +- .../include/ignite/impl/binary/binary_schema.h | 6 +- .../ignite/impl/binary/binary_writer_impl.h | 2 +- .../core/src/impl/binary/binary_reader_impl.cpp | 8 +- .../cpp/core/src/impl/binary/binary_schema.cpp | 12 +- .../core/src/impl/binary/binary_writer_impl.cpp | 29 +- .../Config/Compute/compute-grid1.xml | 1 + .../Config/marshaller-explicit.xml | 4 +- .../Impl/Binary/BinaryObjectBuilder.cs | 25 +- .../Impl/Binary/BinaryObjectHeader.cs | 131 +- .../Impl/Binary/BinaryObjectSchemaHolder.cs | 9 +- .../Impl/Binary/BinaryReader.cs | 2 +- .../Impl/Binary/BinaryWriter.cs | 27 +- .../Impl/Binary/IgniteBinary.cs | 3 +- 63 files changed, 8154 insertions(+), 7186 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index c4829a4..2b6eaad 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -157,6 +157,7 @@ import org.apache.ignite.lifecycle.LifecycleBean; import org.apache.ignite.lifecycle.LifecycleEventType; import org.apache.ignite.marshaller.MarshallerExclusions; import org.apache.ignite.marshaller.optimized.OptimizedMarshaller; +import org.apache.ignite.marshaller.portable.PortableMarshaller; import org.apache.ignite.mxbean.ClusterLocalNodeMetricsMXBean; import org.apache.ignite.mxbean.IgniteMXBean; import org.apache.ignite.mxbean.ThreadPoolMXBean; @@ -201,6 +202,7 @@ import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_JVM_PID; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LANG_RUNTIME; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MACS; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER; +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_COMPACT_FOOTER; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CONSISTENT_ID; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PEER_CLASSLOADING; @@ -1272,6 +1274,10 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable { add(ATTR_MARSHALLER, cfg.getMarshaller().getClass().getName()); add(ATTR_MARSHALLER_USE_DFLT_SUID, getBoolean(IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID, OptimizedMarshaller.USE_DFLT_SUID)); + + if (cfg.getMarshaller() instanceof PortableMarshaller) + add(ATTR_MARSHALLER_COMPACT_FOOTER, ((PortableMarshaller)cfg.getMarshaller()).isCompactFooter()); + add(ATTR_USER_NAME, System.getProperty("user.name")); add(ATTR_GRID_NAME, gridName); http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java index 86a460d..946b686 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java @@ -36,6 +36,9 @@ public final class IgniteNodeAttributes { /** Internal attribute name constant. */ public static final String ATTR_MARSHALLER_USE_DFLT_SUID = ATTR_PREFIX + ".marshaller.useDefaultSUID"; + /** Attribute for marshaller compact footers. */ + public static final String ATTR_MARSHALLER_COMPACT_FOOTER = ATTR_PREFIX + ".marshaller.compactFooter"; + /** Internal attribute name constant. */ public static final String ATTR_JIT_NAME = ATTR_PREFIX + ".jit.name"; http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryCachingMetadataHandler.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryCachingMetadataHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryCachingMetadataHandler.java new file mode 100644 index 0000000..a3c846b --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryCachingMetadataHandler.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.portable; + +import org.apache.ignite.binary.BinaryObjectException; +import org.apache.ignite.binary.BinaryType; + +import java.util.HashMap; + +/** + * Simple caching metadata handler. Used mainly in tests. + */ +public class BinaryCachingMetadataHandler implements BinaryMetadataHandler { + /** Cached metadatas. */ + private final HashMap<Integer, BinaryType> metas = new HashMap<>(); + + /** + * Create new handler instance. + * + * @return New handler. + */ + public static BinaryCachingMetadataHandler create() { + return new BinaryCachingMetadataHandler(); + } + + /** + * Private constructor. + */ + private BinaryCachingMetadataHandler() { + // No-op. + } + + /** {@inheritDoc} */ + @Override public synchronized void addMeta(int typeId, BinaryType type) throws BinaryObjectException { + synchronized (this) { + BinaryType oldType = metas.put(typeId, type); + + if (oldType != null) { + BinaryMetadata oldMeta = ((BinaryTypeImpl)oldType).metadata(); + BinaryMetadata newMeta = ((BinaryTypeImpl)type).metadata(); + + BinaryMetadata mergedMeta = PortableUtils.mergeMetadata(oldMeta, newMeta); + + BinaryType mergedType = mergedMeta.wrap(((BinaryTypeImpl)oldType).context()); + + metas.put(typeId, mergedType); + } + } + } + + /** {@inheritDoc} */ + @Override public synchronized BinaryType metadata(int typeId) throws BinaryObjectException { + return metas.get(typeId); + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryMetadata.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryMetadata.java b/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryMetadata.java index fe88d11..a464d6e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryMetadata.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryMetadata.java @@ -49,6 +49,9 @@ public class BinaryMetadata implements Externalizable { /** Affinity key field name. */ private String affKeyFieldName; + /** Schemas associated with type. */ + private Collection<PortableSchema> schemas; + /** * For {@link Externalizable}. */ @@ -63,15 +66,17 @@ public class BinaryMetadata implements Externalizable { * @param typeName Type name. * @param fields Fields map. * @param affKeyFieldName Affinity key field name. + * @param schemas Schemas. */ public BinaryMetadata(int typeId, String typeName, @Nullable Map<String, Integer> fields, - @Nullable String affKeyFieldName) { + @Nullable String affKeyFieldName, @Nullable Collection<PortableSchema> schemas) { assert typeName != null; this.typeId = typeId; this.typeName = typeName; this.fields = fields; this.affKeyFieldName = affKeyFieldName; + this.schemas = schemas; } /** @@ -120,6 +125,13 @@ public class BinaryMetadata implements Externalizable { } /** + * @return Schemas. + */ + public Collection<PortableSchema> schemas() { + return schemas != null ? schemas : Collections.<PortableSchema>emptyList(); + } + + /** * Wrap metadata into binary type. * * @param ctx Portable context. @@ -135,6 +147,7 @@ public class BinaryMetadata implements Externalizable { U.writeString(out, typeName); U.writeMap(out, fields); U.writeString(out, affKeyFieldName); + U.writeCollection(out, schemas); } /** {@inheritDoc} */ @@ -143,6 +156,7 @@ public class BinaryMetadata implements Externalizable { typeName = U.readString(in); fields = U.readMap(in); affKeyFieldName = U.readString(in); + schemas = U.readCollection(in); } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryMetadataCollector.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryMetadataCollector.java b/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryMetadataCollector.java index 67e1a0d..28eb1d0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryMetadataCollector.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryMetadataCollector.java @@ -17,6 +17,12 @@ package org.apache.ignite.internal.portable; +import org.apache.ignite.binary.BinaryIdMapper; +import org.apache.ignite.binary.BinaryObjectException; +import org.apache.ignite.binary.BinaryRawWriter; +import org.apache.ignite.binary.BinaryWriter; +import org.jetbrains.annotations.Nullable; + import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.lang.reflect.Proxy; @@ -27,26 +33,37 @@ import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.UUID; -import org.apache.ignite.binary.BinaryObjectException; -import org.apache.ignite.binary.BinaryRawWriter; -import org.apache.ignite.binary.BinaryWriter; -import org.jetbrains.annotations.Nullable; /** * Writer for meta data collection. */ class BinaryMetadataCollector implements BinaryWriter { - /** */ - private final Map<String, Integer> meta = new HashMap<>(); + /** Type ID. */ + private final int typeId; - /** */ + /** Type name. */ private final String typeName; + /** ID mapper. */ + private final BinaryIdMapper idMapper; + + /** Collected metadata. */ + private final Map<String, Integer> meta = new HashMap<>(); + + /** Schema builder. */ + private PortableSchema.Builder schemaBuilder = PortableSchema.Builder.newBuilder(); + /** + * Constructor. + * + * @param typeId Type ID. * @param typeName Type name. + * @param idMapper ID mapper. */ - BinaryMetadataCollector(String typeName) { + BinaryMetadataCollector(int typeId, String typeName, BinaryIdMapper idMapper) { + this.typeId = typeId; this.typeName = typeName; + this.idMapper = idMapper; } /** @@ -56,6 +73,13 @@ class BinaryMetadataCollector implements BinaryWriter { return meta; } + /** + * @return Schemas. + */ + PortableSchema schema() { + return schemaBuilder.build(); + } + /** {@inheritDoc} */ @Override public void writeByte(String fieldName, byte val) throws BinaryObjectException { add(fieldName, PortableClassDescriptor.Mode.BYTE); @@ -242,13 +266,12 @@ class BinaryMetadataCollector implements BinaryWriter { if (oldFieldTypeId != null && !oldFieldTypeId.equals(fieldTypeId)) { throw new BinaryObjectException( - "Field is written twice with different types [" + - "typeName=" + typeName + - ", fieldName=" + name + + "Field is written twice with different types [" + "typeName=" + typeName + ", fieldName=" + name + ", fieldTypeName1=" + PortableUtils.fieldTypeName(oldFieldTypeId) + - ", fieldTypeName2=" + PortableUtils.fieldTypeName(fieldTypeId) + - ']' + ", fieldTypeName2=" + PortableUtils.fieldTypeName(fieldTypeId) + ']' ); } + + schemaBuilder.addField(idMapper.fieldId(typeId, name)); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryObjectEx.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryObjectEx.java b/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryObjectEx.java index b3512ce..6902675 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryObjectEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryObjectEx.java @@ -170,7 +170,7 @@ public abstract class BinaryObjectEx implements BinaryObject { } if (meta == null) - return "PortableObject [hash=" + idHash + ", typeId=" + typeId() + ']'; + return BinaryObject.class.getSimpleName() + " [hash=" + idHash + ", typeId=" + typeId() + ']'; handles.put(this, idHash); http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryObjectImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryObjectImpl.java index d432ea0..d9339f8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryObjectImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryObjectImpl.java @@ -251,7 +251,7 @@ public final class BinaryObjectImpl extends BinaryObjectEx implements Externaliz if (ctx == null) throw new BinaryObjectException("PortableContext is not set for the object."); - return ctx.metaData(typeId()); + return ctx.metadata(typeId()); } /** {@inheritDoc} */ @@ -279,15 +279,17 @@ public final class BinaryObjectImpl extends BinaryObjectEx implements Externaliz int schemaOffset = PortablePrimitives.readInt(arr, start + GridPortableMarshaller.SCHEMA_OR_RAW_OFF_POS); short flags = PortablePrimitives.readShort(arr, start + GridPortableMarshaller.FLAGS_POS); - int fieldOffsetSize = PortableUtils.fieldOffsetSize(flags); - int fieldOffsetPos = start + schemaOffset + order * (4 + fieldOffsetSize) + 4; + int fieldIdLen = PortableUtils.isCompactFooter(flags) ? 0 : PortableUtils.FIELD_ID_LEN; + int fieldOffsetLen = PortableUtils.fieldOffsetLength(flags); + + int fieldOffsetPos = start + schemaOffset + order * (fieldIdLen + fieldOffsetLen) + fieldIdLen; int fieldPos; - if (fieldOffsetSize == PortableUtils.OFFSET_1) + if (fieldOffsetLen == PortableUtils.OFFSET_1) fieldPos = start + ((int)PortablePrimitives.readByte(arr, fieldOffsetPos) & 0xFF); - else if (fieldOffsetSize == PortableUtils.OFFSET_2) + else if (fieldOffsetLen == PortableUtils.OFFSET_2) fieldPos = start + ((int)PortablePrimitives.readShort(arr, fieldOffsetPos) & 0xFFFF); else fieldPos = start + PortablePrimitives.readInt(arr, fieldOffsetPos); @@ -458,7 +460,7 @@ public final class BinaryObjectImpl extends BinaryObjectEx implements Externaliz @Override protected PortableSchema createSchema() { BinaryReaderExImpl reader = new BinaryReaderExImpl(ctx, arr, start, null); - return reader.createSchema(); + return reader.getOrCreateSchema(); } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryObjectOffheapImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryObjectOffheapImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryObjectOffheapImpl.java index f7cb844..a71c98a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryObjectOffheapImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryObjectOffheapImpl.java @@ -136,7 +136,7 @@ public class BinaryObjectOffheapImpl extends BinaryObjectEx implements Externali start, null); - return reader.createSchema(); + return reader.getOrCreateSchema(); } /** {@inheritDoc} */ @@ -164,7 +164,7 @@ public class BinaryObjectOffheapImpl extends BinaryObjectEx implements Externali if (ctx == null) throw new BinaryObjectException("PortableContext is not set for the object."); - return ctx.metaData(typeId()); + return ctx.metadata(typeId()); } /** {@inheritDoc} */ @@ -198,15 +198,17 @@ public class BinaryObjectOffheapImpl extends BinaryObjectEx implements Externali int schemaOffset = PortablePrimitives.readInt(ptr, start + GridPortableMarshaller.SCHEMA_OR_RAW_OFF_POS); short flags = PortablePrimitives.readShort(ptr, start + GridPortableMarshaller.FLAGS_POS); - int fieldOffsetSize = PortableUtils.fieldOffsetSize(flags); - int fieldOffsetPos = start + schemaOffset + order * (4 + fieldOffsetSize) + 4; + int fieldIdLen = PortableUtils.isCompactFooter(flags) ? 0 : PortableUtils.FIELD_ID_LEN; + int fieldOffsetLen = PortableUtils.fieldOffsetLength(flags); + + int fieldOffsetPos = start + schemaOffset + order * (fieldIdLen + fieldOffsetLen) + fieldIdLen; int fieldPos; - if (fieldOffsetSize == PortableUtils.OFFSET_1) + if (fieldOffsetLen == PortableUtils.OFFSET_1) fieldPos = start + ((int)PortablePrimitives.readByte(ptr, fieldOffsetPos) & 0xFF); - else if (fieldOffsetSize == PortableUtils.OFFSET_2) + else if (fieldOffsetLen == PortableUtils.OFFSET_2) fieldPos = start + ((int)PortablePrimitives.readShort(ptr, fieldOffsetPos) & 0xFFFF); else fieldPos = start + PortablePrimitives.readInt(ptr, fieldOffsetPos); http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryReaderExImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryReaderExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryReaderExImpl.java index 669ba01..6ff3047 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryReaderExImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryReaderExImpl.java @@ -17,6 +17,23 @@ package org.apache.ignite.internal.portable; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.binary.BinaryIdMapper; +import org.apache.ignite.binary.BinaryInvalidTypeException; +import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.binary.BinaryObjectException; +import org.apache.ignite.binary.BinaryRawReader; +import org.apache.ignite.binary.BinaryReader; +import org.apache.ignite.internal.portable.streams.PortableHeapInputStream; +import org.apache.ignite.internal.portable.streams.PortableInputStream; +import org.apache.ignite.internal.util.GridEnumCache; +import org.apache.ignite.internal.util.lang.GridMapEntry; +import org.apache.ignite.internal.util.typedef.internal.SB; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiTuple; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + import java.io.ByteArrayInputStream; import java.io.EOFException; import java.io.IOException; @@ -30,7 +47,6 @@ import java.sql.Timestamp; import java.util.ArrayList; import java.util.Collection; import java.util.Date; -import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.Map; import java.util.Properties; @@ -39,22 +55,6 @@ import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentSkipListSet; -import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.binary.BinaryObject; -import org.apache.ignite.binary.BinaryObjectException; -import org.apache.ignite.binary.BinaryIdMapper; -import org.apache.ignite.binary.BinaryInvalidTypeException; -import org.apache.ignite.binary.BinaryRawReader; -import org.apache.ignite.binary.BinaryReader; -import org.apache.ignite.internal.portable.streams.PortableHeapInputStream; -import org.apache.ignite.internal.portable.streams.PortableInputStream; -import org.apache.ignite.internal.util.GridEnumCache; -import org.apache.ignite.internal.util.lang.GridMapEntry; -import org.apache.ignite.internal.util.typedef.internal.SB; -import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.lang.IgniteBiTuple; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.ignite.internal.portable.GridPortableMarshaller.ARR_LIST; @@ -117,9 +117,6 @@ import static org.apache.ignite.internal.portable.GridPortableMarshaller.UUID_AR */ @SuppressWarnings("unchecked") public class BinaryReaderExImpl implements BinaryReader, BinaryRawReaderEx, ObjectInput { - /** Length of a single field descriptor. */ - private static final int FIELD_DESC_LEN = 16; - /** */ private final PortableContext ctx; @@ -162,8 +159,14 @@ public class BinaryReaderExImpl implements BinaryReader, BinaryRawReaderEx, Obje /** Schema Id. */ private int schemaId; + /** Whether this is user type or not. */ + private boolean userType; + + /** Whether field IDs exist. */ + private int fieldIdLen; + /** Offset size in bytes. */ - private int offsetSize; + private int fieldOffsetLen; /** Object schema. */ private PortableSchema schema; @@ -225,18 +228,21 @@ public class BinaryReaderExImpl implements BinaryReader, BinaryRawReaderEx, Obje short flags = in.readShort(); - offsetSize = PortableUtils.fieldOffsetSize(flags); + userType = PortableUtils.isUserType(flags); + + fieldIdLen = PortableUtils.fieldIdLength(flags); + fieldOffsetLen = PortableUtils.fieldOffsetLength(flags); typeId = in.readIntPositioned(start + GridPortableMarshaller.TYPE_ID_POS); - IgniteBiTuple<Integer, Integer> footer = PortableUtils.footerAbsolute(in, start, offsetSize); + IgniteBiTuple<Integer, Integer> footer = PortableUtils.footerAbsolute(in, start); footerStart = footer.get1(); footerLen = footer.get2() - footerStart; schemaId = in.readIntPositioned(start + GridPortableMarshaller.SCHEMA_ID_POS); - rawOff = PortableUtils.rawOffsetAbsolute(in, start, offsetSize); + rawOff = PortableUtils.rawOffsetAbsolute(in, start); if (typeId == UNREGISTERED_TYPE_ID) { // Skip to the class name position. @@ -2555,29 +2561,68 @@ public class BinaryReaderExImpl implements BinaryReader, BinaryRawReaderEx, Obje } /** - * Create schema. + * Get or create object schema. * * @return Schema. */ - public PortableSchema createSchema() { + public PortableSchema getOrCreateSchema() { parseHeaderIfNeeded(); - LinkedHashMap<Integer, Integer> fields = new LinkedHashMap<>(); + PortableSchema schema = ctx.schemaRegistry(typeId).schema(schemaId); + + if (schema == null) { + if (fieldIdLen != PortableUtils.FIELD_ID_LEN) { + BinaryTypeImpl type = (BinaryTypeImpl)ctx.metadata(typeId); + + if (type == null || type.metadata() == null) + throw new BinaryObjectException("Cannot find metadata for object with compact footer: " + + typeId); + + for (PortableSchema typeSchema : type.metadata().schemas()) { + if (schemaId == typeSchema.schemaId()) { + schema = typeSchema; + + break; + } + } + + if (schema == null) + throw new BinaryObjectException("Cannot find schema for object with compact footer [" + + "typeId=" + typeId + ", schemaId=" + schemaId + ']'); + } + else + schema = createSchema(); + + assert schema != null; + + ctx.schemaRegistry(typeId).addSchema(schemaId, schema); + } + + return schema; + } + + /** + * Create schema. + * + * @return Schema. + */ + private PortableSchema createSchema() { + assert fieldIdLen == PortableUtils.FIELD_ID_LEN; + + PortableSchema.Builder builder = PortableSchema.Builder.newBuilder(); int searchPos = footerStart; int searchEnd = searchPos + footerLen; - int idx = 0; - while (searchPos < searchEnd) { int fieldId = in.readIntPositioned(searchPos); - fields.put(fieldId, idx++); + builder.addField(fieldId); - searchPos += 4 + offsetSize; + searchPos += PortableUtils.FIELD_ID_LEN + fieldOffsetLen; } - return new PortableSchema(fields); + return builder.build(); } /** @@ -2593,7 +2638,7 @@ public class BinaryReaderExImpl implements BinaryReader, BinaryRawReaderEx, Obje int searchPos = footerStart; int searchTail = searchPos + footerLen; - if (hasLowFieldsCount(footerLen)) { + if (!userType || (fieldIdLen != 0 && hasLowFieldsCount(footerLen))) { while (true) { if (searchPos >= searchTail) return 0; @@ -2601,37 +2646,32 @@ public class BinaryReaderExImpl implements BinaryReader, BinaryRawReaderEx, Obje int id0 = in.readIntPositioned(searchPos); if (id0 == id) { - int pos = start + PortableUtils.fieldOffsetRelative(in, searchPos + 4, offsetSize); + int pos = start + PortableUtils.fieldOffsetRelative(in, searchPos + PortableUtils.FIELD_ID_LEN, + fieldOffsetLen); in.position(pos); return pos; } - searchPos += 4 + offsetSize; + searchPos += PortableUtils.FIELD_ID_LEN + fieldOffsetLen; } } else { PortableSchema schema0 = schema; if (schema0 == null) { - schema0 = ctx.schemaRegistry(typeId).schema(schemaId); - - if (schema0 == null) { - schema0 = createSchema(); - - ctx.schemaRegistry(typeId).addSchema(schemaId, schema0); - } + schema0 = getOrCreateSchema(); schema = schema0; } - int order = schema.order(id); + int order = schema0.order(id); if (order != PortableSchema.ORDER_NOT_FOUND) { - int offsetPos = footerStart + order * (4 + offsetSize) + 4; + int offsetPos = footerStart + order * (fieldIdLen + fieldOffsetLen) + fieldIdLen; - int pos = start + PortableUtils.fieldOffsetRelative(in, offsetPos, offsetSize); + int pos = start + PortableUtils.fieldOffsetRelative(in, offsetPos, fieldOffsetLen); in.position(pos); @@ -2650,7 +2690,7 @@ public class BinaryReaderExImpl implements BinaryReader, BinaryRawReaderEx, Obje private boolean hasLowFieldsCount(int footerLen) { assert hdrParsed; - return footerLen < (FIELD_DESC_LEN << 4); + return footerLen < ((fieldOffsetLen + fieldIdLen) << 3); } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryTypeImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryTypeImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryTypeImpl.java index 40b6252..60c135d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryTypeImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryTypeImpl.java @@ -62,10 +62,15 @@ public class BinaryTypeImpl implements BinaryType { return ctx.createField(meta.typeId(), fieldName); } - public String affinityKeyFieldName() { + /** {@inheritDoc} */ + @Override public String affinityKeyFieldName() { return meta.affinityKeyFieldName(); } + /** {@inheritDoc} */ + public PortableContext context() { + return ctx; + } /** * @return Metadata. */ http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryWriterExImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryWriterExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryWriterExImpl.java index cedf1c8..6cb18fb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryWriterExImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/portable/BinaryWriterExImpl.java @@ -90,15 +90,9 @@ public class BinaryWriterExImpl implements BinaryWriter, BinaryRawWriterEx, Obje /** Length: integer. */ private static final int LEN_INT = 4; - /** */ + /** Initial capacity. */ private static final int INIT_CAP = 1024; - /** FNV1 hash offset basis. */ - private static final int FNV1_OFFSET_BASIS = 0x811C9DC5; - - /** FNV1 hash prime. */ - private static final int FNV1_PRIME = 0x01000193; - /** Maximum offset which fits in 1 byte. */ private static final int MAX_OFFSET_1 = 1 << 8; @@ -139,7 +133,7 @@ public class BinaryWriterExImpl implements BinaryWriter, BinaryRawWriterEx, Obje private SchemaHolder schema; /** Schema ID. */ - private int schemaId; + private int schemaId = PortableUtils.schemaInitialId(); /** Amount of written fields. */ private int fieldCnt; @@ -332,6 +326,7 @@ public class BinaryWriterExImpl implements BinaryWriter, BinaryRawWriterEx, Obje /** * Perform post-write activity. This includes: + * - writing flags; * - writing object length; * - writing schema offset; * - writing schema to the tail. @@ -339,7 +334,16 @@ public class BinaryWriterExImpl implements BinaryWriter, BinaryRawWriterEx, Obje * @param userType User type flag. */ public void postWrite(boolean userType) { + short flags = userType ? PortableUtils.FLAG_USR_TYP : 0; + + boolean useCompactFooter = ctx.isCompactFooter() && userType; + + if (useCompactFooter) + flags |= PortableUtils.FLAG_COMPACT_FOOTER; + if (schema != null) { + flags |= PortableUtils.FLAG_HAS_SCHEMA; + // Write schema ID. out.writeInt(start + SCHEMA_ID_POS, schemaId); @@ -347,34 +351,35 @@ public class BinaryWriterExImpl implements BinaryWriter, BinaryRawWriterEx, Obje out.writeInt(start + SCHEMA_OR_RAW_OFF_POS, out.position() - start); // Write the schema. - int offsetByteCnt = schema.write(this, fieldCnt); + int offsetByteCnt = schema.write(this, fieldCnt, useCompactFooter); + if (offsetByteCnt == PortableUtils.OFFSET_1) + flags |= PortableUtils.FLAG_OFFSET_ONE_BYTE; + else if (offsetByteCnt == PortableUtils.OFFSET_2) + flags |= PortableUtils.FLAG_OFFSET_TWO_BYTES; + // Write raw offset if needed. - if (rawOffPos != 0) - out.writeInt(rawOffPos - start); - - if (offsetByteCnt == PortableUtils.OFFSET_1) { - int flags = (userType ? PortableUtils.FLAG_USR_TYP : 0) | PortableUtils.FLAG_OFFSET_ONE_BYTE; - - out.writeShort(start + FLAGS_POS, (short)flags); - } - else if (offsetByteCnt == PortableUtils.OFFSET_2) { - int flags = (userType ? PortableUtils.FLAG_USR_TYP : 0) | PortableUtils.FLAG_OFFSET_TWO_BYTES; + if (rawOffPos != 0) { + flags |= PortableUtils.FLAG_HAS_RAW; - out.writeShort(start + FLAGS_POS, (short)flags); + out.writeInt(rawOffPos - start); } } else { - // Write raw-only flag is needed. - int flags = (userType ? PortableUtils.FLAG_USR_TYP : 0) | PortableUtils.FLAG_RAW_ONLY; - - out.writeShort(start + FLAGS_POS, (short)flags); + if (rawOffPos != 0) { + // If there are no schema, we are free to write raw offset to schema offset. + flags |= PortableUtils.FLAG_HAS_RAW; - // If there are no schema, we are free to write raw offset to schema offset. - out.writeInt(start + SCHEMA_OR_RAW_OFF_POS, (rawOffPos == 0 ? out.position() : rawOffPos) - start); + out.writeInt(start + SCHEMA_OR_RAW_OFF_POS, rawOffPos - start); + } + else + out.writeInt(start + SCHEMA_OR_RAW_OFF_POS, 0); } - // 5. Write length. + // Write flags. + out.writeShort(start + FLAGS_POS, flags); + + // Write length. out.writeInt(start + TOTAL_LEN_POS, out.position() - start); } @@ -1737,22 +1742,9 @@ public class BinaryWriterExImpl implements BinaryWriter, BinaryRawWriterEx, Obje SCHEMA.set(schema); } - - // Initialize offset when the first field is written. - schemaId = FNV1_OFFSET_BASIS; } - // Advance schema hash. - int schemaId0 = schemaId ^ (fieldId & 0xFF); - schemaId0 = schemaId0 * FNV1_PRIME; - schemaId0 = schemaId0 ^ ((fieldId >> 8) & 0xFF); - schemaId0 = schemaId0 * FNV1_PRIME; - schemaId0 = schemaId0 ^ ((fieldId >> 16) & 0xFF); - schemaId0 = schemaId0 * FNV1_PRIME; - schemaId0 = schemaId0 ^ ((fieldId >> 24) & 0xFF); - schemaId0 = schemaId0 * FNV1_PRIME; - - schemaId = schemaId0; + schemaId = PortableUtils.updateSchemaId(schemaId, fieldId); schema.push(fieldId, fieldOff); @@ -1760,6 +1752,25 @@ public class BinaryWriterExImpl implements BinaryWriter, BinaryRawWriterEx, Obje } /** + * @return Current schema ID. + */ + public int schemaId() { + return schemaId; + } + + /** + * @return Current writer's schema. + */ + public PortableSchema currentSchema() { + PortableSchema.Builder builder = PortableSchema.Builder.newBuilder(); + + if (schema != null) + schema.build(builder, fieldCnt); + + return builder.build(); + } + + /** * Attempts to write the object as a handle. * * @param obj Object to write. @@ -1844,13 +1855,25 @@ public class BinaryWriterExImpl implements BinaryWriter, BinaryRawWriterEx, Obje } /** + * Build the schema. + * + * @param builder Builder. + * @param fieldCnt Fields count. + */ + public void build(PortableSchema.Builder builder, int fieldCnt) { + for (int curIdx = idx - fieldCnt * 2; curIdx < idx; curIdx += 2) + builder.addField(data[curIdx]); + } + + /** * Write collected frames and pop them. * * @param writer Writer. * @param fieldCnt Count. - * @return Amount of bytes dedicated to + * @param compactFooter Whether footer should be written in compact form. + * @return Amount of bytes dedicated to each field offset. Could be 1, 2 or 4. */ - public int write(BinaryWriterExImpl writer, int fieldCnt) { + public int write(BinaryWriterExImpl writer, int fieldCnt, boolean compactFooter) { int startIdx = idx - fieldCnt * 2; assert startIdx >= 0; @@ -1859,29 +1882,51 @@ public class BinaryWriterExImpl implements BinaryWriter, BinaryRawWriterEx, Obje int res; - if (lastOffset < MAX_OFFSET_1) { - for (int idx0 = startIdx; idx0 < idx; ) { - writer.writeInt(data[idx0++]); - writer.writeByte((byte) data[idx0++]); + if (compactFooter) { + if (lastOffset < MAX_OFFSET_1) { + for (int curIdx = startIdx + 1; curIdx < idx; curIdx += 2) + writer.writeByte((byte)data[curIdx]); + + res = PortableUtils.OFFSET_1; } + else if (lastOffset < MAX_OFFSET_2) { + for (int curIdx = startIdx + 1; curIdx < idx; curIdx += 2) + writer.writeShort((short)data[curIdx]); - res = PortableUtils.OFFSET_1; - } - else if (lastOffset < MAX_OFFSET_2) { - for (int idx0 = startIdx; idx0 < idx; ) { - writer.writeInt(data[idx0++]); - writer.writeShort((short)data[idx0++]); + res = PortableUtils.OFFSET_2; } + else { + for (int curIdx = startIdx + 1; curIdx < idx; curIdx += 2) + writer.writeInt(data[curIdx]); - res = PortableUtils.OFFSET_2; + res = PortableUtils.OFFSET_4; + } } else { - for (int idx0 = startIdx; idx0 < idx; ) { - writer.writeInt(data[idx0++]); - writer.writeInt(data[idx0++]); + if (lastOffset < MAX_OFFSET_1) { + for (int curIdx = startIdx; curIdx < idx;) { + writer.writeInt(data[curIdx++]); + writer.writeByte((byte) data[curIdx++]); + } + + res = PortableUtils.OFFSET_1; + } + else if (lastOffset < MAX_OFFSET_2) { + for (int curIdx = startIdx; curIdx < idx;) { + writer.writeInt(data[curIdx++]); + writer.writeShort((short)data[curIdx++]); + } + + res = PortableUtils.OFFSET_2; } + else { + for (int curIdx = startIdx; curIdx < idx;) { + writer.writeInt(data[curIdx++]); + writer.writeInt(data[curIdx++]); + } - res = PortableUtils.OFFSET_4; + res = PortableUtils.OFFSET_4; + } } return res; http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableClassDescriptor.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableClassDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableClassDescriptor.java index 225e0ba..8543ce6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableClassDescriptor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableClassDescriptor.java @@ -38,6 +38,7 @@ import java.math.BigDecimal; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -79,6 +80,9 @@ public class PortableClassDescriptor { /** */ private final String typeName; + /** Affinity key field name. */ + private final String affKeyFieldName; + /** */ private final Constructor<?> ctor; @@ -92,7 +96,13 @@ public class PortableClassDescriptor { private final Method readResolveMtd; /** */ - private final Map<String, Integer> fieldsMeta; + private final Map<String, Integer> stableFieldsMeta; + + /** Object schemas. Initialized only for serializable classes and contains only 1 entry. */ + private final Collection<PortableSchema> stableSchemas; + + /** Schema registry. */ + private final PortableSchemaRegistry schemaReg; /** */ private final boolean keepDeserialized; @@ -112,13 +122,14 @@ public class PortableClassDescriptor { * @param userType User type flag. * @param typeId Type ID. * @param typeName Type name. + * @param affKeyFieldName Affinity key field name. * @param idMapper ID mapper. * @param serializer Serializer. * @param metaDataEnabled Metadata enabled flag. * @param keepDeserialized Keep deserialized flag. * @param registered Whether typeId has been successfully registered by MarshallerContext or not. * @param predefined Whether the class is predefined or not. - * @throws org.apache.ignite.binary.BinaryObjectException In case of error. + * @throws BinaryObjectException In case of error. */ PortableClassDescriptor( PortableContext ctx, @@ -126,6 +137,7 @@ public class PortableClassDescriptor { boolean userType, int typeId, String typeName, + @Nullable String affKeyFieldName, @Nullable BinaryIdMapper idMapper, @Nullable BinarySerializer serializer, boolean metaDataEnabled, @@ -135,17 +147,21 @@ public class PortableClassDescriptor { ) throws BinaryObjectException { assert ctx != null; assert cls != null; + assert idMapper != null; this.ctx = ctx; this.cls = cls; - this.userType = userType; this.typeId = typeId; + this.userType = userType; this.typeName = typeName; + this.affKeyFieldName = affKeyFieldName; this.serializer = serializer; this.idMapper = idMapper; this.keepDeserialized = keepDeserialized; this.registered = registered; + schemaReg = ctx.schemaRegistry(typeId); + excluded = MarshallerExclusions.isExcluded(cls); useOptMarshaller = !predefined && initUseOptimizedMarshallerFlag(); @@ -193,7 +209,8 @@ public class PortableClassDescriptor { case EXCLUSION: ctor = null; fields = null; - fieldsMeta = null; + stableFieldsMeta = null; + stableSchemas = null; break; @@ -201,16 +218,17 @@ public class PortableClassDescriptor { case EXTERNALIZABLE: ctor = constructor(cls); fields = null; - fieldsMeta = null; + stableFieldsMeta = null; + stableSchemas = null; break; case OBJECT: - assert idMapper != null; - ctor = constructor(cls); fields = new ArrayList<>(); - fieldsMeta = metaDataEnabled ? new HashMap<String, Integer>() : null; + stableFieldsMeta = metaDataEnabled ? new HashMap<String, Integer>() : null; + + PortableSchema.Builder schemaBuilder = PortableSchema.Builder.newBuilder(); Collection<String> names = new HashSet<>(); Collection<Integer> ids = new HashSet<>(); @@ -236,12 +254,16 @@ public class PortableClassDescriptor { fields.add(fieldInfo); + schemaBuilder.addField(fieldId); + if (metaDataEnabled) - fieldsMeta.put(name, fieldInfo.fieldMode().typeId()); + stableFieldsMeta.put(name, fieldInfo.fieldMode().typeId()); } } } + stableSchemas = Collections.singleton(schemaBuilder.build()); + break; default: @@ -284,7 +306,14 @@ public class PortableClassDescriptor { * @return Fields meta data. */ Map<String, Integer> fieldsMeta() { - return fieldsMeta; + return stableFieldsMeta; + } + + /** + * @return Schemas. + */ + Collection<PortableSchema> schemas() { + return stableSchemas; } /** @@ -345,7 +374,7 @@ public class PortableClassDescriptor { /** * @param obj Object. * @param writer Writer. - * @throws org.apache.ignite.binary.BinaryObjectException In case of error. + * @throws BinaryObjectException In case of error. */ void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException { assert obj != null; @@ -539,21 +568,34 @@ public class PortableClassDescriptor { ((Binarylizable)obj).writeBinary(writer); writer.postWrite(userType); - } - finally { - writer.popSchema(); - } - if (obj.getClass() != BinaryMetadata.class - && ctx.isMetaDataChanged(typeId, writer.metaDataHashSum())) { - BinaryMetadataCollector metaCollector = new BinaryMetadataCollector(typeName); + // Check whether we need to update metadata. + if (obj.getClass() != BinaryMetadata.class) { + int schemaId = writer.schemaId(); - if (serializer != null) - serializer.writeBinary(obj, metaCollector); - else - ((Binarylizable)obj).writeBinary(metaCollector); + if (schemaReg.schema(schemaId) == null) { + // This is new schema, let's update metadata. + BinaryMetadataCollector collector = + new BinaryMetadataCollector(typeId, typeName, idMapper); + + if (serializer != null) + serializer.writeBinary(obj, collector); + else + ((Binarylizable)obj).writeBinary(collector); + + PortableSchema newSchema = collector.schema(); - ctx.updateMetaData(typeId, typeName, metaCollector.meta()); + BinaryMetadata meta = new BinaryMetadata(typeId, typeName, collector.meta(), + affKeyFieldName, Collections.singleton(newSchema)); + + ctx.updateMetadata(typeId, meta); + + schemaReg.addSchema(newSchema.schemaId(), newSchema); + } + } + } + finally { + writer.popSchema(); } } @@ -601,7 +643,7 @@ public class PortableClassDescriptor { /** * @param reader Reader. * @return Object. - * @throws org.apache.ignite.binary.BinaryObjectException If failed. + * @throws BinaryObjectException If failed. */ Object read(BinaryReaderExImpl reader) throws BinaryObjectException { assert reader != null; @@ -683,7 +725,6 @@ public class PortableClassDescriptor { PortableUtils.writeHeader( writer, - userType, registered ? typeId : GridPortableMarshaller.UNREGISTERED_TYPE_ID, obj instanceof CacheObjectImpl ? 0 : obj.hashCode(), registered ? null : cls.getName() @@ -694,7 +735,7 @@ public class PortableClassDescriptor { /** * @return Instance. - * @throws org.apache.ignite.binary.BinaryObjectException In case of error. + * @throws BinaryObjectException In case of error. */ private Object newInstance() throws BinaryObjectException { assert ctor != null; @@ -710,7 +751,7 @@ public class PortableClassDescriptor { /** * @param cls Class. * @return Constructor. - * @throws org.apache.ignite.binary.BinaryObjectException If constructor doesn't exist. + * @throws BinaryObjectException If constructor doesn't exist. */ @SuppressWarnings("ConstantConditions") @Nullable private static Constructor<?> constructor(Class<?> cls) throws BinaryObjectException { @@ -719,6 +760,9 @@ public class PortableClassDescriptor { try { Constructor<?> ctor = U.forceEmptyConstructor(cls); + if (ctor == null) + throw new BinaryObjectException("Failed to find empty constructor for class: " + cls.getName()); + ctor.setAccessible(true); return ctor; @@ -871,7 +915,7 @@ public class PortableClassDescriptor { /** * @param obj Object. * @param writer Writer. - * @throws org.apache.ignite.binary.BinaryObjectException In case of error. + * @throws BinaryObjectException In case of error. */ public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException { assert obj != null; @@ -1074,7 +1118,7 @@ public class PortableClassDescriptor { /** * @param obj Object. * @param reader Reader. - * @throws org.apache.ignite.binary.BinaryObjectException In case of error. + * @throws BinaryObjectException In case of error. */ public void read(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException { Object val = null; http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableContext.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableContext.java b/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableContext.java index 86578ad..afc23e1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableContext.java @@ -61,7 +61,6 @@ import org.apache.ignite.binary.BinarySerializer; import org.apache.ignite.internal.IgniteKernal; import org.apache.ignite.internal.IgnitionEx; import org.apache.ignite.internal.processors.cache.portable.CacheObjectBinaryProcessorImpl; -import org.apache.ignite.internal.util.GridConcurrentHashSet; import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.internal.util.lang.GridMapEntry; import org.apache.ignite.internal.util.typedef.F; @@ -107,9 +106,6 @@ public class PortableContext implements Externalizable { } /** */ - private final ConcurrentMap<Integer, Collection<Integer>> metaDataCache = new ConcurrentHashMap8<>(); - - /** */ private final ConcurrentMap<Class<?>, PortableClassDescriptor> descByCls = new ConcurrentHashMap8<>(); /** Holds classes loaded by default class loader only. */ @@ -130,6 +126,9 @@ public class PortableContext implements Externalizable { /** */ private final ConcurrentMap<Integer, BinaryIdMapper> mappers = new ConcurrentHashMap8<>(0); + /** Affinity key field names. */ + private final ConcurrentMap<Integer, String> affKeyFieldNames = new ConcurrentHashMap8<>(0); + /** */ private final Map<String, BinaryIdMapper> typeMappers = new ConcurrentHashMap8<>(0); @@ -151,6 +150,9 @@ public class PortableContext implements Externalizable { /** */ private boolean keepDeserialized; + /** Compact footer flag. */ + private boolean compactFooter; + /** Object schemas. */ private volatile Map<Integer, PortableSchemaRegistry> schemas; @@ -262,6 +264,8 @@ public class PortableContext implements Externalizable { marsh.getClassNames(), marsh.getTypeConfigurations() ); + + compactFooter = marsh.isCompactFooter(); } /** @@ -504,6 +508,7 @@ public class PortableContext implements Externalizable { false, clsName.hashCode(), clsName, + null, BASIC_CLS_ID_MAPPER, null, false, @@ -550,6 +555,7 @@ public class PortableContext implements Externalizable { true, typeId, typeName, + null, idMapper, null, true, @@ -567,7 +573,8 @@ public class PortableContext implements Externalizable { mappers.putIfAbsent(typeId, idMapper); - metaHnd.addMeta(typeId, new BinaryMetadata(typeId, typeName, desc.fieldsMeta(), null).wrap(this)); + metaHnd.addMeta(typeId, + new BinaryMetadata(typeId, typeName, desc.fieldsMeta(), null, desc.schemas()).wrap(this)); return desc; } @@ -694,6 +701,7 @@ public class PortableContext implements Externalizable { false, id, typeName, + null, DFLT_ID_MAPPER, null, false, @@ -745,11 +753,17 @@ public class PortableContext implements Externalizable { if (mappers.put(id, idMapper) != null) throw new BinaryObjectException("Duplicate type ID [clsName=" + clsName + ", id=" + id + ']'); + if (affKeyFieldName != null) { + if (affKeyFieldNames.put(id, affKeyFieldName) != null) + throw new BinaryObjectException("Duplicate type ID [clsName=" + clsName + ", id=" + id + ']'); + } + String typeName = typeName(clsName); typeMappers.put(typeName, idMapper); Map<String, Integer> fieldsMeta = null; + Collection<PortableSchema> schemas = null; if (cls != null) { PortableClassDescriptor desc = new PortableClassDescriptor( @@ -758,6 +772,7 @@ public class PortableContext implements Externalizable { true, id, typeName, + affKeyFieldName, idMapper, serializer, true, @@ -767,6 +782,7 @@ public class PortableContext implements Externalizable { ); fieldsMeta = desc.fieldsMeta(); + schemas = desc.schemas(); if (IgniteUtils.detectClassLoader(cls).equals(dfltLdr)) userTypes.put(id, desc); @@ -774,7 +790,7 @@ public class PortableContext implements Externalizable { descByCls.put(cls, desc); } - metaHnd.addMeta(id, new BinaryMetadata(id, typeName, fieldsMeta, affKeyFieldName).wrap(this)); + metaHnd.addMeta(id, new BinaryMetadata(id, typeName, fieldsMeta, affKeyFieldName, schemas).wrap(this)); } /** @@ -797,48 +813,32 @@ public class PortableContext implements Externalizable { * @return Meta data. * @throws org.apache.ignite.binary.BinaryObjectException In case of error. */ - @Nullable public BinaryType metaData(int typeId) throws BinaryObjectException { + @Nullable public BinaryType metadata(int typeId) throws BinaryObjectException { return metaHnd != null ? metaHnd.metadata(typeId) : null; } /** * @param typeId Type ID. - * @param metaHashSum Meta data hash sum. - * @return Whether meta is changed. + * @return Affinity key field name. */ - boolean isMetaDataChanged(int typeId, @Nullable Integer metaHashSum) { - if (metaHashSum == null) - return false; - - Collection<Integer> hist = metaDataCache.get(typeId); - - if (hist == null) { - Collection<Integer> old = metaDataCache.putIfAbsent(typeId, hist = new GridConcurrentHashSet<>()); - - if (old != null) - hist = old; - } - - return hist.add(metaHashSum); + public String affinityKeyFieldName(int typeId) { + return affKeyFieldNames.get(typeId); } /** * @param typeId Type ID. - * @param typeName Type name. - * @param fields Fields map. - * @throws org.apache.ignite.binary.BinaryObjectException In case of error. + * @param meta Meta data. + * @throws BinaryObjectException In case of error. */ - public void updateMetaData(int typeId, String typeName, Map<String, Integer> fields) throws BinaryObjectException { - updateMetaData(typeId, new BinaryMetadata(typeId, typeName, fields, null)); + public void updateMetadata(int typeId, BinaryMetadata meta) throws BinaryObjectException { + metaHnd.addMeta(typeId, meta.wrap(this)); } /** - * @param typeId Type ID. - * @param meta Meta data. - * @throws org.apache.ignite.binary.BinaryObjectException In case of error. + * @return Whether field IDs should be skipped in footer or not. */ - public void updateMetaData(int typeId, BinaryMetadata meta) throws BinaryObjectException { - metaHnd.addMeta(typeId, meta.wrap(this)); + public boolean isCompactFooter() { + return compactFooter; } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableSchema.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableSchema.java b/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableSchema.java index 96a93f4..86ca5f8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableSchema.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableSchema.java @@ -17,10 +17,16 @@ package org.apache.ignite.internal.portable; +import org.apache.ignite.internal.util.typedef.internal.U; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.Map; +import java.util.List; /** * Schema describing portable object content. We rely on the following assumptions: @@ -28,130 +34,150 @@ import java.util.Map; * for quick comparisons performed within already fetched L1 cache line. * - When there are more fields, we store them inside a hash map. */ -public class PortableSchema { +public class PortableSchema implements Externalizable { + /** */ + private static final long serialVersionUID = 0L; + /** Order returned if field is not found. */ public static final int ORDER_NOT_FOUND = -1; /** Inline flag. */ - private final boolean inline; + private boolean inline; + + /** Map with ID to order. */ + private HashMap<Integer, Integer> idToOrder; - /** Map with offsets. */ - private final HashMap<Integer, Integer> map; + /** IDs depending on order. */ + private ArrayList<Integer> ids; /** ID 1. */ - private final int id0; + private int id0; /** ID 2. */ - private final int id1; + private int id1; /** ID 3. */ - private final int id2; + private int id2; /** ID 4. */ - private final int id3; + private int id3; /** ID 1. */ - private final int id4; + private int id4; /** ID 2. */ - private final int id5; + private int id5; /** ID 3. */ - private final int id6; + private int id6; /** ID 4. */ - private final int id7; + private int id7; + + /** Schema ID. */ + private int schemaId; + + /** + * {@link Externalizable} support. + */ + public PortableSchema() { + // No-op. + } /** * Constructor. * - * @param vals Values. + * @param schemaId Schema ID. + * @param fieldIds Field IDs. */ - public PortableSchema(LinkedHashMap<Integer, Integer> vals) { - if (vals.size() <= 8) { - inline = true; - - Iterator<Map.Entry<Integer, Integer>> iter = vals.entrySet().iterator(); + private PortableSchema(int schemaId, List<Integer> fieldIds) { + this.schemaId = schemaId; - Map.Entry<Integer, Integer> entry = iter.hasNext() ? iter.next() : null; - - if (entry != null) { - id0 = entry.getKey(); + if (fieldIds.size() <= 8) { + inline = true; - assert entry.getValue() == 0; - } - else - id0 = 0; + Iterator<Integer> iter = fieldIds.iterator(); - if ((entry = iter.hasNext() ? iter.next() : null) != null) { - id1 = entry.getKey(); + id0 = iter.hasNext() ? iter.next() : 0; + id1 = iter.hasNext() ? iter.next() : 0; + id2 = iter.hasNext() ? iter.next() : 0; + id3 = iter.hasNext() ? iter.next() : 0; + id4 = iter.hasNext() ? iter.next() : 0; + id5 = iter.hasNext() ? iter.next() : 0; + id6 = iter.hasNext() ? iter.next() : 0; + id7 = iter.hasNext() ? iter.next() : 0; - assert entry.getValue() == 1; - } - else - id1 = 0; + idToOrder = null; + } + else { + inline = false; - if ((entry = iter.hasNext() ? iter.next() : null) != null) { - id2 = entry.getKey(); + id0 = id1 = id2 = id3 = id4 = id5 = id6 = id7 = 0; - assert entry.getValue() == 2; - } - else - id2 = 0; + ids = new ArrayList<>(); + idToOrder = new HashMap<>(); - if ((entry = iter.hasNext() ? iter.next() : null) != null) { - id3 = entry.getKey(); + for (int i = 0; i < fieldIds.size(); i++) { + int fieldId = fieldIds.get(i); - assert entry.getValue() == 3; + ids.add(fieldId); + idToOrder.put(fieldId, i); } - else - id3 = 0; + } + } - if ((entry = iter.hasNext() ? iter.next() : null) != null) { - id4 = entry.getKey(); + /** + * @return Schema ID. + */ + public int schemaId() { + return schemaId; + } - assert entry.getValue() == 4; - } - else - id4 = 0; + /** + * Get field ID by order in footer. + * + * @param order Order. + * @return Field ID. + */ + public int fieldId(int order) { + if (inline) { + switch (order) { + case 0: + return id0; - if ((entry = iter.hasNext() ? iter.next() : null) != null) { - id5 = entry.getKey(); + case 1: + return id1; - assert entry.getValue() == 5; - } - else - id5 = 0; + case 2: + return id2; - if ((entry = iter.hasNext() ? iter.next() : null) != null) { - id6 = entry.getKey(); + case 3: + return id3; - assert entry.getValue() == 6; - } - else - id6 = 0; + case 4: + return id4; - if ((entry = iter.hasNext() ? iter.next() : null) != null) { - id7 = entry.getKey(); + case 5: + return id5; - assert entry.getValue() == 7; - } - else - id7 = 0; + case 6: + return id6; - map = null; - } - else { - inline = false; + case 7: + return id7; - id0 = id1 = id2 = id3 = id4 = id5 = id6 = id7 = 0; + default: + assert false : "Should not reach here."; - map = new HashMap<>(vals); + return 0; + } } + else + return ids.get(order); } /** - * Get field position in footer by schema ID. + * Get field order in footer by field ID. * * @param id Field ID. * @return Offset or {@code 0} if there is no such field. @@ -185,9 +211,125 @@ public class PortableSchema { return ORDER_NOT_FOUND; } else { - Integer order = map.get(id); + Integer order = idToOrder.get(id); return order != null ? order : ORDER_NOT_FOUND; } } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return schemaId; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + return o != null && o instanceof PortableSchema && schemaId == ((PortableSchema)o).schemaId; + } + + /** {@inheritDoc} */ + @Override public void writeExternal(ObjectOutput out) throws IOException { + out.writeInt(schemaId); + + if (inline) { + out.writeBoolean(true); + + out.writeInt(id0); + out.writeInt(id1); + out.writeInt(id2); + out.writeInt(id3); + out.writeInt(id4); + out.writeInt(id5); + out.writeInt(id6); + out.writeInt(id7); + } + else { + out.writeBoolean(false); + + out.writeInt(ids.size()); + + for (Integer id : ids) + out.writeInt(id); + } + } + + /** {@inheritDoc} */ + @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + schemaId = in.readInt(); + + if (in.readBoolean()) { + inline = true; + + id0 = in.readInt(); + id1 = in.readInt(); + id2 = in.readInt(); + id3 = in.readInt(); + id4 = in.readInt(); + id5 = in.readInt(); + id6 = in.readInt(); + id7 = in.readInt(); + } + else { + inline = false; + + int size = in.readInt(); + + ids = new ArrayList<>(size); + idToOrder = U.newHashMap(size); + + for (int i = 0; i < size; i++) { + int fieldId = in.readInt(); + + ids.add(fieldId); + idToOrder.put(fieldId, i); + } + } + } + + /** + * Schema builder. + */ + public static class Builder { + /** Schema ID. */ + private int schemaId = PortableUtils.schemaInitialId(); + + /** Fields. */ + private final ArrayList<Integer> fields = new ArrayList<>(); + + /** + * Create new schema builder. + * + * @return Schema builder. + */ + public static Builder newBuilder() { + return new Builder(); + } + + /** + * Private constructor. + */ + private Builder() { + // No-op. + } + + /** + * Add field. + * + * @param fieldId Field ID. + */ + public void addField(int fieldId) { + fields.add(fieldId); + + schemaId = PortableUtils.updateSchemaId(schemaId, fieldId); + } + + /** + * Build schema. + * + * @return Schema. + */ + public PortableSchema build() { + return new PortableSchema(schemaId, fields); + } + } }