http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableUtils.java index 31f2bf9..95ef9591 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/portable/PortableUtils.java @@ -98,16 +98,22 @@ public class PortableUtils { private static final Collection<Class<?>> PORTABLE_CLS = new HashSet<>(); /** Flag: user type. */ - public static final short FLAG_USR_TYP = 0x1; + public static final short FLAG_USR_TYP = 0x0001; /** Flag: only raw data exists. */ - public static final short FLAG_RAW_ONLY = 0x2; + public static final short FLAG_HAS_SCHEMA = 0x0002; + + /** Flag indicating that object has raw data. */ + public static final short FLAG_HAS_RAW = 0x0004; /** Flag: offsets take 1 byte. */ - public static final short FLAG_OFFSET_ONE_BYTE = 0x4; + public static final short FLAG_OFFSET_ONE_BYTE = 0x0008; /** Flag: offsets take 2 bytes. */ - public static final short FLAG_OFFSET_TWO_BYTES = 0x8; + public static final short FLAG_OFFSET_TWO_BYTES = 0x0010; + + /** Flag: compact footer, no field IDs. */ + public static final short FLAG_COMPACT_FOOTER = 0x0020; /** Offset which fits into 1 byte. */ public static final int OFFSET_1 = 1; @@ -118,10 +124,99 @@ public class PortableUtils { /** Offset which fits into 4 bytes. */ public static final int OFFSET_4 = 4; + /** Field ID length. */ + public static final int FIELD_ID_LEN = 4; + /** Field type names. */ private static final String[] FIELD_TYPE_NAMES; + /** FNV1 hash offset basis. */ + private static final int FNV1_OFFSET_BASIS = 0x811C9DC5; + + /** FNV1 hash prime. */ + private static final int FNV1_PRIME = 0x01000193; + + /** + * Static class initializer. + */ static { + PLAIN_CLASS_TO_FLAG.put(Byte.class, GridPortableMarshaller.BYTE); + PLAIN_CLASS_TO_FLAG.put(Short.class, GridPortableMarshaller.SHORT); + PLAIN_CLASS_TO_FLAG.put(Integer.class, GridPortableMarshaller.INT); + PLAIN_CLASS_TO_FLAG.put(Long.class, GridPortableMarshaller.LONG); + PLAIN_CLASS_TO_FLAG.put(Float.class, GridPortableMarshaller.FLOAT); + PLAIN_CLASS_TO_FLAG.put(Double.class, GridPortableMarshaller.DOUBLE); + PLAIN_CLASS_TO_FLAG.put(Character.class, GridPortableMarshaller.CHAR); + PLAIN_CLASS_TO_FLAG.put(Boolean.class, GridPortableMarshaller.BOOLEAN); + PLAIN_CLASS_TO_FLAG.put(BigDecimal.class, GridPortableMarshaller.DECIMAL); + PLAIN_CLASS_TO_FLAG.put(String.class, GridPortableMarshaller.STRING); + PLAIN_CLASS_TO_FLAG.put(UUID.class, GridPortableMarshaller.UUID); + PLAIN_CLASS_TO_FLAG.put(Date.class, GridPortableMarshaller.DATE); + PLAIN_CLASS_TO_FLAG.put(Timestamp.class, GridPortableMarshaller.TIMESTAMP); + + PLAIN_CLASS_TO_FLAG.put(byte[].class, GridPortableMarshaller.BYTE_ARR); + PLAIN_CLASS_TO_FLAG.put(short[].class, GridPortableMarshaller.SHORT_ARR); + PLAIN_CLASS_TO_FLAG.put(int[].class, GridPortableMarshaller.INT_ARR); + PLAIN_CLASS_TO_FLAG.put(long[].class, GridPortableMarshaller.LONG_ARR); + PLAIN_CLASS_TO_FLAG.put(float[].class, GridPortableMarshaller.FLOAT_ARR); + PLAIN_CLASS_TO_FLAG.put(double[].class, GridPortableMarshaller.DOUBLE_ARR); + PLAIN_CLASS_TO_FLAG.put(char[].class, GridPortableMarshaller.CHAR_ARR); + PLAIN_CLASS_TO_FLAG.put(boolean[].class, GridPortableMarshaller.BOOLEAN_ARR); + PLAIN_CLASS_TO_FLAG.put(BigDecimal[].class, GridPortableMarshaller.DECIMAL_ARR); + PLAIN_CLASS_TO_FLAG.put(String[].class, GridPortableMarshaller.STRING_ARR); + PLAIN_CLASS_TO_FLAG.put(UUID[].class, GridPortableMarshaller.UUID_ARR); + PLAIN_CLASS_TO_FLAG.put(Date[].class, GridPortableMarshaller.DATE_ARR); + PLAIN_CLASS_TO_FLAG.put(Timestamp[].class, GridPortableMarshaller.TIMESTAMP_ARR); + + for (Map.Entry<Class<?>, Byte> entry : PLAIN_CLASS_TO_FLAG.entrySet()) + FLAG_TO_CLASS.put(entry.getValue(), entry.getKey()); + + PLAIN_CLASS_TO_FLAG.put(byte.class, GridPortableMarshaller.BYTE); + PLAIN_CLASS_TO_FLAG.put(short.class, GridPortableMarshaller.SHORT); + PLAIN_CLASS_TO_FLAG.put(int.class, GridPortableMarshaller.INT); + PLAIN_CLASS_TO_FLAG.put(long.class, GridPortableMarshaller.LONG); + PLAIN_CLASS_TO_FLAG.put(float.class, GridPortableMarshaller.FLOAT); + PLAIN_CLASS_TO_FLAG.put(double.class, GridPortableMarshaller.DOUBLE); + PLAIN_CLASS_TO_FLAG.put(char.class, GridPortableMarshaller.CHAR); + PLAIN_CLASS_TO_FLAG.put(boolean.class, GridPortableMarshaller.BOOLEAN); + + for (byte b : new byte[] { + BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, + CHAR, BOOLEAN, DECIMAL, STRING, UUID, DATE, TIMESTAMP, + BYTE_ARR, SHORT_ARR, INT_ARR, LONG_ARR, FLOAT_ARR, DOUBLE_ARR, + CHAR_ARR, BOOLEAN_ARR, DECIMAL_ARR, STRING_ARR, UUID_ARR, DATE_ARR, TIMESTAMP_ARR, + ENUM, ENUM_ARR, NULL}) { + + PLAIN_TYPE_FLAG[b] = true; + } + + PORTABLE_CLS.add(Byte.class); + PORTABLE_CLS.add(Short.class); + PORTABLE_CLS.add(Integer.class); + PORTABLE_CLS.add(Long.class); + PORTABLE_CLS.add(Float.class); + PORTABLE_CLS.add(Double.class); + PORTABLE_CLS.add(Character.class); + PORTABLE_CLS.add(Boolean.class); + PORTABLE_CLS.add(String.class); + PORTABLE_CLS.add(UUID.class); + PORTABLE_CLS.add(Date.class); + PORTABLE_CLS.add(Timestamp.class); + PORTABLE_CLS.add(BigDecimal.class); + PORTABLE_CLS.add(byte[].class); + PORTABLE_CLS.add(short[].class); + PORTABLE_CLS.add(int[].class); + PORTABLE_CLS.add(long[].class); + PORTABLE_CLS.add(float[].class); + PORTABLE_CLS.add(double[].class); + PORTABLE_CLS.add(char[].class); + PORTABLE_CLS.add(boolean[].class); + PORTABLE_CLS.add(String[].class); + PORTABLE_CLS.add(UUID[].class); + PORTABLE_CLS.add(Date[].class); + PORTABLE_CLS.add(Timestamp[].class); + PORTABLE_CLS.add(BigDecimal[].class); + FIELD_TYPE_NAMES = new String[104]; FIELD_TYPE_NAMES[BYTE] = "byte"; @@ -162,168 +257,113 @@ public class PortableUtils { } /** - * @param typeName Field type name. - * @return Field type ID; + * Check if user type flag is set. + * + * @param flags Flags. + * @return {@code True} if set. */ - @SuppressWarnings("StringEquality") - public static int fieldTypeId(String typeName) { - for (int i = 0; i < FIELD_TYPE_NAMES.length; i++) { - String typeName0 = FIELD_TYPE_NAMES[i]; - - if (typeName.equals(typeName0)) - return i; - } - - throw new IllegalArgumentException("Invalid metadata type name: " + typeName); + public static boolean isUserType(short flags) { + return isFlagSet(flags, FLAG_USR_TYP); } /** - * @param typeId Field type ID. - * @return Field type name. + * Check if raw-only flag is set. + * + * @param flags Flags. + * @return {@code True} if set. */ - public static String fieldTypeName(int typeId) { - assert typeId >= 0 && typeId < FIELD_TYPE_NAMES.length : typeId; - - String typeName = FIELD_TYPE_NAMES[typeId]; - - assert typeName != null : typeId; - - return typeName; + public static boolean hasSchema(short flags) { + return isFlagSet(flags, FLAG_HAS_SCHEMA); } /** - * @param typeIds Field type IDs. - * @return Field type names. + * Check if raw-only flag is set. + * + * @param flags Flags. + * @return {@code True} if set. */ - public static Map<String, String> fieldTypeNames(Map<String, Integer> typeIds) { - Map<String, String> names = U.newHashMap(typeIds.size()); - - for (Map.Entry<String, Integer> e : typeIds.entrySet()) - names.put(e.getKey(), fieldTypeName(e.getValue())); - - return names; + public static boolean hasRaw(short flags) { + return isFlagSet(flags, FLAG_HAS_RAW); } /** - * Write flags. + * Check if "no-field-ids" flag is set. * - * @param writer Writer. - * @param userType User type flag. + * @param flags Flags. + * @return {@code True} if set. */ - public static void writeFlags(BinaryWriterExImpl writer, boolean userType) { - short val = 0; - - if (userType) - val |= FLAG_USR_TYP; - - writer.doWriteShort(val); + public static boolean isCompactFooter(short flags) { + return isFlagSet(flags, FLAG_COMPACT_FOOTER); } /** - * Check if user type flag is set. + * Check whether particular flag is set. * * @param flags Flags. - * @return {@code True} if set. + * @param flag Flag. + * @return {@code True} if flag is set in flags. */ - public static boolean isUserType(short flags) { - return (flags & FLAG_USR_TYP) == FLAG_USR_TYP; + private static boolean isFlagSet(short flags, short flag) { + return (flags & flag) == flag; } - + /** - * Check if raw-only flag is set. + * Schema initial ID. * - * @param flags Flags. - * @return {@code True} if set. + * @return ID. */ - public static boolean isRawOnly(short flags) { - return (flags & FLAG_RAW_ONLY) == FLAG_RAW_ONLY; + public static int schemaInitialId() { + return FNV1_OFFSET_BASIS; } /** + * Update schema ID when new field is added. * + * @param schemaId Current schema ID. + * @param fieldId Field ID. + * @return New schema ID. */ - static { - PORTABLE_CLS.add(Byte.class); - PORTABLE_CLS.add(Short.class); - PORTABLE_CLS.add(Integer.class); - PORTABLE_CLS.add(Long.class); - PORTABLE_CLS.add(Float.class); - PORTABLE_CLS.add(Double.class); - PORTABLE_CLS.add(Character.class); - PORTABLE_CLS.add(Boolean.class); - PORTABLE_CLS.add(String.class); - PORTABLE_CLS.add(UUID.class); - PORTABLE_CLS.add(Date.class); - PORTABLE_CLS.add(Timestamp.class); - PORTABLE_CLS.add(BigDecimal.class); - PORTABLE_CLS.add(byte[].class); - PORTABLE_CLS.add(short[].class); - PORTABLE_CLS.add(int[].class); - PORTABLE_CLS.add(long[].class); - PORTABLE_CLS.add(float[].class); - PORTABLE_CLS.add(double[].class); - PORTABLE_CLS.add(char[].class); - PORTABLE_CLS.add(boolean[].class); - PORTABLE_CLS.add(String[].class); - PORTABLE_CLS.add(UUID[].class); - PORTABLE_CLS.add(Date[].class); - PORTABLE_CLS.add(Timestamp[].class); - PORTABLE_CLS.add(BigDecimal[].class); + public static int updateSchemaId(int schemaId, int fieldId) { + schemaId = schemaId ^ (fieldId & 0xFF); + schemaId = schemaId * FNV1_PRIME; + schemaId = schemaId ^ ((fieldId >> 8) & 0xFF); + schemaId = schemaId * FNV1_PRIME; + schemaId = schemaId ^ ((fieldId >> 16) & 0xFF); + schemaId = schemaId * FNV1_PRIME; + schemaId = schemaId ^ ((fieldId >> 24) & 0xFF); + schemaId = schemaId * FNV1_PRIME; + + return schemaId; } /** - * + * @param typeName Field type name. + * @return Field type ID; */ - static { - PLAIN_CLASS_TO_FLAG.put(Byte.class, GridPortableMarshaller.BYTE); - PLAIN_CLASS_TO_FLAG.put(Short.class, GridPortableMarshaller.SHORT); - PLAIN_CLASS_TO_FLAG.put(Integer.class, GridPortableMarshaller.INT); - PLAIN_CLASS_TO_FLAG.put(Long.class, GridPortableMarshaller.LONG); - PLAIN_CLASS_TO_FLAG.put(Float.class, GridPortableMarshaller.FLOAT); - PLAIN_CLASS_TO_FLAG.put(Double.class, GridPortableMarshaller.DOUBLE); - PLAIN_CLASS_TO_FLAG.put(Character.class, GridPortableMarshaller.CHAR); - PLAIN_CLASS_TO_FLAG.put(Boolean.class, GridPortableMarshaller.BOOLEAN); - PLAIN_CLASS_TO_FLAG.put(BigDecimal.class, GridPortableMarshaller.DECIMAL); - PLAIN_CLASS_TO_FLAG.put(String.class, GridPortableMarshaller.STRING); - PLAIN_CLASS_TO_FLAG.put(UUID.class, GridPortableMarshaller.UUID); - PLAIN_CLASS_TO_FLAG.put(Date.class, GridPortableMarshaller.DATE); - PLAIN_CLASS_TO_FLAG.put(Timestamp.class, GridPortableMarshaller.TIMESTAMP); + @SuppressWarnings("StringEquality") + public static int fieldTypeId(String typeName) { + for (int i = 0; i < FIELD_TYPE_NAMES.length; i++) { + String typeName0 = FIELD_TYPE_NAMES[i]; - PLAIN_CLASS_TO_FLAG.put(byte[].class, GridPortableMarshaller.BYTE_ARR); - PLAIN_CLASS_TO_FLAG.put(short[].class, GridPortableMarshaller.SHORT_ARR); - PLAIN_CLASS_TO_FLAG.put(int[].class, GridPortableMarshaller.INT_ARR); - PLAIN_CLASS_TO_FLAG.put(long[].class, GridPortableMarshaller.LONG_ARR); - PLAIN_CLASS_TO_FLAG.put(float[].class, GridPortableMarshaller.FLOAT_ARR); - PLAIN_CLASS_TO_FLAG.put(double[].class, GridPortableMarshaller.DOUBLE_ARR); - PLAIN_CLASS_TO_FLAG.put(char[].class, GridPortableMarshaller.CHAR_ARR); - PLAIN_CLASS_TO_FLAG.put(boolean[].class, GridPortableMarshaller.BOOLEAN_ARR); - PLAIN_CLASS_TO_FLAG.put(BigDecimal[].class, GridPortableMarshaller.DECIMAL_ARR); - PLAIN_CLASS_TO_FLAG.put(String[].class, GridPortableMarshaller.STRING_ARR); - PLAIN_CLASS_TO_FLAG.put(UUID[].class, GridPortableMarshaller.UUID_ARR); - PLAIN_CLASS_TO_FLAG.put(Date[].class, GridPortableMarshaller.DATE_ARR); - PLAIN_CLASS_TO_FLAG.put(Timestamp[].class, GridPortableMarshaller.TIMESTAMP_ARR); + if (typeName.equals(typeName0)) + return i; + } - for (Map.Entry<Class<?>, Byte> entry : PLAIN_CLASS_TO_FLAG.entrySet()) - FLAG_TO_CLASS.put(entry.getValue(), entry.getKey()); + throw new IllegalArgumentException("Invalid metadata type name: " + typeName); + } - PLAIN_CLASS_TO_FLAG.put(byte.class, GridPortableMarshaller.BYTE); - PLAIN_CLASS_TO_FLAG.put(short.class, GridPortableMarshaller.SHORT); - PLAIN_CLASS_TO_FLAG.put(int.class, GridPortableMarshaller.INT); - PLAIN_CLASS_TO_FLAG.put(long.class, GridPortableMarshaller.LONG); - PLAIN_CLASS_TO_FLAG.put(float.class, GridPortableMarshaller.FLOAT); - PLAIN_CLASS_TO_FLAG.put(double.class, GridPortableMarshaller.DOUBLE); - PLAIN_CLASS_TO_FLAG.put(char.class, GridPortableMarshaller.CHAR); - PLAIN_CLASS_TO_FLAG.put(boolean.class, GridPortableMarshaller.BOOLEAN); + /** + * @param typeId Field type ID. + * @return Field type name. + */ + public static String fieldTypeName(int typeId) { + assert typeId >= 0 && typeId < FIELD_TYPE_NAMES.length : typeId; - for (byte b : new byte[] { - BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, - CHAR, BOOLEAN, DECIMAL, STRING, UUID, DATE, TIMESTAMP, - BYTE_ARR, SHORT_ARR, INT_ARR, LONG_ARR, FLOAT_ARR, DOUBLE_ARR, - CHAR_ARR, BOOLEAN_ARR, DECIMAL_ARR, STRING_ARR, UUID_ARR, DATE_ARR, TIMESTAMP_ARR, - ENUM, ENUM_ARR, NULL}) { + String typeName = FIELD_TYPE_NAMES[typeId]; - PLAIN_TYPE_FLAG[b] = true; - } + assert typeName != null : typeId; + + return typeName; } /** @@ -623,18 +663,16 @@ public class PortableUtils { * Write portable header. * * @param writer Writer. - * @param usrTyp User type flag. * @param typeId Type ID. * @param hashCode Hash code. * @param clsName Class name (optional). * @return Position where length should be written. */ - public static int writeHeader(BinaryWriterExImpl writer, boolean usrTyp, int typeId, int hashCode, - @Nullable String clsName) { + public static int writeHeader(BinaryWriterExImpl writer, int typeId, int hashCode, @Nullable String clsName) { writer.doWriteByte(GridPortableMarshaller.OBJ); writer.doWriteByte(GridPortableMarshaller.PROTO_VER); - PortableUtils.writeFlags(writer, usrTyp); + writer.doWriteShort((short) 0); writer.doWriteInt(typeId); writer.doWriteInt(hashCode); @@ -668,12 +706,12 @@ public class PortableUtils { public static int footerStartRelative(PortablePositionReadable in, int start) { short flags = in.readShortPositioned(start + GridPortableMarshaller.FLAGS_POS); - if (PortableUtils.isRawOnly(flags)) - // No schema, footer start equals to object end. - return length(in, start); - else + if (hasSchema(flags)) // Schema exists, use offset. return in.readIntPositioned(start + GridPortableMarshaller.SCHEMA_OR_RAW_OFF_POS); + else + // No schema, footer start equals to object end. + return length(in, start); } /** @@ -692,56 +730,73 @@ public class PortableUtils { * * @param in Input stream. * @param start Start position. - * @param fieldOffsetSize Field offset size. * @return Footer. */ - public static IgniteBiTuple<Integer, Integer> footerAbsolute(PortablePositionReadable in, int start, - int fieldOffsetSize) { - int footerStart = footerStartRelative(in, start); + public static IgniteBiTuple<Integer, Integer> footerAbsolute(PortablePositionReadable in, int start) { + short flags = in.readShortPositioned(start + GridPortableMarshaller.FLAGS_POS); + int footerEnd = length(in, start); - // Take in count possible raw offset. - if ((footerEnd - footerStart) % (4 + fieldOffsetSize) != 0) - footerEnd -= 4; + if (hasSchema(flags)) { + // Schema exists. + int footerStart = in.readIntPositioned(start + GridPortableMarshaller.SCHEMA_OR_RAW_OFF_POS); + + if (hasRaw(flags)) + footerEnd -= 4; - return F.t(start + footerStart, start + footerEnd); + assert footerStart <= footerEnd; + + return F.t(start + footerStart, start + footerEnd); + } + else + // No schema. + return F.t(start + footerEnd, start + footerEnd); } /** - * Get raw offset of the object. + * Get relative raw offset of the object. * * @param in Input stream. * @param start Object start position inside the stream. - * @param fieldOffsetSize Field offset size. * @return Raw offset. */ - public static int rawOffsetAbsolute(PortablePositionReadable in, int start, int fieldOffsetSize) { - int len = length(in, start); - + public static int rawOffsetRelative(PortablePositionReadable in, int start) { short flags = in.readShortPositioned(start + GridPortableMarshaller.FLAGS_POS); - if (PortableUtils.isRawOnly(flags)) - // No schema, raw offset is located on schema offset position. - return start + in.readIntPositioned(start + GridPortableMarshaller.SCHEMA_OR_RAW_OFF_POS); - else { - // Schema exists. - int schemaOff = in.readIntPositioned(start + GridPortableMarshaller.SCHEMA_OR_RAW_OFF_POS); + int len = length(in, start); - if (((len - schemaOff) % (4 + fieldOffsetSize)) == 0x0) - // Even amount of records in schema => no raw offset. - return start + schemaOff; + if (hasSchema(flags)){ + // Schema exists. + if (hasRaw(flags)) + // Raw offset is set, it is at the very end of the object. + return in.readIntPositioned(start + len - 4); else - // Odd amount of records in schema => raw offset is the very last 4 bytes in object. - return start + in.readIntPositioned(start + len - 4); + // Raw offset is not set, so just return schema offset. + return in.readIntPositioned(start + GridPortableMarshaller.SCHEMA_OR_RAW_OFF_POS); } + else + // No schema, raw offset is located on schema offset position. + return in.readIntPositioned(start + GridPortableMarshaller.SCHEMA_OR_RAW_OFF_POS); + } + + /** + * Get absolute raw offset of the object. + * + * @param in Input stream. + * @param start Object start position inside the stream. + * @return Raw offset. + */ + public static int rawOffsetAbsolute(PortablePositionReadable in, int start) { + return start + rawOffsetRelative(in, start); } /** - * Get offset size for the given flags. + * Get offset length for the given flags. + * * @param flags Flags. * @return Offset size. */ - public static int fieldOffsetSize(short flags) { + public static int fieldOffsetLength(short flags) { if ((flags & FLAG_OFFSET_ONE_BYTE) == FLAG_OFFSET_ONE_BYTE) return OFFSET_1; else if ((flags & FLAG_OFFSET_TWO_BYTES) == FLAG_OFFSET_TWO_BYTES) @@ -751,6 +806,16 @@ public class PortableUtils { } /** + * Get field ID length. + * + * @param flags Flags. + * @return Field ID length. + */ + public static int fieldIdLength(short flags) { + return isCompactFooter(flags) ? 0 : FIELD_ID_LEN; + } + + /** * Get relative field offset. * * @param stream Stream. @@ -770,4 +835,72 @@ public class PortableUtils { return res; } + + /** + * Merge old and new metas. + * + * @param oldMeta Old meta. + * @param newMeta New meta. + * @return New meta if old meta was null, old meta if no changes detected, merged meta otherwise. + * @throws BinaryObjectException If merge failed due to metadata conflict. + */ + public static BinaryMetadata mergeMetadata(@Nullable BinaryMetadata oldMeta, BinaryMetadata newMeta) { + assert newMeta != null; + + if (oldMeta == null) + return newMeta; + else { + assert oldMeta.typeId() == newMeta.typeId(); + + // Check type name. + if (!F.eq(oldMeta.typeName(), newMeta.typeName())) { + throw new BinaryObjectException( + "Two portable types have duplicate type ID [" + "typeId=" + oldMeta.typeId() + + ", typeName1=" + oldMeta.typeName() + ", typeName2=" + newMeta.typeName() + ']' + ); + } + + // Check affinity field names. + if (!F.eq(oldMeta.affinityKeyFieldName(), newMeta.affinityKeyFieldName())) { + throw new BinaryObjectException( + "Binary type has different affinity key fields [" + "typeName=" + newMeta.typeName() + + ", affKeyFieldName1=" + oldMeta.affinityKeyFieldName() + + ", affKeyFieldName2=" + newMeta.affinityKeyFieldName() + ']' + ); + } + + // Check and merge fields. + boolean changed = false; + + Map<String, Integer> mergedFields = new HashMap<>(oldMeta.fieldsMap()); + Map<String, Integer> newFields = newMeta.fieldsMap(); + + for (Map.Entry<String, Integer> newField : newFields.entrySet()) { + Integer oldFieldType = mergedFields.put(newField.getKey(), newField.getValue()); + + if (oldFieldType == null) + changed = true; + else if (!F.eq(oldFieldType, newField.getValue())) { + throw new BinaryObjectException( + "Binary type has different field types [" + "typeName=" + oldMeta.typeName() + + ", fieldName=" + newField.getKey() + + ", fieldTypeName1=" + PortableUtils.fieldTypeName(oldFieldType) + + ", fieldTypeName2=" + PortableUtils.fieldTypeName(newField.getValue()) + ']' + ); + } + } + + // Check and merge schemas. + Collection<PortableSchema> mergedSchemas = new HashSet<>(oldMeta.schemas()); + + for (PortableSchema newSchema : newMeta.schemas()) { + if (mergedSchemas.add(newSchema)) + changed = true; + } + + // Return either old meta if no changes detected, or new merged meta. + return changed ? new BinaryMetadata(oldMeta.typeId(), oldMeta.typeName(), mergedFields, + oldMeta.affinityKeyFieldName(), mergedSchemas) : oldMeta; + } + } } \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/portable/builder/BinaryObjectBuilderImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/portable/builder/BinaryObjectBuilderImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/portable/builder/BinaryObjectBuilderImpl.java index ca8f09b..dfc2330 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/portable/builder/BinaryObjectBuilderImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/portable/builder/BinaryObjectBuilderImpl.java @@ -22,11 +22,14 @@ import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.binary.BinaryObjectBuilder; import org.apache.ignite.binary.BinaryObjectException; import org.apache.ignite.binary.BinaryType; +import org.apache.ignite.internal.portable.BinaryMetadata; import org.apache.ignite.internal.portable.BinaryObjectImpl; import org.apache.ignite.internal.portable.BinaryObjectOffheapImpl; import org.apache.ignite.internal.portable.BinaryWriterExImpl; import org.apache.ignite.internal.portable.GridPortableMarshaller; import org.apache.ignite.internal.portable.PortableContext; +import org.apache.ignite.internal.portable.PortableSchema; +import org.apache.ignite.internal.portable.PortableSchemaRegistry; import org.apache.ignite.internal.portable.PortableUtils; import org.apache.ignite.internal.util.GridArgumentCheck; import org.apache.ignite.internal.util.typedef.F; @@ -176,7 +179,6 @@ public class BinaryObjectBuilderImpl implements BinaryObjectBuilder { /** {@inheritDoc} */ @Override public BinaryObject build() { try (BinaryWriterExImpl writer = new BinaryWriterExImpl(ctx, typeId, false)) { - PortableBuilderSerializer serializationCtx = new PortableBuilderSerializer(); serializationCtx.registerObjectWriting(this, 0); @@ -196,50 +198,57 @@ public class BinaryObjectBuilderImpl implements BinaryObjectBuilder { void serializeTo(BinaryWriterExImpl writer, PortableBuilderSerializer serializer) { try { PortableUtils.writeHeader(writer, - true, registeredType ? typeId : UNREGISTERED_TYPE_ID, hashCode, - registeredType ? null : clsNameToWrite); + registeredType ? null : clsNameToWrite + ); Set<Integer> remainsFlds = null; if (reader != null) { + PortableSchema schema = reader.schema(start); + Map<Integer, Object> assignedFldsById; if (assignedVals != null) { assignedFldsById = U.newHashMap(assignedVals.size()); for (Map.Entry<String, Object> entry : assignedVals.entrySet()) { - int fldId = ctx.fieldId(typeId, entry.getKey()); + int fieldId = ctx.fieldId(typeId, entry.getKey()); - assignedFldsById.put(fldId, entry.getValue()); + assignedFldsById.put(fieldId, entry.getValue()); } remainsFlds = assignedFldsById.keySet(); - } else + } + else assignedFldsById = Collections.emptyMap(); // Get footer details. - int fieldOffsetSize = PortableUtils.fieldOffsetSize(flags); + int fieldIdLen = PortableUtils.fieldIdLength(flags); + int fieldOffsetLen = PortableUtils.fieldOffsetLength(flags); - IgniteBiTuple<Integer, Integer> footer = PortableUtils.footerAbsolute(reader, start, fieldOffsetSize); + IgniteBiTuple<Integer, Integer> footer = PortableUtils.footerAbsolute(reader, start); int footerPos = footer.get1(); int footerEnd = footer.get2(); // Get raw position. - int rawPos = PortableUtils.rawOffsetAbsolute(reader, start, fieldOffsetSize); + int rawPos = PortableUtils.rawOffsetAbsolute(reader, start); // Position reader on data. reader.position(start + hdrLen); - while (reader.position() + 4 < rawPos) { - int fieldId = reader.readIntPositioned(footerPos); - int fieldLen = fieldPositionAndLength(footerPos, footerEnd, rawPos, fieldOffsetSize).get2(); + int idx = 0; + + while (reader.position() < rawPos) { + int fieldId = schema.fieldId(idx++); + int fieldLen = + fieldPositionAndLength(footerPos, footerEnd, rawPos, fieldIdLen, fieldOffsetLen).get2(); int postPos = reader.position() + fieldLen; // Position where reader will be placed afterwards. - footerPos += 4 + fieldOffsetSize; + footerPos += fieldIdLen + fieldOffsetLen; if (assignedFldsById.containsKey(fieldId)) { Object assignedVal = assignedFldsById.remove(fieldId); @@ -281,11 +290,11 @@ public class BinaryObjectBuilderImpl implements BinaryObjectBuilder { } } - if (assignedVals != null && (remainsFlds == null || !remainsFlds.isEmpty())) { - BinaryType metadata = ctx.metaData(typeId); + BinaryType meta = ctx.metadata(typeId); - Map<String, Integer> newFldsMetadata = null; + Map<String, Integer> fieldsMeta = null; + if (assignedVals != null && (remainsFlds == null || !remainsFlds.isEmpty())) { for (Map.Entry<String, Object> entry : assignedVals.entrySet()) { Object val = entry.getValue(); @@ -294,16 +303,16 @@ public class BinaryObjectBuilderImpl implements BinaryObjectBuilder { String name = entry.getKey(); - int fldId = ctx.fieldId(typeId, name); + int fieldId = ctx.fieldId(typeId, name); - if (remainsFlds != null && !remainsFlds.contains(fldId)) + if (remainsFlds != null && !remainsFlds.contains(fieldId)) continue; - writer.writeFieldId(fldId); + writer.writeFieldId(fieldId); serializer.writeValue(writer, val); - String oldFldTypeName = metadata == null ? null : metadata.fieldTypeName(name); + String oldFldTypeName = meta == null ? null : meta.fieldTypeName(name); int newFldTypeId; @@ -316,11 +325,10 @@ public class BinaryObjectBuilderImpl implements BinaryObjectBuilder { if (oldFldTypeName == null) { // It's a new field, we have to add it to metadata. + if (fieldsMeta == null) + fieldsMeta = new HashMap<>(); - if (newFldsMetadata == null) - newFldsMetadata = new HashMap<>(); - - newFldsMetadata.put(name, PortableUtils.fieldTypeId(newFldTypeName)); + fieldsMeta.put(name, PortableUtils.fieldTypeId(newFldTypeName)); } else { String objTypeName = PortableUtils.fieldTypeName(GridPortableMarshaller.OBJ); @@ -328,7 +336,7 @@ public class BinaryObjectBuilderImpl implements BinaryObjectBuilder { if (!objTypeName.equals(oldFldTypeName) && !oldFldTypeName.equals(newFldTypeName)) { throw new BinaryObjectException( "Wrong value has been set [" + - "typeName=" + (typeName == null ? metadata.typeName() : typeName) + + "typeName=" + (typeName == null ? meta.typeName() : typeName) + ", fieldName=" + name + ", fieldType=" + oldFldTypeName + ", assignedValueType=" + newFldTypeName + ']' @@ -336,25 +344,11 @@ public class BinaryObjectBuilderImpl implements BinaryObjectBuilder { } } } - - if (newFldsMetadata != null) { - String typeName = this.typeName; - - if (typeName == null) { - assert metadata != null; - - typeName = metadata.typeName(); - } - - ctx.updateMetaData(typeId, typeName, newFldsMetadata); - } } if (reader != null) { // Write raw data if any. - int fieldOffsetSize = PortableUtils.fieldOffsetSize(flags); - - int rawOff = PortableUtils.rawOffsetAbsolute(reader, start, fieldOffsetSize); + int rawOff = PortableUtils.rawOffsetAbsolute(reader, start); int footerStart = PortableUtils.footerStartAbsolute(reader, start); if (rawOff < footerStart) { @@ -368,6 +362,28 @@ public class BinaryObjectBuilderImpl implements BinaryObjectBuilder { } writer.postWrite(true); + + // Update metadata if needed. + int schemaId = writer.schemaId(); + + PortableSchemaRegistry schemaReg = ctx.schemaRegistry(typeId); + + if (schemaReg.schema(schemaId) == null) { + String typeName = this.typeName; + + if (typeName == null) { + assert meta != null; + + typeName = meta.typeName(); + } + + PortableSchema curSchema = writer.currentSchema(); + + ctx.updateMetadata(typeId, new BinaryMetadata(typeId, typeName, fieldsMeta, + ctx.affinityKeyFieldName(typeId), Collections.singleton(curSchema))); + + schemaReg.addSchema(curSchema.schemaId(), curSchema); + } } finally { writer.popSchema(); @@ -387,25 +403,26 @@ public class BinaryObjectBuilderImpl implements BinaryObjectBuilder { * @param footerPos Field position inside the footer (absolute). * @param footerEnd Footer end (absolute). * @param rawPos Raw data position (absolute). - * @param fieldOffsetSize Size of field's offset. + * @param fieldIdLen Field ID length. + * @param fieldOffsetLen Field offset length. * @return Tuple with field position and length. */ private IgniteBiTuple<Integer, Integer> fieldPositionAndLength(int footerPos, int footerEnd, int rawPos, - int fieldOffsetSize) { + int fieldIdLen, int fieldOffsetLen) { // Get field offset first. - int fieldOffset = PortableUtils.fieldOffsetRelative(reader, footerPos + 4, fieldOffsetSize); + int fieldOffset = PortableUtils.fieldOffsetRelative(reader, footerPos + fieldIdLen, fieldOffsetLen); int fieldPos = start + fieldOffset; // Get field length. int fieldLen; - if (footerPos + 4 + fieldOffsetSize == footerEnd) + if (footerPos + fieldIdLen + fieldOffsetLen == footerEnd) // This is the last field, compare to raw offset. fieldLen = rawPos - fieldPos; else { // Field is somewhere in the middle, get difference with the next offset. - int nextFieldOffset = PortableUtils.fieldOffsetRelative(reader, footerPos + 4 + fieldOffsetSize + 4, - fieldOffsetSize); + int nextFieldOffset = PortableUtils.fieldOffsetRelative(reader, + footerPos + fieldIdLen + fieldOffsetLen + fieldIdLen, fieldOffsetLen); fieldLen = nextFieldOffset - fieldOffset; } @@ -417,30 +434,37 @@ public class BinaryObjectBuilderImpl implements BinaryObjectBuilder { * Initialize read cache if needed. */ private void ensureReadCacheInit() { + assert reader != null; + if (readCache == null) { - int fieldOffsetSize = PortableUtils.fieldOffsetSize(flags); + int fieldIdLen = PortableUtils.fieldIdLength(flags); + int fieldOffsetLen = PortableUtils.fieldOffsetLength(flags); + + PortableSchema schema = reader.schema(start); Map<Integer, Object> readCache = new HashMap<>(); - IgniteBiTuple<Integer, Integer> footer = PortableUtils.footerAbsolute(reader, start, fieldOffsetSize); + IgniteBiTuple<Integer, Integer> footer = PortableUtils.footerAbsolute(reader, start); int footerPos = footer.get1(); int footerEnd = footer.get2(); - int rawPos = PortableUtils.rawOffsetAbsolute(reader, start, fieldOffsetSize); + int rawPos = PortableUtils.rawOffsetAbsolute(reader, start); + + int idx = 0; - while (footerPos + 4 < footerEnd) { - int fieldId = reader.readIntPositioned(footerPos); + while (footerPos + fieldIdLen < footerEnd) { + int fieldId = schema.fieldId(idx++); IgniteBiTuple<Integer, Integer> posAndLen = - fieldPositionAndLength(footerPos, footerEnd, rawPos, fieldOffsetSize); + fieldPositionAndLength(footerPos, footerEnd, rawPos, fieldIdLen, fieldOffsetLen); Object val = reader.getValueQuickly(posAndLen.get1(), posAndLen.get2()); readCache.put(fieldId, val); // Shift current footer position. - footerPos += 4 + fieldOffsetSize; + footerPos += fieldIdLen + fieldOffsetLen; } this.readCache = readCache; http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/portable/builder/PortableBuilderReader.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/portable/builder/PortableBuilderReader.java b/modules/core/src/main/java/org/apache/ignite/internal/portable/builder/PortableBuilderReader.java index 5c6a131..b6a6b54 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/portable/builder/PortableBuilderReader.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/portable/builder/PortableBuilderReader.java @@ -27,6 +27,7 @@ import org.apache.ignite.internal.portable.PortablePositionReadable; import org.apache.ignite.internal.portable.BinaryObjectImpl; import org.apache.ignite.internal.portable.PortablePrimitives; import org.apache.ignite.internal.portable.BinaryReaderExImpl; +import org.apache.ignite.internal.portable.PortableSchema; import org.apache.ignite.internal.portable.PortableUtils; import org.apache.ignite.internal.portable.BinaryWriterExImpl; import org.apache.ignite.binary.BinaryObjectException; @@ -63,7 +64,7 @@ public class PortableBuilderReader implements PortablePositionReadable { pos = objImpl.start(); // TODO: IGNITE-1272 - Is class loader needed here? - reader = new BinaryReaderExImpl(portableContext(), arr, pos, null); + reader = new BinaryReaderExImpl(ctx, arr, pos, null); } /** @@ -81,6 +82,24 @@ public class PortableBuilderReader implements PortablePositionReadable { } /** + * Get schema of the object, starting at the given position. + * + * @param start Start position. + * @return Object's schema. + */ + public PortableSchema schema(int start) { + // We can use current reader in case start is equal to initially recorded position. + BinaryReaderExImpl targetReader; + + if (start == pos) + targetReader = reader; + else + targetReader = new BinaryReaderExImpl(ctx, arr, start, null); + + return targetReader.getOrCreateSchema(); + } + + /** * @return Read int value. */ public int readInt() { http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/portable/CacheObjectBinaryProcessor.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/portable/CacheObjectBinaryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/portable/CacheObjectBinaryProcessor.java index cac0dcf..e4db77c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/portable/CacheObjectBinaryProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/portable/CacheObjectBinaryProcessor.java @@ -59,7 +59,7 @@ public interface CacheObjectBinaryProcessor extends IgniteCacheObjectProcessor { * @param fieldTypeIds Fields map. * @throws IgniteException In case of error. */ - public void updateMetaData(int typeId, String typeName, @Nullable String affKeyFieldName, + public void updateMetadata(int typeId, String typeName, @Nullable String affKeyFieldName, Map<String, Integer> fieldTypeIds) throws IgniteException; /** http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/portable/CacheObjectBinaryProcessorImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/portable/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/portable/CacheObjectBinaryProcessorImpl.java index 117eece..551ada5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/portable/CacheObjectBinaryProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/portable/CacheObjectBinaryProcessorImpl.java @@ -31,12 +31,12 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; import org.apache.ignite.internal.portable.BinaryMetadata; +import org.apache.ignite.internal.portable.BinaryMetadataHandler; import org.apache.ignite.internal.portable.BinaryObjectImpl; import org.apache.ignite.internal.portable.BinaryObjectOffheapImpl; import org.apache.ignite.internal.portable.BinaryTypeImpl; import org.apache.ignite.internal.portable.GridPortableMarshaller; import org.apache.ignite.internal.portable.PortableContext; -import org.apache.ignite.internal.portable.BinaryMetadataHandler; import org.apache.ignite.internal.portable.PortableUtils; import org.apache.ignite.internal.portable.builder.BinaryObjectBuilderImpl; import org.apache.ignite.internal.portable.streams.PortableInputStream; @@ -88,7 +88,6 @@ import java.io.ObjectInput; import java.io.ObjectOutput; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -168,17 +167,14 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm if (metaDataCache == null) { BinaryMetadata oldMeta = metaBuf.get(typeId); + BinaryMetadata mergedMeta = PortableUtils.mergeMetadata(oldMeta, newMeta0); - if (oldMeta == null || checkMeta(typeId, oldMeta, newMeta0, null)) { + if (oldMeta != mergedMeta) { synchronized (this) { - Map<String, Integer> fields = new HashMap<>(); - - if (checkMeta(typeId, oldMeta, newMeta0, fields)) { - newMeta0 = new BinaryMetadata(typeId, newMeta0.typeName(), fields, - newMeta0.affinityKeyFieldName()); + mergedMeta = PortableUtils.mergeMetadata(oldMeta, newMeta0); - metaBuf.put(typeId, newMeta0); - } + if (oldMeta != mergedMeta) + metaBuf.put(typeId, mergedMeta); else return; } @@ -192,6 +188,8 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm return; } + assert metaDataCache != null; + CacheObjectBinaryProcessorImpl.this.addMeta(typeId, newMeta0.wrap(portableCtx)); } @@ -297,24 +295,22 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm private void addClientCacheMetaData(PortableMetadataKey key, final BinaryMetadata newMeta) { int key0 = key.typeId(); - clientMetaDataCache.compute(key0, - new ConcurrentHashMap8.BiFun<Integer, BinaryTypeImpl, BinaryTypeImpl>() { - @Override public BinaryTypeImpl apply(Integer key, BinaryTypeImpl oldMeta) { - BinaryMetadata res; + clientMetaDataCache.compute(key0, new ConcurrentHashMap8.BiFun<Integer, BinaryTypeImpl, BinaryTypeImpl>() { + @Override public BinaryTypeImpl apply(Integer key, BinaryTypeImpl oldMeta) { + BinaryMetadata res; - BinaryMetadata oldMeta0 = oldMeta != null ? oldMeta.metadata() : null; + BinaryMetadata oldMeta0 = oldMeta != null ? oldMeta.metadata() : null; - try { - res = checkMeta(key, oldMeta0, newMeta, null) ? newMeta : oldMeta0; - } - catch (BinaryObjectException e) { - res = oldMeta0; - } - - return res != null ? res.wrap(portableCtx) : null; + try { + res = PortableUtils.mergeMetadata(oldMeta0, newMeta); + } + catch (BinaryObjectException e) { + res = oldMeta0; } + + return res != null ? res.wrap(portableCtx) : null; } - ); + }); } /** {@inheritDoc} */ @@ -448,9 +444,9 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm } /** {@inheritDoc} */ - @Override public void updateMetaData(int typeId, String typeName, @Nullable String affKeyFieldName, + @Override public void updateMetadata(int typeId, String typeName, @Nullable String affKeyFieldName, Map<String, Integer> fieldTypeIds) throws BinaryObjectException { - portableCtx.updateMetaData(typeId, new BinaryMetadata(typeId, typeName, fieldTypeIds, affKeyFieldName)); + portableCtx.updateMetadata(typeId, new BinaryMetadata(typeId, typeName, fieldTypeIds, affKeyFieldName, null)); } /** {@inheritDoc} */ @@ -464,13 +460,12 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm try { BinaryMetadata oldMeta = metaDataCache.localPeek(key); + BinaryMetadata mergedMeta = PortableUtils.mergeMetadata(oldMeta, newMeta0); - if (oldMeta == null || checkMeta(typeId, oldMeta, newMeta0, null)) { - BinaryObjectException err = metaDataCache.invoke(key, new MetaDataProcessor(typeId, newMeta0)); + BinaryObjectException err = metaDataCache.invoke(key, new MetadataProcessor(mergedMeta)); - if (err != null) - throw err; - } + if (err != null) + throw err; } catch (CacheException e) { throw new BinaryObjectException("Failed to update meta data for type: " + newMeta.typeName(), e); @@ -727,125 +722,44 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm } /** - * @param typeId Type ID. - * @param oldMeta Old meta. - * @param newMeta New meta. - * @param fields Fields map. - * @return Whether meta is changed. - * @throws org.apache.ignite.binary.BinaryObjectException In case of error. - */ - private static boolean checkMeta(int typeId, @Nullable BinaryMetadata oldMeta, - BinaryMetadata newMeta, @Nullable Map<String, Integer> fields) throws BinaryObjectException { - assert newMeta != null; - - Map<String, Integer> oldFields = oldMeta != null ? oldMeta.fieldsMap() : null; - Map<String, Integer> newFields = newMeta.fieldsMap(); - - boolean changed = false; - - if (oldMeta != null) { - if (!oldMeta.typeName().equals(newMeta.typeName())) { - throw new BinaryObjectException( - "Two portable types have duplicate type ID [" + - "typeId=" + typeId + - ", typeName1=" + oldMeta.typeName() + - ", typeName2=" + newMeta.typeName() + - ']' - ); - } - - if (!F.eq(oldMeta.affinityKeyFieldName(), newMeta.affinityKeyFieldName())) { - throw new BinaryObjectException( - "Portable type has different affinity key fields on different clients [" + - "typeName=" + newMeta.typeName() + - ", affKeyFieldName1=" + oldMeta.affinityKeyFieldName() + - ", affKeyFieldName2=" + newMeta.affinityKeyFieldName() + - ']' - ); - } - - if (fields != null) - fields.putAll(oldFields); - } - else - changed = true; - - for (Map.Entry<String, Integer> e : newFields.entrySet()) { - Integer oldTypeId = oldFields != null ? oldFields.get(e.getKey()) : null; - - if (oldTypeId != null) { - if (!oldTypeId.equals(e.getValue())) { - throw new BinaryObjectException( - "Portable field has different types on different clients [" + - "typeName=" + newMeta.typeName() + - ", fieldName=" + e.getKey() + - ", fieldTypeName1=" + PortableUtils.fieldTypeName(oldTypeId) + - ", fieldTypeName2=" + PortableUtils.fieldTypeName(e.getValue()) + - ']' - ); - } - } - else { - if (fields != null) - fields.put(e.getKey(), e.getValue()); - - changed = true; - } - } - - return changed; - } - - /** + * Processor responsible for metadata update. */ - private static class MetaDataProcessor implements - EntryProcessor<PortableMetadataKey, BinaryMetadata, BinaryObjectException>, Externalizable { + private static class MetadataProcessor + implements EntryProcessor<PortableMetadataKey, BinaryMetadata, BinaryObjectException>, Externalizable { /** */ private static final long serialVersionUID = 0L; /** */ - private int typeId; - - /** */ private BinaryMetadata newMeta; /** * For {@link Externalizable}. */ - public MetaDataProcessor() { + public MetadataProcessor() { // No-op. } /** - * @param typeId Type ID. * @param newMeta New metadata. */ - private MetaDataProcessor(int typeId, BinaryMetadata newMeta) { + private MetadataProcessor(BinaryMetadata newMeta) { assert newMeta != null; - this.typeId = typeId; this.newMeta = newMeta; } /** {@inheritDoc} */ - @Override public BinaryObjectException process( - MutableEntry<PortableMetadataKey, BinaryMetadata> entry, + @Override public BinaryObjectException process(MutableEntry<PortableMetadataKey, BinaryMetadata> entry, Object... args) { try { BinaryMetadata oldMeta = entry.getValue(); - Map<String, Integer> fields = new HashMap<>(); + BinaryMetadata mergedMeta = PortableUtils.mergeMetadata(oldMeta, newMeta); - if (checkMeta(typeId, oldMeta, newMeta, fields)) { - BinaryMetadata res = new BinaryMetadata(typeId, newMeta.typeName(), fields, - newMeta.affinityKeyFieldName()); + if (mergedMeta != oldMeta) + entry.setValue(mergedMeta); - entry.setValue(res); - - return null; - } - else - return null; + return null; } catch (BinaryObjectException e) { return e; @@ -854,19 +768,17 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm /** {@inheritDoc} */ @Override public void writeExternal(ObjectOutput out) throws IOException { - out.writeInt(typeId); out.writeObject(newMeta); } /** {@inheritDoc} */ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { - typeId = in.readInt(); newMeta = (BinaryMetadata)in.readObject(); } /** {@inheritDoc} */ @Override public String toString() { - return S.toString(MetaDataProcessor.class, this); + return S.toString(MetadataProcessor.class, this); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java index 05d3515..d999466 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.platform; import org.apache.ignite.IgniteException; +import org.apache.ignite.binary.BinaryType; import org.apache.ignite.cluster.ClusterMetrics; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.events.CacheEvent; @@ -33,10 +34,10 @@ import org.apache.ignite.events.JobEvent; import org.apache.ignite.events.SwapSpaceEvent; import org.apache.ignite.events.TaskEvent; import org.apache.ignite.internal.GridKernalContext; -import org.apache.ignite.internal.portable.GridPortableMarshaller; -import org.apache.ignite.internal.portable.BinaryMetadata; import org.apache.ignite.internal.portable.BinaryRawReaderEx; import org.apache.ignite.internal.portable.BinaryRawWriterEx; +import org.apache.ignite.internal.portable.BinaryTypeImpl; +import org.apache.ignite.internal.portable.GridPortableMarshaller; import org.apache.ignite.internal.processors.cache.portable.CacheObjectBinaryProcessorImpl; import org.apache.ignite.internal.processors.platform.cache.PlatformCacheEntryFilter; import org.apache.ignite.internal.processors.platform.cache.PlatformCacheEntryFilterImpl; @@ -69,7 +70,6 @@ import org.apache.ignite.internal.processors.platform.utils.PlatformUtils; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T4; import org.apache.ignite.lang.IgniteBiTuple; -import org.apache.ignite.binary.BinaryType; import org.jetbrains.annotations.Nullable; import java.sql.Timestamp; @@ -359,7 +359,7 @@ public class PlatformContextImpl implements PlatformContext { ); for (T4<Integer, String, String, Map<String, Integer>> meta : metas) - cacheObjProc.updateMetaData(meta.get1(), meta.get2(), meta.get3(), meta.get4()); + cacheObjProc.updateMetadata(meta.get1(), meta.get2(), meta.get3(), meta.get4()); } /** {@inheritDoc} */ @@ -390,7 +390,7 @@ public class PlatformContextImpl implements PlatformContext { else { writer.writeBoolean(true); - Map<String, Integer> fields = ((BinaryMetadata)meta).fieldsMap(); + Map<String, Integer> fields = ((BinaryTypeImpl)meta).metadata().fieldsMap(); writer.writeInt(typeId); writer.writeString(meta.typeName()); http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cpp/PlatformCppConfigurationClosure.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cpp/PlatformCppConfigurationClosure.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cpp/PlatformCppConfigurationClosure.java index 9f17bdd..e9cd1e3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cpp/PlatformCppConfigurationClosure.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cpp/PlatformCppConfigurationClosure.java @@ -71,7 +71,11 @@ public class PlatformCppConfigurationClosure extends PlatformAbstractConfigurati Marshaller marsh = igniteCfg.getMarshaller(); if (marsh == null) { - igniteCfg.setMarshaller(new PortableMarshaller()); + PortableMarshaller marsh0 = new PortableMarshaller(); + + marsh0.setCompactFooter(false); + + igniteCfg.setMarshaller(marsh0); cppCfg0.warnings(Collections.singleton("Marshaller is automatically set to " + PortableMarshaller.class.getName() + " (other nodes must have the same marshaller type).")); @@ -79,6 +83,9 @@ public class PlatformCppConfigurationClosure extends PlatformAbstractConfigurati else if (!(marsh instanceof PortableMarshaller)) throw new IgniteException("Unsupported marshaller (only " + PortableMarshaller.class.getName() + " can be used when running Apache Ignite C++): " + marsh.getClass().getName()); + else if (((PortableMarshaller)marsh).isCompactFooter()) + throw new IgniteException("Unsupported " + PortableMarshaller.class.getName() + + " \"compactFooter\" flag: must be false when running Apache Ignite C++."); // Set Ignite home so that marshaller context works. String ggHome = igniteCfg.getIgniteHome(); http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetConfigurationClosure.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetConfigurationClosure.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetConfigurationClosure.java index d0462e9..a59fd22 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetConfigurationClosure.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetConfigurationClosure.java @@ -92,7 +92,11 @@ public class PlatformDotNetConfigurationClosure extends PlatformAbstractConfigur Marshaller marsh = igniteCfg.getMarshaller(); if (marsh == null) { - igniteCfg.setMarshaller(new PortableMarshaller()); + PortableMarshaller marsh0 = new PortableMarshaller(); + + marsh0.setCompactFooter(false); + + igniteCfg.setMarshaller(marsh0); dotNetCfg0.warnings(Collections.singleton("Marshaller is automatically set to " + PortableMarshaller.class.getName() + " (other nodes must have the same marshaller type).")); @@ -100,6 +104,9 @@ public class PlatformDotNetConfigurationClosure extends PlatformAbstractConfigur else if (!(marsh instanceof PortableMarshaller)) throw new IgniteException("Unsupported marshaller (only " + PortableMarshaller.class.getName() + " can be used when running Apache Ignite.NET): " + marsh.getClass().getName()); + else if (((PortableMarshaller)marsh).isCompactFooter()) + throw new IgniteException("Unsupported " + PortableMarshaller.class.getName() + + " \"compactFooter\" flag: must be false when running Apache Ignite.NET."); // Set Ignite home so that marshaller context works. String ggHome = igniteCfg.getIgniteHome(); http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java index 259d8c9..7337378 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java @@ -4983,6 +4983,31 @@ public abstract class IgniteUtils { } /** + * Read hash map. + * + * @param in Input. + * @return Read map. + * @throws IOException If de-serialization failed. + * @throws ClassNotFoundException If deserialized class could not be found. + */ + @SuppressWarnings({"unchecked"}) + @Nullable public static <K, V> HashMap<K, V> readHashMap(ObjectInput in) + throws IOException, ClassNotFoundException { + int size = in.readInt(); + + // Check null flag. + if (size == -1) + return null; + + HashMap<K, V> map = U.newHashMap(size); + + for (int i = 0; i < size; i++) + map.put((K)in.readObject(), (V)in.readObject()); + + return map; + } + + /** * * @param in Input. * @return Read map. http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/marshaller/portable/PortableMarshaller.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/portable/PortableMarshaller.java b/modules/core/src/main/java/org/apache/ignite/marshaller/portable/PortableMarshaller.java index 409a893..1704c8a 100644 --- a/modules/core/src/main/java/org/apache/ignite/marshaller/portable/PortableMarshaller.java +++ b/modules/core/src/main/java/org/apache/ignite/marshaller/portable/PortableMarshaller.java @@ -74,6 +74,12 @@ import org.jetbrains.annotations.Nullable; * For information about Spring framework visit <a href="http://www.springframework.org/">www.springframework.org</a> */ public class PortableMarshaller extends AbstractMarshaller { + /** Default value of "keep deserialized" flag. */ + public static final boolean DFLT_KEEP_DESERIALIZED = true; + + /** Default value of "compact footer" flag. */ + public static final boolean DFLT_COMPACT_FOOTER = true; + // TODO ignite-1282 Move to IgniteConfiguration. /** Class names. */ private Collection<String> clsNames; @@ -88,7 +94,10 @@ public class PortableMarshaller extends AbstractMarshaller { private Collection<BinaryTypeConfiguration> typeCfgs; /** Keep deserialized flag. */ - private boolean keepDeserialized = true; + private boolean keepDeserialized = DFLT_KEEP_DESERIALIZED; + + /** Compact footer. */ + private boolean compactFooter = DFLT_COMPACT_FOOTER; /** */ private GridPortableMarshaller impl; @@ -192,6 +201,33 @@ public class PortableMarshaller extends AbstractMarshaller { } /** + * Get whether to write footers in compact form. When enabled, Ignite will not write fields metadata + * when serializing objects, because internally {@code PortableMarshaller} already distribute metadata inside + * cluster. This increases serialization performance. + * <p> + * <b>WARNING!</b> This mode should be disabled when already serialized data can be taken from some external + * sources (e.g. cache store which stores data in binary form, data center replication, etc.). Otherwise binary + * objects without any associated metadata could appear in the cluster and Ignite will not be able to deserialize + * it. + * <p> + * Defaults to {@link #DFLT_COMPACT_FOOTER}. + * + * @return Whether to write footers in compact form. + */ + public boolean isCompactFooter() { + return compactFooter; + } + + /** + * Set whether to write footers in compact form. See {@link #isCompactFooter()} for more info. + * + * @param compactFooter Whether to write footers in compact form. + */ + public void setCompactFooter(boolean compactFooter) { + this.compactFooter = compactFooter; + } + + /** * Returns currently set {@link MarshallerContext}. * * @return Marshaller context. http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java index ae23d0e..45c8e0f 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java @@ -135,6 +135,7 @@ import static org.apache.ignite.events.EventType.EVT_NODE_LEFT; import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED; import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER; +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_COMPACT_FOOTER; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID; import static org.apache.ignite.spi.IgnitePortProtocol.TCP; import static org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoverySpiState.AUTH_FAILED; @@ -3160,8 +3161,8 @@ class ServerImpl extends TcpDiscoveryImpl { " property value differs from remote node's value " + "(to make sure all nodes in topology have identical marshaller settings, " + "configure system property explicitly) " + - "[locMarshUseDfltSuid=" + locMarshUseDfltSuid + - ", rmtMarshUseDfltSuid=" + rmtMarshUseDfltSuid + + "[locMarshUseDfltSuid=" + rmtMarshUseDfltSuid + + ", rmtMarshUseDfltSuid=" + locMarshUseDfltSuid + ", locNodeAddrs=" + U.addressesAsString(node) + ", locPort=" + node.discoveryPort() + ", rmtNodeAddr=" + U.addressesAsString(locNode) + ", locNodeId=" + node.id() + ", rmtNodeId=" + locNode.id() + ']'; @@ -3182,6 +3183,52 @@ class ServerImpl extends TcpDiscoveryImpl { return; } + // Validate compact footer flags. + Boolean locMarshCompactFooter = locNode.attribute(ATTR_MARSHALLER_COMPACT_FOOTER); + boolean locMarshCompactFooterBool = locMarshCompactFooter != null ? locMarshCompactFooter : false; + + Boolean rmtMarshCompactFooter = node.attribute(ATTR_MARSHALLER_COMPACT_FOOTER); + boolean rmtMarshCompactFooterBool = rmtMarshCompactFooter != null ? rmtMarshCompactFooter : false; + + if (locMarshCompactFooterBool != rmtMarshCompactFooterBool) { + String errMsg = "Local node's portable marshaller \"compactFooter\" property differs from " + + "the same property on remote node (make sure all nodes in topology have the same value " + + "of \"compactFooter\" property) [locMarshallerCompactFooter=" + locMarshCompactFooterBool + + ", rmtMarshallerCompactFooter=" + rmtMarshCompactFooterBool + + ", locNodeAddrs=" + U.addressesAsString(locNode) + + ", rmtNodeAddrs=" + U.addressesAsString(node) + + ", locNodeId=" + locNode.id() + ", rmtNodeId=" + msg.creatorNodeId() + ']'; + + LT.warn(log, null, errMsg); + + // Always output in debug. + if (log.isDebugEnabled()) + log.debug(errMsg); + + try { + String sndMsg = "Local node's portable marshaller \"compactFooter\" property differs from " + + "the same property on remote node (make sure all nodes in topology have the same value " + + "of \"compactFooter\" property) [locMarshallerCompactFooter=" + rmtMarshCompactFooterBool + + ", rmtMarshallerCompactFooter=" + locMarshCompactFooterBool + + ", locNodeAddrs=" + U.addressesAsString(node) + ", locPort=" + node.discoveryPort() + + ", rmtNodeAddr=" + U.addressesAsString(locNode) + ", locNodeId=" + node.id() + + ", rmtNodeId=" + locNode.id() + ']'; + + trySendMessageDirectly(node, new TcpDiscoveryCheckFailedMessage(locNodeId, sndMsg)); + } + catch (IgniteSpiException e) { + if (log.isDebugEnabled()) + log.debug("Failed to send marshaller check failed message to node " + + "[node=" + node + ", err=" + e.getMessage() + ']'); + + onException("Failed to send marshaller check failed message to node " + + "[node=" + node + ", err=" + e.getMessage() + ']', e); + } + + // Ignore join request. + return; + } + // Handle join. node.internalOrder(ring.nextNodeOrder()); http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/test/java/org/apache/ignite/internal/portable/BinaryFieldsAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/portable/BinaryFieldsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/portable/BinaryFieldsAbstractSelfTest.java index 14fc6f3..8f79db1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/portable/BinaryFieldsAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/portable/BinaryFieldsAbstractSelfTest.java @@ -46,11 +46,13 @@ public abstract class BinaryFieldsAbstractSelfTest extends GridCommonAbstractTes * @return Portable marshaller. * @throws Exception If failed. */ - protected static PortableMarshaller createMarshaller() throws Exception { - PortableContext ctx = new PortableContext(new TestCachingMetadataHandler(), new IgniteConfiguration()); + protected PortableMarshaller createMarshaller() throws Exception { + PortableContext ctx = new PortableContext(BinaryCachingMetadataHandler.create(), new IgniteConfiguration()); PortableMarshaller marsh = new PortableMarshaller(); + marsh.setCompactFooter(compactFooter()); + marsh.setTypeConfigurations(Arrays.asList( new BinaryTypeConfiguration(TestObject.class.getName()), new BinaryTypeConfiguration(TestOuterObject.class.getName()), @@ -65,6 +67,13 @@ public abstract class BinaryFieldsAbstractSelfTest extends GridCommonAbstractTes } /** + * @return Whether to use compact footer. + */ + protected boolean compactFooter() { + return true; + } + + /** * Get portable context for the current marshaller. * * @param marsh Marshaller. http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/test/java/org/apache/ignite/internal/portable/BinaryFooterOffsetsAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/portable/BinaryFooterOffsetsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/portable/BinaryFooterOffsetsAbstractSelfTest.java new file mode 100644 index 0000000..3ec0b83 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/portable/BinaryFooterOffsetsAbstractSelfTest.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.portable; + +import org.apache.ignite.binary.BinaryField; +import org.apache.ignite.binary.BinaryTypeConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.util.IgniteUtils; +import org.apache.ignite.marshaller.MarshallerContextTestImpl; +import org.apache.ignite.marshaller.portable.PortableMarshaller; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import java.util.Arrays; + +/** + * Contains tests for compact offsets. + */ +public abstract class BinaryFooterOffsetsAbstractSelfTest extends GridCommonAbstractTest { + /** 2 pow 8. */ + private static int POW_8 = 1 << 8; + + /** 2 pow 16. */ + private static int POW_16 = 1 << 16; + + /** Marshaller. */ + protected PortableMarshaller marsh; + + /** Portable context. */ + protected PortableContext ctx; + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + ctx = new PortableContext(BinaryCachingMetadataHandler.create(), new IgniteConfiguration()); + + marsh = new PortableMarshaller(); + + marsh.setCompactFooter(compactFooter()); + + marsh.setTypeConfigurations(Arrays.asList(new BinaryTypeConfiguration(TestObject.class.getName()))); + marsh.setContext(new MarshallerContextTestImpl(null)); + + IgniteUtils.invoke(PortableMarshaller.class, marsh, "setPortableContext", ctx); + } + + /** + * @return Whether to use compact footers. + */ + protected boolean compactFooter() { + return true; + } + + /** + * Test 1 byte. + * + * @throws Exception If failed. + */ + public void test1Byte() throws Exception { + check(POW_8 >> 2); + } + + /** + * Test 1 byte with sign altering. + * + * @throws Exception If failed. + */ + public void test1ByteSign() throws Exception { + check(POW_8 >> 1); + } + + /** + * Test 2 bytes. + * + * @throws Exception If failed. + */ + public void test2Bytes() throws Exception { + check(POW_16 >> 2); + } + + /** + * Test 2 bytes with sign altering. + * + * @throws Exception If failed. + */ + public void test2BytesSign() throws Exception { + check(POW_16 >> 1); + } + + /** + * Test 4 bytes. + * + * @throws Exception If failed. + */ + public void test4Bytes() throws Exception { + check(POW_16 << 2); + } + + /** + * Main check routine. + * + * @param len Length of the first field. + * + * @throws Exception If failed. + */ + private void check(int len) throws Exception { + TestObject obj = new TestObject(len); + + BinaryObjectEx portObj = toPortable(marsh, obj); + + // 1. Test portable object content. + assert portObj.hasField("field1"); + assert portObj.hasField("field2"); + + byte[] field1 = portObj.field("field1"); + Integer field2 = portObj.field("field2"); + + assert field1 != null; + assert field2 != null; + + assert Arrays.equals(obj.field1, field1); + assert obj.field2 == field2; + + // 2. Test fields API. + BinaryField field1Desc = portObj.type().field("field1"); + BinaryField field2Desc = portObj.type().field("field2"); + + assert field1Desc.exists(portObj); + assert field2Desc.exists(portObj); + + assert Arrays.equals(obj.field1, (byte[])field1Desc.value(portObj)); + assert obj.field2 == (Integer)field2Desc.value(portObj); + + // 3. Test deserialize. + TestObject objRestored = portObj.deserialize(); + + assert objRestored != null; + + assert Arrays.equals(obj.field1, objRestored.field1); + assert obj.field2 == objRestored.field2; + } + + /** + * Convert object to portable object. + * + * @param marsh Marshaller. + * @param obj Object. + * @return Portable object. + * @throws Exception If failed. + */ + protected abstract BinaryObjectEx toPortable(PortableMarshaller marsh, Object obj) throws Exception; + + /** + * Test object. + */ + public static class TestObject { + /** First field with variable length. */ + public byte[] field1; + + /** Second field. */ + public int field2; + + /** + * Default constructor. + */ + public TestObject() { + // No-op. + } + + /** + * Constructor. + * + * @param len Array length. + */ + public TestObject(int len) { + field1 = new byte[len]; + + field1[0] = 1; + field1[len - 1] = 2; + + field2 = len; + } + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/test/java/org/apache/ignite/internal/portable/BinaryFooterOffsetsHeapSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/portable/BinaryFooterOffsetsHeapSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/portable/BinaryFooterOffsetsHeapSelfTest.java new file mode 100644 index 0000000..b23f012 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/portable/BinaryFooterOffsetsHeapSelfTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.portable; + +import org.apache.ignite.marshaller.portable.PortableMarshaller; + +/** + * Compact offsets tests for heap portable objects. + */ +public class BinaryFooterOffsetsHeapSelfTest extends BinaryFooterOffsetsAbstractSelfTest { + /** {@inheritDoc} */ + @Override protected BinaryObjectEx toPortable(PortableMarshaller marsh, Object obj) throws Exception { + byte[] bytes = marsh.marshal(obj); + + return new BinaryObjectImpl(ctx, bytes, 0); + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/0b4a8f83/modules/core/src/test/java/org/apache/ignite/internal/portable/BinaryFooterOffsetsOffheapSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/portable/BinaryFooterOffsetsOffheapSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/portable/BinaryFooterOffsetsOffheapSelfTest.java new file mode 100644 index 0000000..e52ebe7 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/portable/BinaryFooterOffsetsOffheapSelfTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.portable; + +import org.apache.ignite.internal.util.GridUnsafe; +import org.apache.ignite.marshaller.portable.PortableMarshaller; +import org.eclipse.jetty.util.ConcurrentHashSet; +import sun.misc.Unsafe; + +/** + * Compact offsets tests for offheap portable objects. + */ +public class BinaryFooterOffsetsOffheapSelfTest extends BinaryFooterOffsetsAbstractSelfTest { + /** Unsafe instance. */ + private static final Unsafe UNSAFE = GridUnsafe.unsafe(); + + /** Byte array offset for unsafe mechanics. */ + protected static final long BYTE_ARR_OFF = UNSAFE.arrayBaseOffset(byte[].class); + + /** Allocated unsafe pointer. */ + private final ConcurrentHashSet<Long> ptrs = new ConcurrentHashSet<>(); + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + // Cleanup allocated objects. + for (Long ptr : ptrs) + UNSAFE.freeMemory(ptr); + + ptrs.clear(); + } + + /** {@inheritDoc} */ + @Override protected BinaryObjectEx toPortable(PortableMarshaller marsh, Object obj) throws Exception { + byte[] arr = marsh.marshal(obj); + + long ptr = UNSAFE.allocateMemory(arr.length); + + ptrs.add(ptr); + + UNSAFE.copyMemory(arr, BYTE_ARR_OFF, null, ptr, arr.length); + + return new BinaryObjectOffheapImpl(ctx, ptr, 0, arr.length); + } +}