IGNITE-3098: UTF-16 surrogate pairs are not properly serialized by BinaryMarshaller
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a5a446c3 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a5a446c3 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a5a446c3 Branch: refs/heads/ignite-3163 Commit: a5a446c36b8f44367ba6ea2798080431812165b2 Parents: 08d79ff Author: Denis Magda <[email protected]> Authored: Tue May 17 15:46:27 2016 +0300 Committer: Denis Magda <[email protected]> Committed: Tue May 17 16:24:34 2016 +0300 ---------------------------------------------------------------------- .../apache/ignite/IgniteSystemProperties.java | 18 ++ .../apache/ignite/internal/IgniteKernal.java | 7 + .../ignite/internal/IgniteNodeAttributes.java | 4 + .../ignite/internal/binary/BinaryUtils.java | 176 +++++++++++++++-- .../internal/binary/BinaryWriterExImpl.java | 7 +- .../discovery/GridDiscoveryManager.java | 37 +++- .../ignite/spi/discovery/tcp/ServerImpl.java | 189 ++++++++++++------- .../binary/BinaryMarshallerSelfTest.java | 158 +++++++++++----- .../GridDiscoveryManagerAttributesSelfTest.java | 63 +++++++ 9 files changed, 524 insertions(+), 135 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/a5a446c3/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index 7d48608..db1d093 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -392,6 +392,24 @@ public final class IgniteSystemProperties { "IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID"; /** + * Manages type of serialization mechanism for {@link String} that is marshalled/unmarshalled by BinaryMarshaller. + * Should be used for cases when a String contains a surrogate symbol without its pair one. This is frequently used + * in algorithms that encrypts data in String format. + */ + public static final String IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2 = + "IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2"; + + /** + * If set to {@code true}, then default selected keys set is used inside + * {@code GridNioServer} which lead to some extra garbage generation when + * processing selected keys. + * <p> + * Default value is {@code false}. Should be switched to {@code true} if there are + * any problems in communication layer. + */ + public static final String IGNITE_NO_SELECTOR_OPTS = "IGNITE_NO_SELECTOR_OPTS"; + + /** * System property to specify period in milliseconds between calls of the SQL statements cache cleanup task. * <p> * Cleanup tasks clears cache for terminated threads and for threads which did not perform SQL queries within http://git-wip-us.apache.org/repos/asf/ignite/blob/a5a446c3/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index 16df367..194d12b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -83,6 +83,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.internal.binary.BinaryEnumCache; import org.apache.ignite.internal.binary.BinaryMarshaller; +import org.apache.ignite.internal.binary.BinaryUtils; import org.apache.ignite.internal.cluster.ClusterGroupAdapter; import org.apache.ignite.internal.cluster.IgniteClusterEx; import org.apache.ignite.internal.managers.GridManager; @@ -176,6 +177,7 @@ import static org.apache.ignite.IgniteSystemProperties.IGNITE_OPTIMIZED_MARSHALL import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK; import static org.apache.ignite.IgniteSystemProperties.IGNITE_STARVATION_CHECK_INTERVAL; import static org.apache.ignite.IgniteSystemProperties.IGNITE_SUCCESS_FILE; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2; import static org.apache.ignite.IgniteSystemProperties.getBoolean; import static org.apache.ignite.IgniteSystemProperties.snapshot; import static org.apache.ignite.internal.GridKernalState.DISCONNECTED; @@ -203,6 +205,7 @@ import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MACS; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_COMPACT_FOOTER; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID; +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CONSISTENT_ID; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PEER_CLASSLOADING; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PHY_RAM; @@ -1251,6 +1254,10 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable { add(ATTR_MARSHALLER_COMPACT_FOOTER, cfg.getBinaryConfiguration() == null ? BinaryConfiguration.DFLT_COMPACT_FOOTER : cfg.getBinaryConfiguration().isCompactFooter()); + + add(ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2, + getBoolean(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2, + BinaryUtils.USE_STR_SERIALIZATION_VER_2)); } add(ATTR_USER_NAME, System.getProperty("user.name")); http://git-wip-us.apache.org/repos/asf/ignite/blob/a5a446c3/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java index da6f40d..744439c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java @@ -39,6 +39,10 @@ public final class IgniteNodeAttributes { /** Attribute for marshaller compact footers. */ public static final String ATTR_MARSHALLER_COMPACT_FOOTER = ATTR_PREFIX + ".marshaller.compactFooter"; + /** Internal attribute constant that controls which String serialization version to use. */ + public static final String ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2 = ATTR_PREFIX + + ".marshaller.utf8SerializationVer2"; + /** Internal attribute name constant. */ public static final String ATTR_JIT_NAME = ATTR_PREFIX + ".jit.name"; http://git-wip-us.apache.org/repos/asf/ignite/blob/a5a446c3/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java index 37f1d6a..e431474 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java @@ -46,6 +46,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentSkipListSet; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.binary.BinaryCollectionFactory; import org.apache.ignite.binary.BinaryInvalidTypeException; import org.apache.ignite.binary.BinaryMapFactory; @@ -63,6 +64,7 @@ import org.apache.ignite.lang.IgniteUuid; import org.jetbrains.annotations.Nullable; import org.jsr166.ConcurrentHashMap8; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2; import static java.nio.charset.StandardCharsets.UTF_8; /** @@ -75,6 +77,10 @@ public class BinaryUtils { /** */ public static final Map<Byte, Class<?>> FLAG_TO_CLASS = new HashMap<>(); + /** */ + public static final boolean USE_STR_SERIALIZATION_VER_2 = IgniteSystemProperties.getBoolean( + IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2, false); + /** {@code true} if serialized value of this type cannot contain references to objects. */ private static final boolean[] PLAIN_TYPE_FLAG = new boolean[102]; @@ -415,7 +421,7 @@ public class BinaryUtils { break; case GridBinaryMarshaller.TIMESTAMP: - writer.doWriteTimestamp((Timestamp) val); + writer.doWriteTimestamp((Timestamp)val); break; @@ -614,7 +620,8 @@ public class BinaryUtils { } /** - * Attempts to create a new map of the same type as {@code map} has. Otherwise returns new {@code HashMap} instance. + * Attempts to create a new map of the same type as {@code map} has. Otherwise returns new {@code HashMap} + * instance. * * @param map Original map. * @return New map. @@ -648,8 +655,7 @@ public class BinaryUtils { } /** - * Attempts to create a new collection of the same known type. Will return null if collection type is - * unknown. + * Attempts to create a new collection of the same known type. Will return null if collection type is unknown. * * @param col Collection. * @return New empty collection. @@ -674,7 +680,8 @@ public class BinaryUtils { } /** - * Attempts to create a new set of the same type as {@code set} has. Otherwise returns new {@code HashSet} instance. + * Attempts to create a new set of the same type as {@code set} has. Otherwise returns new {@code HashSet} + * instance. * * @param set Original set. * @return New set. @@ -780,7 +787,7 @@ public class BinaryUtils { int len = length(in, start); - if (hasSchema(flags)){ + if (hasSchema(flags)) { // Schema exists. if (hasRaw(flags)) // Raw offset is set, it is at the very end of the object. @@ -1150,15 +1157,28 @@ public class BinaryUtils { * @return Value. */ public static String doReadString(BinaryInputStream in) { - if (!in.hasArray()) - return new String(doReadByteArray(in), UTF_8); + if (!in.hasArray()) { + byte[] arr = doReadByteArray(in); + + if (USE_STR_SERIALIZATION_VER_2) + return utf8BytesToStr(arr, 0, arr.length); + else + return new String(arr, UTF_8); + } int strLen = in.readInt(); int pos = in.position(); // String will copy necessary array part for us. - String res = new String(in.array(), pos, strLen, UTF_8); + String res; + + if (USE_STR_SERIALIZATION_VER_2) { + res = utf8BytesToStr(in.array(), pos, strLen); + } + else { + res = new String(in.array(), pos, strLen, UTF_8); + } in.position(pos + strLen); @@ -1485,7 +1505,7 @@ public class BinaryUtils { private static Object[] doReadBinaryEnumArray(BinaryInputStream in, BinaryContext ctx) { int len = in.readInt(); - Object[] arr = (Object[]) Array.newInstance(BinaryObject.class, len); + Object[] arr = (Object[])Array.newInstance(BinaryObject.class, len); for (int i = 0; i < len; i++) { byte flag = in.readByte(); @@ -1524,7 +1544,7 @@ public class BinaryUtils { throws BinaryObjectException { int len = in.readInt(); - Object[] arr = (Object[]) Array.newInstance(cls, len); + Object[] arr = (Object[])Array.newInstance(cls, len); for (int i = 0; i < len; i++) { byte flag = in.readByte(); @@ -2013,6 +2033,140 @@ public class BinaryUtils { } /** + * Reconstructs string from UTF-8 bytes. + * + * @param arr array Byte array. + * @param off offset Offset in the array. + * @param len length Byte array lenght. + * @return string Resulting string. + */ + public static String utf8BytesToStr(byte[] arr, int off, int len) { + int c, charArrCnt = 0, total = off + len; + int c2, c3; + char[] res = new char[len]; + + // try reading ascii + while (off < total) { + c = (int)arr[off] & 0xff; + + if (c > 127) + break; + + off++; + + res[charArrCnt++] = (char)c; + } + + // read other + while (off < total) { + c = (int)arr[off] & 0xff; + + switch (c >> 4) { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + /* 0xxxxxxx*/ + off++; + + res[charArrCnt++] = (char)c; + + break; + case 12: + case 13: + /* 110x xxxx 10xx xxxx*/ + off += 2; + + if (off > total) + throw new BinaryObjectException("Malformed input: partial character at end"); + + c2 = (int)arr[off - 1]; + + if ((c2 & 0xC0) != 0x80) + throw new BinaryObjectException("Malformed input around byte: " + off); + + res[charArrCnt++] = (char)(((c & 0x1F) << 6) | (c2 & 0x3F)); + + break; + case 14: + /* 1110 xxxx 10xx xxxx 10xx xxxx */ + off += 3; + + if (off > total) + throw new BinaryObjectException("Malformed input: partial character at end"); + + c2 = (int)arr[off - 2]; + + c3 = (int)arr[off - 1]; + + if (((c2 & 0xC0) != 0x80) || ((c3 & 0xC0) != 0x80)) + throw new BinaryObjectException("Malformed input around byte: " + (off - 1)); + + res[charArrCnt++] = (char)(((c & 0x0F) << 12) | + ((c2 & 0x3F) << 6) | + ((c3 & 0x3F) << 0)); + + break; + default: + /* 10xx xxxx, 1111 xxxx */ + throw new BinaryObjectException("Malformed input around byte: " + off); + } + } + + return len == charArrCnt ? new String(res) : new String(res, 0, charArrCnt); + } + + /** + * Converts the string into UTF-8 byte array considering special symbols like the surrogates. + * + * @param val String to convert. + * @return Resulting byte array. + */ + public static byte[] strToUtf8Bytes(String val) { + int strLen = val.length(); + int utfLen = 0; + int c, cnt; + + // Determine length of resulting byte array. + for (cnt = 0; cnt < strLen; cnt++) { + c = val.charAt(cnt); + + if (c >= 0x0001 && c <= 0x007F) + utfLen++; + else if (c > 0x07FF) + utfLen += 3; + else + utfLen += 2; + } + + byte[] arr = new byte[utfLen]; + + int position = 0; + + for (cnt = 0; cnt < strLen; cnt++) { + c = val.charAt(cnt); + + if (c >= 0x0001 && c <= 0x007F) + arr[position++] = (byte)c; + else if (c > 0x07FF) { + arr[position++] = (byte)(0xE0 | (c >> 12) & 0x0F); + arr[position++] = (byte)(0x80 | (c >> 6) & 0x3F); + arr[position++] = (byte)(0x80 | (c & 0x3F)); + } + else { + arr[position++] = (byte)(0xC0 | ((c >> 6) & 0x1F)); + arr[position++] = (byte)(0x80 | (c & 0x3F)); + } + } + + return arr; + } + + /** * Enum type. */ private static class EnumType { http://git-wip-us.apache.org/repos/asf/ignite/blob/a5a446c3/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java index 8060a13..30710f4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java @@ -379,7 +379,12 @@ public class BinaryWriterExImpl implements BinaryWriter, BinaryRawWriterEx, Obje if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - byte[] strArr = val.getBytes(UTF_8); + byte[] strArr; + + if (BinaryUtils.USE_STR_SERIALIZATION_VER_2) + strArr = BinaryUtils.strToUtf8Bytes(val); + else + strArr = val.getBytes(UTF_8); out.unsafeEnsure(1 + 4); out.unsafeWriteByte(GridBinaryMarshaller.STRING); http://git-wip-us.apache.org/repos/asf/ignite/blob/a5a446c3/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java index b55ffb0..2779d6f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java @@ -119,6 +119,7 @@ import org.jsr166.ConcurrentHashMap8; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.ignite.IgniteSystemProperties.IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2; import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED; import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_RECONNECTED; import static org.apache.ignite.events.EventType.EVT_NODE_FAILED; @@ -128,6 +129,7 @@ import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED; import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DEPLOYMENT_MODE; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MACS; +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PEER_CLASSLOADING; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_USER_NAME; @@ -385,7 +387,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> { @Override protected void onKernalStart0() throws IgniteCheckedException { if (Boolean.TRUE.equals(ctx.config().isClientMode()) && !getSpi().isClientMode()) ctx.performance().add("Enable client mode for TcpDiscoverySpi " + - "(set TcpDiscoverySpi.forceServerMode to false)"); + "(set TcpDiscoverySpi.forceServerMode to false)"); } /** {@inheritDoc} */ @@ -725,7 +727,8 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> { * @param msgCls Message class. * @param lsnr Custom event listener. */ - public <T extends DiscoveryCustomMessage> void setCustomEventListener(Class<T> msgCls, CustomEventListener<T> lsnr) { + public <T extends DiscoveryCustomMessage> void setCustomEventListener(Class<T> msgCls, + CustomEventListener<T> lsnr) { List<CustomEventListener<DiscoveryCustomMessage>> list = customEvtLsnrs.get(msgCls); if (list == null) { @@ -998,6 +1001,11 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> { Boolean locMarshUseDfltSuid = locNode.attribute(ATTR_MARSHALLER_USE_DFLT_SUID); boolean locMarshUseDfltSuidBool = locMarshUseDfltSuid == null ? true : locMarshUseDfltSuid; + Boolean locMarshStrSerVer2 = locNode.attribute(ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2); + boolean locMarshStrSerVer2Bool = locMarshStrSerVer2 == null ? + false /* turned on and added to the attributes list by default only when BinaryMarshaller is used. */ : + locMarshStrSerVer2; + for (ClusterNode n : nodes) { int rmtJvmMajVer = nodeJavaMajorVersion(n); @@ -1013,11 +1021,11 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> { if (!F.eq(rmtPreferIpV4, locPreferIpV4)) { if (!ipV4Warned) U.warn(log, "Local node's value of 'java.net.preferIPv4Stack' " + - "system property differs from remote node's " + - "(all nodes in topology should have identical value) " + - "[locPreferIpV4=" + locPreferIpV4 + ", rmtPreferIpV4=" + rmtPreferIpV4 + - ", locId8=" + U.id8(locNode.id()) + ", rmtId8=" + U.id8(n.id()) + - ", rmtAddrs=" + U.addressesAsString(n) + ']', + "system property differs from remote node's " + + "(all nodes in topology should have identical value) " + + "[locPreferIpV4=" + locPreferIpV4 + ", rmtPreferIpV4=" + rmtPreferIpV4 + + ", locId8=" + U.id8(locNode.id()) + ", rmtId8=" + U.id8(n.id()) + + ", rmtAddrs=" + U.addressesAsString(n) + ']', "Local and remote 'java.net.preferIPv4Stack' system properties do not match."); ipV4Warned = true; @@ -1056,6 +1064,21 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> { ", rmtNodeAddrs=" + U.addressesAsString(n) + ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ']'); } + + Boolean rmtMarshStrSerVer2 = n.attribute(ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2); + boolean rmtMarshStrSerVer2Bool = rmtMarshStrSerVer2 == null ? false : rmtMarshStrSerVer2; + + if (locMarshStrSerVer2Bool != rmtMarshStrSerVer2Bool) { + throw new IgniteCheckedException("Local node's " + IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2 + + " property value differs from remote node's value " + + "(to make sure all nodes in topology have identical marshaller settings, " + + "configure system property explicitly) " + + "[locMarshStrSerVer2=" + locMarshStrSerVer2 + ", rmtMarshStrSerVer2=" + rmtMarshStrSerVer2 + + ", locNodeAddrs=" + U.addressesAsString(locNode) + + ", rmtNodeAddrs=" + U.addressesAsString(n) + + ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ']'); + } + } if (log.isDebugEnabled()) http://git-wip-us.apache.org/repos/asf/ignite/blob/a5a446c3/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java index e30dd24..7e0e17f 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java @@ -131,6 +131,7 @@ import org.jsr166.ConcurrentHashMap8; import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISCOVERY_HISTORY_SIZE; import static org.apache.ignite.IgniteSystemProperties.IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2; import static org.apache.ignite.IgniteSystemProperties.getInteger; import static org.apache.ignite.events.EventType.EVT_NODE_FAILED; import static org.apache.ignite.events.EventType.EVT_NODE_JOINED; @@ -139,6 +140,7 @@ import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED; import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_COMPACT_FOOTER; +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID; import static org.apache.ignite.spi.IgnitePortProtocol.TCP; import static org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoverySpiState.AUTH_FAILED; @@ -570,8 +572,8 @@ class ServerImpl extends TcpDiscoveryImpl { * @param addr Address of the node. * @param nodeId Node ID to ping. In case when client node ID is not null this node ID is an ID of the router node. * @param clientNodeId Client node ID. - * @return ID of the remote node and "client exists" flag if node alive or {@code null} if the remote node has - * left a topology during the ping process. + * @return ID of the remote node and "client exists" flag if node alive or {@code null} if the remote node has left + * a topology during the ping process. * @throws IgniteCheckedException If an error occurs. */ private @Nullable IgniteBiTuple<UUID, Boolean> pingNode(InetSocketAddress addr, @Nullable UUID nodeId, @@ -910,9 +912,9 @@ class ServerImpl extends TcpDiscoveryImpl { } /** - * Tries to send join request message to a random node presenting in topology. - * Address is provided by {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder} and message is - * sent to first node connection succeeded to. + * Tries to send join request message to a random node presenting in topology. Address is provided by {@link + * org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder} and message is sent to first node connection + * succeeded to. * * @return {@code true} if send succeeded. * @throws IgniteSpiException If any error occurs. @@ -1071,7 +1073,7 @@ class ServerImpl extends TcpDiscoveryImpl { int reconCnt = 0; - while (true){ + while (true) { // Need to set to false on each new iteration, // since remote node may leave in the middle of the first iteration. joinReqSent = false; @@ -1297,7 +1299,8 @@ class ServerImpl extends TcpDiscoveryImpl { * @param top Topology snapshot. * @return Copy of updated topology history. */ - @Nullable private Map<Long, Collection<ClusterNode>> updateTopologyHistory(long topVer, Collection<ClusterNode> top) { + @Nullable private Map<Long, Collection<ClusterNode>> updateTopologyHistory(long topVer, + Collection<ClusterNode> top) { synchronized (mux) { if (topHist.containsKey(topVer)) return null; @@ -1315,8 +1318,8 @@ class ServerImpl extends TcpDiscoveryImpl { } /** - * Checks whether local node is coordinator. Nodes that are leaving or failed - * (but are still in topology) are removed from search. + * Checks whether local node is coordinator. Nodes that are leaving or failed (but are still in topology) are + * removed from search. * * @return {@code true} if local node is coordinator. */ @@ -1345,23 +1348,22 @@ class ServerImpl extends TcpDiscoveryImpl { } /** - * Resolves coordinator. Nodes that are leaving or failed (but are still in - * topology) are removed from search. + * Resolves coordinator. Nodes that are leaving or failed (but are still in topology) are removed from search. * - * @return Coordinator node or {@code null} if there are no coordinator - * (i.e. local node is the last one and is currently stopping). + * @return Coordinator node or {@code null} if there are no coordinator (i.e. local node is the last one and is + * currently stopping). */ @Nullable private TcpDiscoveryNode resolveCoordinator() { return resolveCoordinator(null); } /** - * Resolves coordinator. Nodes that are leaving or failed (but are still in - * topology) are removed from search as well as provided filter. + * Resolves coordinator. Nodes that are leaving or failed (but are still in topology) are removed from search as + * well as provided filter. * * @param filter Nodes to exclude when resolving coordinator (optional). - * @return Coordinator node or {@code null} if there are no coordinator - * (i.e. local node is the last one and is currently stopping). + * @return Coordinator node or {@code null} if there are no coordinator (i.e. local node is the last one and is + * currently stopping). */ @Nullable private TcpDiscoveryNode resolveCoordinator( @Nullable Collection<TcpDiscoveryNode> filter) { @@ -1417,7 +1419,7 @@ class ServerImpl extends TcpDiscoveryImpl { @Nullable Collection<TcpDiscoveryAbstractMessage> msgs, @Nullable IgniteUuid discardMsgId, @Nullable IgniteUuid discardCustomMsgId - ) { + ) { assert destNodeId != null; if (msg instanceof TcpDiscoveryNodeAddedMessage) { @@ -1511,13 +1513,9 @@ class ServerImpl extends TcpDiscoveryImpl { } /** - * <strong>FOR TEST ONLY!!!</strong> - * <p> - * Simulates situation when next node is still alive but is bypassed - * since it has been excluded from the ring, possibly, due to short time - * network problems. - * <p> - * This method is intended for test purposes only. + * <strong>FOR TEST ONLY!!!</strong> <p> Simulates situation when next node is still alive but is bypassed since it + * has been excluded from the ring, possibly, due to short time network problems. <p> This method is intended for + * test purposes only. */ void forceNextNodeFailure() { U.warn(log, "Next node will be forcibly failed (if any)."); @@ -1534,9 +1532,7 @@ class ServerImpl extends TcpDiscoveryImpl { } /** - * <strong>FOR TEST ONLY!!!</strong> - * <p> - * This method is intended for test purposes only. + * <strong>FOR TEST ONLY!!!</strong> <p> This method is intended for test purposes only. * * @return Nodes ring. */ @@ -1621,8 +1617,8 @@ class ServerImpl extends TcpDiscoveryImpl { } /** - * Checks if two given {@link SecurityPermissionSet} objects contain the same permissions. - * Each permission belongs to one of three groups : cache, task or system. + * Checks if two given {@link SecurityPermissionSet} objects contain the same permissions. Each permission belongs + * to one of three groups : cache, task or system. * * @param locPerms The first set of permissions. * @param rmtPerms The second set of permissions. @@ -1653,11 +1649,9 @@ class ServerImpl extends TcpDiscoveryImpl { } /** - * Thread that cleans IP finder and keeps it in the correct state, unregistering - * addresses of the nodes that has left the topology. - * <p> - * This thread should run only on coordinator node and will clean IP finder - * if and only if {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder#isShared()} is {@code true}. + * Thread that cleans IP finder and keeps it in the correct state, unregistering addresses of the nodes that has + * left the topology. <p> This thread should run only on coordinator node and will clean IP finder if and only if + * {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder#isShared()} is {@code true}. */ private class IpFinderCleaner extends IgniteSpiThread { /** @@ -1848,17 +1842,16 @@ class ServerImpl extends TcpDiscoveryImpl { } /** - * Gets messages starting from provided ID (exclusive). If such - * message is not found, {@code null} is returned (this indicates - * a failure condition when it was already removed from queue). + * Gets messages starting from provided ID (exclusive). If such message is not found, {@code null} is returned + * (this indicates a failure condition when it was already removed from queue). * - * @param lastMsgId Last message ID received on client. {@code Null} if client did not finish connect procedure. + * @param lastMsgId Last message ID received on client. {@code Null} if client did not finish connect + * procedure. * @param node Client node. * @return Collection of messages. */ @Nullable Collection<TcpDiscoveryAbstractMessage> messages(@Nullable IgniteUuid lastMsgId, - TcpDiscoveryNode node) - { + TcpDiscoveryNode node) { assert node != null && node.isClient() : node; if (lastMsgId == null) { @@ -1960,8 +1953,8 @@ class ServerImpl extends TcpDiscoveryImpl { private IgniteUuid customDiscardId; /** - * Adds pending message and shrinks queue if it exceeds limit - * (messages that were not discarded yet are never removed). + * Adds pending message and shrinks queue if it exceeds limit (messages that were not discarded yet are never + * removed). * * @param msg Message to add. */ @@ -2468,10 +2461,11 @@ class ServerImpl extends TcpDiscoveryImpl { List<InetSocketAddress> locNodeAddrs = U.arrayList(locNode.socketAddresses()); - addr: for (InetSocketAddress addr : spi.getNodeAddresses(next, sameHost)) { + addr: + for (InetSocketAddress addr : spi.getNodeAddresses(next, sameHost)) { long ackTimeout0 = spi.getAckTimeout(); - if (locNodeAddrs.contains(addr)){ + if (locNodeAddrs.contains(addr)) { if (log.isDebugEnabled()) log.debug("Skip to send message to the local node (probably remote node has the same " + "loopback address that local node): " + addr); @@ -2636,10 +2630,10 @@ class ServerImpl extends TcpDiscoveryImpl { assert !forceSndPending || msg instanceof TcpDiscoveryNodeLeftMessage; - boolean sndPending= + boolean sndPending = (newNextNode && ring.minimumNodeVersion().compareTo(CUSTOM_MSG_ALLOW_JOINING_FOR_VERIFIED_SINCE) >= 0) || - failure || - forceSndPending; + failure || + forceSndPending; if (sndPending) { if (log.isDebugEnabled()) @@ -2870,8 +2864,8 @@ class ServerImpl extends TcpDiscoveryImpl { } LT.warn(log, null, "Local node has detected failed nodes and started cluster-wide procedure. " + - "To speed up failure detection please see 'Failure Detection' section under javadoc" + - " for 'TcpDiscoverySpi'"); + "To speed up failure detection please see 'Failure Detection' section under javadoc" + + " for 'TcpDiscoverySpi'"); } } @@ -2902,8 +2896,8 @@ class ServerImpl extends TcpDiscoveryImpl { } /** - * Checks whether pending messages queue contains unprocessed {@link TcpDiscoveryNodeAddedMessage} for - * the node with {@code nodeId}. + * Checks whether pending messages queue contains unprocessed {@link TcpDiscoveryNodeAddedMessage} for the node + * with {@code nodeId}. * * @param nodeId Node ID. * @return {@code true} if contains, {@code false} otherwise. @@ -3344,6 +3338,62 @@ class ServerImpl extends TcpDiscoveryImpl { return; } + // Validate String serialization mechanism used by the BinaryMarshaller. + final Boolean locMarshStrSerialVer2 = locNode.attribute(ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2); + final boolean locMarshStrSerialVer2Bool = locMarshStrSerialVer2 != null ? locMarshStrSerialVer2 : false; + + final Boolean rmtMarshStrSerialVer2 = node.attribute(ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2); + final boolean rmtMarshStrSerialVer2Bool = rmtMarshStrSerialVer2 != null ? rmtMarshStrSerialVer2 : false; + + if (locMarshStrSerialVer2Bool != rmtMarshStrSerialVer2Bool) { + utilityPool.submit( + new Runnable() { + @Override public void run() { + String errMsg = "Local node's " + IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2 + + " property value differs from remote node's value " + + "(to make sure all nodes in topology have identical marshaller settings, " + + "configure system property explicitly) " + + "[locMarshStrSerialVer2=" + locMarshStrSerialVer2 + + ", rmtMarshStrSerialVer2=" + rmtMarshStrSerialVer2 + + ", locNodeAddrs=" + U.addressesAsString(locNode) + + ", rmtNodeAddrs=" + U.addressesAsString(node) + + ", locNodeId=" + locNode.id() + ", rmtNodeId=" + msg.creatorNodeId() + ']'; + + + LT.warn(log, null, errMsg); + + // Always output in debug. + if (log.isDebugEnabled()) + log.debug(errMsg); + + String sndMsg = "Local node's " + IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2 + + " property value differs from remote node's value " + + "(to make sure all nodes in topology have identical marshaller settings, " + + "configure system property explicitly) " + + "[locMarshStrSerialVer2=" + rmtMarshStrSerialVer2 + + ", rmtMarshStrSerialVer2=" + locMarshStrSerialVer2 + + ", locNodeAddrs=" + U.addressesAsString(node) + ", locPort=" + node.discoveryPort() + + ", rmtNodeAddr=" + U.addressesAsString(locNode) + ", locNodeId=" + node.id() + + ", rmtNodeId=" + locNode.id() + ']'; + + try { + trySendMessageDirectly(node, new TcpDiscoveryCheckFailedMessage(locNodeId, sndMsg)); + } + catch (IgniteSpiException e) { + if (log.isDebugEnabled()) + log.debug("Failed to send marshaller check failed message to node " + + "[node=" + node + ", err=" + e.getMessage() + ']'); + + onException("Failed to send marshaller check failed message to node " + + "[node=" + node + ", err=" + e.getMessage() + ']', e); + } + } + }); + + // Ignore join request. + return; + } + // Handle join. node.internalOrder(ring.nextNodeOrder()); @@ -3517,9 +3567,8 @@ class ServerImpl extends TcpDiscoveryImpl { * Processes node added message. * * @param msg Node added message. - * @deprecated Due to current protocol node add process cannot be dropped in the middle of the ring, - * if new node auth fails due to config inconsistency. So, we need to finish add - * and only then initiate failure. + * @deprecated Due to current protocol node add process cannot be dropped in the middle of the ring, if new node + * auth fails due to config inconsistency. So, we need to finish add and only then initiate failure. */ @Deprecated private void processNodeAddedMessage(TcpDiscoveryNodeAddedMessage msg) { @@ -3745,7 +3794,7 @@ class ServerImpl extends TcpDiscoveryImpl { return; } } - else { + else { if (log.isDebugEnabled()) log.debug("Discarding node added message (this message has already been processed) " + "[spiState=" + spiState + @@ -4527,8 +4576,7 @@ class ServerImpl extends TcpDiscoveryImpl { private void updateMetrics(UUID nodeId, ClusterMetrics metrics, Map<Integer, CacheMetrics> cacheMetrics, - long tstamp) - { + long tstamp) { assert nodeId != null; assert metrics != null; @@ -4606,7 +4654,8 @@ class ServerImpl extends TcpDiscoveryImpl { try { res = pingNode(msg.nodeToPing()); - } catch (IgniteSpiException e) { + } + catch (IgniteSpiException e) { log.error("Failed to ping node [nodeToPing=" + msg.nodeToPing() + ']', e); res = false; @@ -4724,14 +4773,14 @@ class ServerImpl extends TcpDiscoveryImpl { } /** - * Checks failed nodes list and sends {@link TcpDiscoveryNodeFailedMessage} if failed node - * is still in the ring. + * Checks failed nodes list and sends {@link TcpDiscoveryNodeFailedMessage} if failed node is still in the + * ring. */ private void checkFailedNodesList() { List<TcpDiscoveryNodeFailedMessage> msgs = null; synchronized (mux) { - for (Iterator<TcpDiscoveryNode> it = failedNodes.iterator(); it.hasNext();) { + for (Iterator<TcpDiscoveryNode> it = failedNodes.iterator(); it.hasNext(); ) { TcpDiscoveryNode node = it.next(); if (ring.node(node.id()) != null) { @@ -4889,10 +4938,8 @@ class ServerImpl extends TcpDiscoveryImpl { } /** - * Thread that accepts incoming TCP connections. - * <p> - * Tcp server will call provided closure when accepts incoming connection. - * From that moment server is no more responsible for the socket. + * Thread that accepts incoming TCP connections. <p> Tcp server will call provided closure when accepts incoming + * connection. From that moment server is no more responsible for the socket. */ private class TcpServer extends IgniteSpiThread { /** Socket TCP server listens to. */ @@ -5062,14 +5109,14 @@ class ServerImpl extends TcpDiscoveryImpl { if (log.isDebugEnabled()) log.debug("Unknown connection detected (is some other software connecting to " + "this Ignite port?" + - (!spi.isSslEnabled() ? " missed SSL configuration?" : "" ) + + (!spi.isSslEnabled() ? " missed SSL configuration?" : "") + ") " + "[rmtAddr=" + sock.getRemoteSocketAddress() + ", locAddr=" + sock.getLocalSocketAddress() + ']'); LT.warn(log, null, "Unknown connection detected (is some other software connecting to " + "this Ignite port?" + - (!spi.isSslEnabled() ? " missing SSL configuration on remote node?" : "" ) + + (!spi.isSslEnabled() ? " missing SSL configuration on remote node?" : "") + ") [rmtAddr=" + sock.getInetAddress() + ']', true); return; @@ -5184,7 +5231,7 @@ class ServerImpl extends TcpDiscoveryImpl { } catch (IOException e) { if (log.isDebugEnabled()) - U.error(log, "Caught exception on handshake [err=" + e +", sock=" + sock + ']', e); + U.error(log, "Caught exception on handshake [err=" + e + ", sock=" + sock + ']', e); if (X.hasCause(e, SSLException.class) && spi.isSslEnabled() && !spi.isNodeStopping0()) LT.warn(log, null, "Failed to initialize connection " + @@ -5211,9 +5258,9 @@ class ServerImpl extends TcpDiscoveryImpl { } catch (IgniteCheckedException e) { if (log.isDebugEnabled()) - U.error(log, "Caught exception on handshake [err=" + e +", sock=" + sock + ']', e); + U.error(log, "Caught exception on handshake [err=" + e + ", sock=" + sock + ']', e); - onException("Caught exception on handshake [err=" + e +", sock=" + sock + ']', e); + onException("Caught exception on handshake [err=" + e + ", sock=" + sock + ']', e); if (e.hasCause(SocketTimeoutException.class)) LT.warn(log, null, "Socket operation timed out on handshake " + http://git-wip-us.apache.org/repos/asf/ignite/blob/a5a446c3/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java index 0435168..c02d823 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java @@ -26,6 +26,7 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationHandler; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.lang.reflect.Modifier; import java.lang.reflect.Proxy; import java.math.BigDecimal; import java.math.BigInteger; @@ -90,10 +91,13 @@ import org.jetbrains.annotations.Nullable; import org.jsr166.ConcurrentHashMap8; import sun.misc.Unsafe; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2; import static org.apache.ignite.internal.binary.streams.BinaryMemoryAllocator.INSTANCE; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertNotEquals; +import static java.nio.charset.StandardCharsets.UTF_8; + /** * Binary marshaller tests. */ @@ -186,8 +190,69 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { /** * @throws Exception If failed. */ - public void testString() throws Exception { - assertEquals("str", marshalUnmarshal("str")); + public void testStringVer1() throws Exception { + doTestString(false); + } + + /** + * @throws Exception If failed. + */ + public void testStringVer2() throws Exception { + doTestString(true); + } + + /** + * @throws Exception If failed + */ + private void doTestString(boolean ver2) throws Exception { + // Ascii check. + String str = "ascii0123456789"; + assertEquals(str, marshalUnmarshal(str)); + + byte[] bytes = str.getBytes(UTF_8); + assertEquals(str, BinaryUtils.utf8BytesToStr(bytes, 0, bytes.length)); + + bytes = BinaryUtils.strToUtf8Bytes(str); + assertEquals(str, new String(bytes, UTF_8)); + + // Extended symbols set check set. + str = "ççabcdкиÑиллиÑа"; + assertEquals(str, marshalUnmarshal(str)); + + bytes = str.getBytes(UTF_8); + assertEquals(str, BinaryUtils.utf8BytesToStr(bytes, 0, bytes.length)); + + bytes = BinaryUtils.strToUtf8Bytes(str); + assertEquals(str, new String(bytes, UTF_8)); + + // Special symbols check. + str = new String(new char[] {0xD800, 'ç', 0xD800, 0xD800, 0xDC00, 0xDFFF}); + if (ver2) { + bytes = BinaryUtils.strToUtf8Bytes(str); + assertEquals(str, BinaryUtils.utf8BytesToStr(bytes, 0, bytes.length)); + } + else + assertNotEquals(str, marshalUnmarshal(str)); + + str = new String(new char[] {55296}); + if (ver2) { + bytes = BinaryUtils.strToUtf8Bytes(str); + assertEquals(str, BinaryUtils.utf8BytesToStr(bytes, 0, bytes.length)); + } + else + assertNotEquals(str, marshalUnmarshal(str)); + + bytes = str.getBytes(UTF_8); + assertNotEquals(str, new String(bytes, UTF_8)); + + bytes = str.getBytes(UTF_8); + assertNotEquals(str, BinaryUtils.utf8BytesToStr(bytes, 0, bytes.length)); + + str = new String(new char[] {0xD801, 0xDC37}); + assertEquals(str, marshalUnmarshal(str)); + + bytes = str.getBytes(UTF_8); + assertEquals(str, new String(bytes, UTF_8)); } /** @@ -298,7 +363,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { * @throws Exception If failed. */ public void testDecimalArray() throws Exception { - BigDecimal[] arr = new BigDecimal[] {BigDecimal.ZERO, BigDecimal.ONE, BigDecimal.TEN} ; + BigDecimal[] arr = new BigDecimal[] {BigDecimal.ZERO, BigDecimal.ONE, BigDecimal.TEN}; assertArrayEquals(arr, marshalUnmarshal(arr)); } @@ -919,9 +984,9 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { */ public void testClassWithoutPublicConstructor() throws Exception { BinaryMarshaller marsh = binaryMarshaller(Arrays.asList( - new BinaryTypeConfiguration(NoPublicConstructor.class.getName()), - new BinaryTypeConfiguration(NoPublicDefaultConstructor.class.getName()), - new BinaryTypeConfiguration(ProtectedConstructor.class.getName())) + new BinaryTypeConfiguration(NoPublicConstructor.class.getName()), + new BinaryTypeConfiguration(NoPublicDefaultConstructor.class.getName()), + new BinaryTypeConfiguration(ProtectedConstructor.class.getName())) ); NoPublicConstructor npc = new NoPublicConstructor(); @@ -1041,8 +1106,10 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { @Override public int fieldId(int typeId, String fieldName) { assert typeId == 44444; - if ("val1".equals(fieldName)) return 55555; - else if ("val2".equals(fieldName)) return 66666; + if ("val1".equals(fieldName)) + return 55555; + else if ("val2".equals(fieldName)) + return 66666; assert false : "Unknown field: " + fieldName; @@ -1410,14 +1477,14 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { BinaryMarshaller marsh = binaryMarshaller(new BinaryBasicNameMapper(true), new BinaryBasicIdMapper(true), Arrays.asList( - new BinaryTypeConfiguration(Key.class.getName()), - new BinaryTypeConfiguration("org.gridgain.NonExistentClass3"), - new BinaryTypeConfiguration("NonExistentClass4"), - customType1, - customType2, - customType3, - customType4 - )); + new BinaryTypeConfiguration(Key.class.getName()), + new BinaryTypeConfiguration("org.gridgain.NonExistentClass3"), + new BinaryTypeConfiguration("NonExistentClass4"), + customType1, + customType2, + customType3, + customType4 + )); BinaryContext ctx = binaryContext(marsh); @@ -1598,6 +1665,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { // BinaryIdMapper.typeId() contract. assertEquals("nonexistentclass0".hashCode(), ctx.typeId("NonExistentClass0")); } + /** * @throws Exception If failed. */ @@ -1700,7 +1768,6 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { assertEquals(992, ctx.typeId("org.gridgain.NonExistentClass3")); assertEquals(993, ctx.typeId("NonExistentClass4")); - // Custom types. assertEquals(300, ctx.typeId(Value.class.getName())); assertEquals(400, ctx.typeId("org.gridgain.NonExistentClass1")); @@ -1997,7 +2064,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { BinaryObject po = marshal(obj, marsh); - BinaryObject copy = copy(po, F.<String, Object>asMap("bArr", new byte[]{1, 2, 3})); + BinaryObject copy = copy(po, F.<String, Object>asMap("bArr", new byte[] {1, 2, 3})); assertArrayEquals(new byte[] {1, 2, 3}, copy.<byte[]>field("bArr")); @@ -2034,7 +2101,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { BinaryObject po = marshal(obj, marsh); - BinaryObject copy = copy(po, F.<String, Object>asMap("sArr", new short[]{1, 2, 3})); + BinaryObject copy = copy(po, F.<String, Object>asMap("sArr", new short[] {1, 2, 3})); assertArrayEquals(new short[] {1, 2, 3}, copy.<short[]>field("sArr")); @@ -2055,7 +2122,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { BinaryObject po = marshal(obj, marsh); - BinaryObject copy = copy(po, F.<String, Object>asMap("iArr", new int[]{1, 2, 3})); + BinaryObject copy = copy(po, F.<String, Object>asMap("iArr", new int[] {1, 2, 3})); assertArrayEquals(new int[] {1, 2, 3}, copy.<int[]>field("iArr")); @@ -2076,7 +2143,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { BinaryObject po = marshal(obj, marsh); - BinaryObject copy = copy(po, F.<String, Object>asMap("lArr", new long[]{1, 2, 3})); + BinaryObject copy = copy(po, F.<String, Object>asMap("lArr", new long[] {1, 2, 3})); assertArrayEquals(new long[] {1, 2, 3}, copy.<long[]>field("lArr")); @@ -2097,7 +2164,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { BinaryObject po = marshal(obj, marsh); - BinaryObject copy = copy(po, F.<String, Object>asMap("fArr", new float[]{1, 2, 3})); + BinaryObject copy = copy(po, F.<String, Object>asMap("fArr", new float[] {1, 2, 3})); assertArrayEquals(new float[] {1, 2, 3}, copy.<float[]>field("fArr"), 0); @@ -2118,7 +2185,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { BinaryObject po = marshal(obj, marsh); - BinaryObject copy = copy(po, F.<String, Object>asMap("dArr", new double[]{1, 2, 3})); + BinaryObject copy = copy(po, F.<String, Object>asMap("dArr", new double[] {1, 2, 3})); assertArrayEquals(new double[] {1, 2, 3}, copy.<double[]>field("dArr"), 0); @@ -2139,13 +2206,13 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { BinaryObject po = marshal(obj, marsh); - BinaryObject copy = copy(po, F.<String, Object>asMap("cArr", new char[]{1, 2, 3})); + BinaryObject copy = copy(po, F.<String, Object>asMap("cArr", new char[] {1, 2, 3})); - assertArrayEquals(new char[]{1, 2, 3}, copy.<char[]>field("cArr")); + assertArrayEquals(new char[] {1, 2, 3}, copy.<char[]>field("cArr")); SimpleObject obj0 = copy.deserialize(); - assertArrayEquals(new char[]{1, 2, 3}, obj0.cArr); + assertArrayEquals(new char[] {1, 2, 3}, obj0.cArr); } /** @@ -2160,7 +2227,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { BinaryObject po = marshal(obj, marsh); - BinaryObject copy = copy(po, F.<String, Object>asMap("strArr", new String[]{"str1", "str2"})); + BinaryObject copy = copy(po, F.<String, Object>asMap("strArr", new String[] {"str1", "str2"})); assertArrayEquals(new String[] {"str1", "str2"}, copy.<String[]>field("strArr")); @@ -2218,13 +2285,13 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { map.put("str", "str555"); map.put("inner", newObj); - map.put("bArr", new byte[]{6, 7, 9}); + map.put("bArr", new byte[] {6, 7, 9}); BinaryObject copy = copy(po, map); assertEquals("str555", copy.<String>field("str")); assertEquals(newObj, copy.<BinaryObject>field("inner").deserialize()); - assertArrayEquals(new byte[]{6, 7, 9}, copy.<byte[]>field("bArr")); + assertArrayEquals(new byte[] {6, 7, 9}, copy.<byte[]>field("bArr")); SimpleObject obj0 = copy.deserialize(); @@ -2447,7 +2514,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { DecimalReflective obj1 = new DecimalReflective(); obj1.val = BigDecimal.ZERO; - obj1.valArr = new BigDecimal[] { BigDecimal.ONE, BigDecimal.TEN }; + obj1.valArr = new BigDecimal[] {BigDecimal.ONE, BigDecimal.TEN}; BinaryObjectImpl portObj = marshal(obj1, marsh); @@ -2461,9 +2528,9 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { DecimalMarshalAware obj2 = new DecimalMarshalAware(); obj2.val = BigDecimal.ZERO; - obj2.valArr = new BigDecimal[] { BigDecimal.ONE, BigDecimal.TEN.negate() }; + obj2.valArr = new BigDecimal[] {BigDecimal.ONE, BigDecimal.TEN.negate()}; obj2.rawVal = BigDecimal.TEN; - obj2.rawValArr = new BigDecimal[] { BigDecimal.ZERO, BigDecimal.ONE }; + obj2.rawValArr = new BigDecimal[] {BigDecimal.ZERO, BigDecimal.ONE}; portObj = marshal(obj2, marsh); @@ -2795,7 +2862,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { /** * Some non-serializable class. */ - @SuppressWarnings( {"PublicField","TransientFieldInNonSerializableClass","FieldMayBeStatic"}) + @SuppressWarnings({"PublicField", "TransientFieldInNonSerializableClass", "FieldMayBeStatic"}) private static class NonSerializableA { /** */ private final long longVal = 0x33445566778899AAL; @@ -2804,7 +2871,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { protected Short shortVal = (short)0xAABB; /** */ - public String[] strArr = {"AA","BB"}; + public String[] strArr = {"AA", "BB"}; /** */ public boolean flag1 = true; @@ -2828,7 +2895,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { * @param strArr Array. * @param shortVal Short value. */ - @SuppressWarnings( {"UnusedDeclaration"}) + @SuppressWarnings({"UnusedDeclaration"}) private NonSerializableA(@Nullable String[] strArr, @Nullable Short shortVal) { // No-op. } @@ -2841,7 +2908,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { assertEquals(shortVal.shortValue(), (short)0xAABB); - assertTrue(Arrays.equals(strArr, new String[] {"AA","BB"})); + assertTrue(Arrays.equals(strArr, new String[] {"AA", "BB"})); assertEquals(0, intVal); @@ -2856,7 +2923,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { /** * Some non-serializable class. */ - @SuppressWarnings( {"PublicField","TransientFieldInNonSerializableClass","PackageVisibleInnerClass"}) + @SuppressWarnings({"PublicField", "TransientFieldInNonSerializableClass", "PackageVisibleInnerClass"}) static class NonSerializableB extends NonSerializableA { /** */ public Short shortValue = 0x1122; @@ -2900,7 +2967,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { /** * Some non-serializable class. */ - @SuppressWarnings( {"TransientFieldInNonSerializableClass","PublicField"}) + @SuppressWarnings({"TransientFieldInNonSerializableClass", "PublicField"}) private static class NonSerializable extends NonSerializableB { /** */ private int idVal = -17; @@ -2925,7 +2992,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { * * @param aVal Unused. */ - @SuppressWarnings( {"UnusedDeclaration"}) + @SuppressWarnings({"UnusedDeclaration"}) private NonSerializable(NonSerializableA aVal) { } @@ -3115,7 +3182,8 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { /** * */ - protected BinaryMarshaller binaryMarshaller(BinaryNameMapper nameMapper, BinaryIdMapper mapper, Collection<BinaryTypeConfiguration> cfgs) + protected BinaryMarshaller binaryMarshaller(BinaryNameMapper nameMapper, BinaryIdMapper mapper, + Collection<BinaryTypeConfiguration> cfgs) throws IgniteCheckedException { return binaryMarshaller(nameMapper, mapper, null, cfgs); } @@ -3257,7 +3325,6 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { outer.inner = inner; outer.bdArr = new BigDecimal[] {new BigDecimal(5000), BigDecimal.TEN}; - outer.col.add("str4"); outer.col.add("str5"); outer.col.add("str6"); @@ -4336,7 +4403,9 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { // No-op. } - /**n + /** + * n + * * @param key Key. */ private Key(int key) { @@ -4447,7 +4516,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { */ private static class ProtectedConstructor { /** - * Protected constructor. + * Protected constructor. */ protected ProtectedConstructor() { // No-op. @@ -4503,8 +4572,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { /** */ private String s; - /** Initializer. */ - { + /** Initializer. */ { StringBuilder builder = new StringBuilder(); for (int i = 0; i < 10000; i++) { http://git-wip-us.apache.org/repos/asf/ignite/blob/a5a446c3/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java index 3a2f3ba..28380df 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java @@ -21,6 +21,7 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.configuration.DeploymentMode; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.binary.BinaryMarshaller; import org.apache.ignite.marshaller.optimized.OptimizedMarshaller; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; @@ -28,6 +29,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.IgniteSystemProperties.IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2; import static org.apache.ignite.configuration.DeploymentMode.CONTINUOUS; import static org.apache.ignite.configuration.DeploymentMode.SHARED; @@ -47,6 +49,9 @@ public abstract class GridDiscoveryManagerAttributesSelfTest extends GridCommonA /** */ private static boolean p2pEnabled; + /** */ + private static boolean binaryMarshallerEnabled; + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(gridName); @@ -54,6 +59,9 @@ public abstract class GridDiscoveryManagerAttributesSelfTest extends GridCommonA if (gridName.equals(getTestGridName(1))) cfg.setClientMode(true); + if (binaryMarshallerEnabled) + cfg.setMarshaller(new BinaryMarshaller()); + cfg.setIncludeProperties(PREFER_IPV4); cfg.setDeploymentMode(mode); cfg.setPeerClassLoadingEnabled(p2pEnabled); @@ -161,6 +169,61 @@ public abstract class GridDiscoveryManagerAttributesSelfTest extends GridCommonA } } + public void testUseStringSerVer2() throws Exception { + String old = System.getProperty(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2); + + binaryMarshallerEnabled = true; + + try { + doTestUseStrSerVer2(Boolean.TRUE.toString(), Boolean.FALSE.toString(), true); + doTestUseStrSerVer2(Boolean.FALSE.toString(), Boolean.TRUE.toString(), true); + + doTestUseStrSerVer2(Boolean.TRUE.toString(), Boolean.TRUE.toString(), false); + doTestUseStrSerVer2(Boolean.FALSE.toString(), Boolean.FALSE.toString(), false); + } + finally { + if (old != null) + System.setProperty(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2, old); + else + System.clearProperty(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2); + + binaryMarshallerEnabled = false; + } + } + + /** + * @throws Exception If failed. + */ + private void doTestUseStrSerVer2(String first, String second, boolean fail) throws Exception { + try { + if (first != null) + System.setProperty(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2, first); + else + System.clearProperty(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2); + + startGrid(0); + + if (second != null) + System.setProperty(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2, second); + else + System.clearProperty(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2); + + try { + startGrid(1); + + if (fail) + fail("Node should not join"); + } + catch (Exception e) { + if (!fail) + fail("Node should join"); + } + } + finally { + stopAllGrids(); + } + } + /** * @throws Exception If failed. */
