This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 9019223a868 HIVE-29866: Optimize LLAP IO cache encoding path for text
tables with MultiDelimitSerDe (#6747)
9019223a868 is described below
commit 9019223a868a730e97e2694a99445511d57cbfda
Author: Bodor Laszlo <[email protected]>
AuthorDate: Tue Sep 8 15:52:51 2026 +0200
HIVE-29866: Optimize LLAP IO cache encoding path for text tables with
MultiDelimitSerDe (#6747)
---
.../io/encoded/VectorDeserializeOrcWriter.java | 70 ++-
.../io/encoded/TestVectorDeserializeOrcWriter.java | 136 ++++--
.../hive/serde2/lazy/LazySerDeParameters.java | 34 +-
.../lazy/fast/LazySimpleDeserializeRead.java | 312 +++++++++---
.../lazy/fast/TestLazySimpleDeserializeRead.java | 535 ++++++++++++++++++++-
5 files changed, 973 insertions(+), 114 deletions(-)
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java
index a55b64cdcbf..46e8a12bdb5 100644
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java
@@ -29,6 +29,7 @@
import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import java.nio.charset.StandardCharsets;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
@@ -55,6 +56,7 @@
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.Deserializer;
+import org.apache.hadoop.hive.serde2.MultiDelimitSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters;
@@ -102,33 +104,56 @@ public static EncodingWriter create(InputFormat<?, ?>
sourceIf, Deserializer ser
Configuration daemonConf, Configuration jobConf, Path splitPath,
StructObjectInspector sourceOi,
List<Integer> sourceIncludes, boolean[] cacheIncludes, int allocSize,
ExecutorService encodeExecutor)
throws IOException {
- // Vector SerDe can be disabled both on client and server side.
- if (!HiveConf.getBoolVar(daemonConf,
ConfVars.LLAP_IO_ENCODE_VECTOR_SERDE_ENABLED)
- || !HiveConf.getBoolVar(jobConf,
ConfVars.LLAP_IO_ENCODE_VECTOR_SERDE_ENABLED)
- || !(sourceIf instanceof TextInputFormat) || !(serDe instanceof
LazySimpleSerDe)) {
+ /*
+ * Vector SerDe can be disabled both on client and server side.
+ * MultiDelimitSerDe is accepted alongside LazySimpleSerDe: it's the same
+ * text-row shape, only its top-level FIELD_DELIM can span multiple bytes.
+ * The reader (LazySimpleDeserializeRead) picks the multi-byte scanner
+ * when LazySerDeParameters.setFieldDelimMulti has been called below —
+ * otherwise the specialized single-byte hot loop runs unchanged.
+ */
+ final boolean vectorSerdeEnabled =
+ HiveConf.getBoolVar(daemonConf,
ConfVars.LLAP_IO_ENCODE_VECTOR_SERDE_ENABLED)
+ && HiveConf.getBoolVar(jobConf,
ConfVars.LLAP_IO_ENCODE_VECTOR_SERDE_ENABLED);
+ final boolean isMultiDelim = serDe instanceof MultiDelimitSerDe;
+ if (!vectorSerdeEnabled
+ || !(sourceIf instanceof TextInputFormat)
+ || (!(serDe instanceof LazySimpleSerDe) && !isMultiDelim)) {
return new DeserializerOrcWriter(serDe, sourceOi, allocSize);
}
Path path = splitPath.getFileSystem(jobConf).makeQualified(splitPath);
PartitionDesc partDesc = HiveFileFormatUtils.getFromPathRecursively(parts,
path, null);
if (partDesc == null) {
- LlapIoImpl.LOG.info("Not using VertorDeserializeOrcWriter: no partition
desc for " + path);
+ LlapIoImpl.LOG.info("Not using VectorDeserializeOrcWriter: no partition
desc for {}", path);
return new DeserializerOrcWriter(serDe, sourceOi, allocSize);
}
Properties tblProps = partDesc.getTableDesc().getProperties();
if ("true".equalsIgnoreCase(tblProps.getProperty(
serdeConstants.SERIALIZATION_LAST_COLUMN_TAKES_REST))) {
- LlapIoImpl.LOG.info("Not using VertorDeserializeOrcWriter due to "
- + serdeConstants.SERIALIZATION_LAST_COLUMN_TAKES_REST);
+ LlapIoImpl.LOG.info("Not using VectorDeserializeOrcWriter due to {}",
+ serdeConstants.SERIALIZATION_LAST_COLUMN_TAKES_REST);
+ return new DeserializerOrcWriter(serDe, sourceOi, allocSize);
+ }
+ /*
+ * MultiDelimitSerDe's own row parser (LazyStruct.parseMultiDelimit) does
+ * NOT honour escape.delim for the top-level delimiter. Combining a
+ * multi-byte field delim with an escape char in the fast path would let
+ * it silently diverge from the slow path, so bail to DeserializerOrcWriter
+ * whenever both are configured on a MultiDelimit table.
+ */
+ if (isMultiDelim && tblProps.getProperty(serdeConstants.ESCAPE_CHAR) !=
null) {
+ LlapIoImpl.LOG.info("Not using VectorDeserializeOrcWriter:
MultiDelimitSerDe with {} "
+ + "is not supported by the vectorized fast path",
serdeConstants.ESCAPE_CHAR);
return new DeserializerOrcWriter(serDe, sourceOi, allocSize);
}
for (StructField sf : sourceOi.getAllStructFieldRefs()) {
Category c = sf.getFieldObjectInspector().getCategory();
if (c != Category.PRIMITIVE) {
- LlapIoImpl.LOG.info("Not using VertorDeserializeOrcWriter: " + c + "
is not supported");
+ LlapIoImpl.LOG.info("Not using VectorDeserializeOrcWriter: {} is not
supported", c);
return new DeserializerOrcWriter(serDe, sourceOi, allocSize);
}
}
- LlapIoImpl.LOG.info("Creating VertorDeserializeOrcWriter for " + path);
+ LlapIoImpl.LOG.info("Creating VectorDeserializeOrcWriter for {}", path);
return new VectorDeserializeOrcWriter(
jobConf, tblProps, sourceOi, sourceIncludes, cacheIncludes, allocSize,
encodeExecutor);
}
@@ -143,7 +168,8 @@ private VectorDeserializeOrcWriter(Configuration conf,
Properties tblProps, Stru
this.cacheIncludes = cacheIncludes;
this.sourceBatch = vrbCtx.createVectorizedRowBatch();
deserializeRead = new
LazySimpleDeserializeRead(vrbCtx.getRowColumnTypeInfos(),
- vrbCtx.getRowdataTypePhysicalVariations(),/* useExternalBuffer */ true,
createSerdeParams(conf, tblProps));
+ vrbCtx.getRowdataTypePhysicalVariations(), /* useExternalBuffer */ true,
+ createSerdeParams(conf, tblProps));
vectorDeserializeRow = new
VectorDeserializeRow<LazySimpleDeserializeRead>(deserializeRead);
int colCount = vrbCtx.getRowColumnTypeInfos().length;
boolean[] includes = null;
@@ -212,8 +238,9 @@ private static VectorizedRowBatchCtx
createVrbCtx(StructObjectInspector oi, fina
.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64");
final String serde =
tblProps.getProperty(serdeConstants.SERIALIZATION_LIB);
final String inputFormat =
tblProps.getProperty(hive_metastoreConstants.FILE_INPUT_FORMAT);
- final boolean isTextFormat = inputFormat != null &&
inputFormat.equals(TextInputFormat.class.getName()) &&
- serde != null && serde.equals(LazySimpleSerDe.class.getName());
+ final boolean isTextFormat =
TextInputFormat.class.getName().equals(inputFormat)
+ && (LazySimpleSerDe.class.getName().equals(serde)
+ || MultiDelimitSerDe.class.getName().equals(serde));
List<DataTypePhysicalVariation> dataTypePhysicalVariations = new
ArrayList<>();
if (isTextFormat) {
StructTypeInfo structTypeInfo = (StructTypeInfo)
TypeInfoUtils.getTypeInfoFromObjectInspector(oi);
@@ -247,7 +274,24 @@ private static VectorizedRowBatchCtx
createVrbCtx(StructObjectInspector oi, fina
private static LazySerDeParameters createSerdeParams(
Configuration conf, Properties tblProps) throws IOException {
try {
- return new LazySerDeParameters(conf, tblProps,
LazySimpleSerDe.class.getName());
+ LazySerDeParameters params =
+ new LazySerDeParameters(conf, tblProps,
LazySimpleSerDe.class.getName());
+ /*
+ * For MultiDelimitSerDe tables, if the field delimiter is > 1 byte, opt
+ * the reader into its multi-byte scan branch. LazySerDeParameters
+ * silently ignores single-byte values here, so misconfigured MultiDelim
+ * tables (e.g. FIELD_DELIM=",") still take the specialized single-byte
+ * fast path via separators[0]. The SerDe class is read from tblProps
+ * (same source of truth used by createVrbCtx above).
+ */
+ String serdeClass =
tblProps.getProperty(serdeConstants.SERIALIZATION_LIB);
+ if (MultiDelimitSerDe.class.getName().equals(serdeClass)) {
+ String rawDelim = tblProps.getProperty(serdeConstants.FIELD_DELIM);
+ if (rawDelim != null && rawDelim.length() > 1) {
+ params.setFieldDelimMulti(rawDelim.getBytes(StandardCharsets.UTF_8));
+ }
+ }
+ return params;
} catch (SerDeException e) {
throw new IOException(e);
}
diff --git
a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestVectorDeserializeOrcWriter.java
b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestVectorDeserializeOrcWriter.java
index beca9fa7ed8..45baff6db33 100644
---
a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestVectorDeserializeOrcWriter.java
+++
b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestVectorDeserializeOrcWriter.java
@@ -19,10 +19,14 @@
package org.apache.hadoop.hive.llap.io.encoded;
import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Properties;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch;
import org.apache.hadoop.hive.llap.io.api.impl.ColumnVectorBatch;
import org.apache.hadoop.hive.llap.io.decode.EncodedDataConsumer;
@@ -30,13 +34,20 @@
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.io.orc.WriterImpl;
import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.Deserializer;
+import org.apache.hadoop.hive.serde2.MultiDelimitSerDe;
+import org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
import org.apache.hive.common.util.FixedSizedObjectPool;
import org.apache.orc.impl.SchemaEvolution;
import org.junit.Test;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.CALLS_REAL_METHODS;
import static org.mockito.Mockito.mock;
@@ -49,34 +60,6 @@ public class TestVectorDeserializeOrcWriter {
private static final int TEST_NUM_COLS = 2;
- private static Field reflectField(Class<?> classToReflect, String
fieldNameValueToFetch) {
- try {
- Field reflectField = null;
- Class<?> classForReflect = classToReflect;
- do {
- try {
- reflectField =
classForReflect.getDeclaredField(fieldNameValueToFetch);
- } catch (NoSuchFieldException e) {
- classForReflect = classForReflect.getSuperclass();
- }
- } while (reflectField == null || classForReflect == null);
- reflectField.setAccessible(true);
- return reflectField;
- } catch (Exception e) {
- fail("Failed to reflect " + fieldNameValueToFetch + " from " +
classToReflect);
- }
- return null;
- }
-
- private static void reflectSetValue(Object objToReflect, String
fieldNameToSet, Object valueToSet) {
- try {
- Field reflectField = reflectField(objToReflect.getClass(),
fieldNameToSet);
- reflectField.set(objToReflect, valueToSet);
- } catch (Exception e) {
- fail("Failed to reflectively set " + fieldNameToSet + "=" + valueToSet);
- }
- }
-
@Test
public void testConcurrencyIssueWhileWriting() throws Exception {
@@ -123,6 +106,90 @@ public void testConcurrencyIssueWhileWriting() throws
Exception {
assertEquals(newCvb, cvb);
}
+ // --- createSerdeParams: MultiDelimit routing → fieldDelimMulti wiring -----
+
+ /**
+ * When routing sees a MultiDelimitSerDe with a multi-byte field.delim, the
+ * LazySerDeParameters we hand to LazySimpleDeserializeRead must carry the
+ * raw delimiter bytes — that's the signal the reader uses to switch to its
+ * multi-byte scan branch (and hence keep the row on the vectorized LLAP
+ * fast path instead of falling back to DeserializerOrcWriter).
+ */
+ @Test
+ public void testCreateSerdeParamsMultiDelimSetsFieldDelimMulti() throws
Exception {
+ Properties tblProps = new Properties();
+ tblProps.setProperty(serdeConstants.FIELD_DELIM, "~|");
+ tblProps.setProperty(serdeConstants.SERIALIZATION_FORMAT, "~|");
+ tblProps.setProperty(serdeConstants.LIST_COLUMNS, "a,b");
+ tblProps.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string:string");
+
+ LazySerDeParameters params = invokeCreateSerdeParams(tblProps, new
MultiDelimitSerDe());
+ assertArrayEquals("~|".getBytes(StandardCharsets.UTF_8),
params.getFieldDelimMulti());
+ }
+
+ /**
+ * A LazySimpleSerDe table must NEVER opt into the multi-byte scan branch,
+ * even if someone shoved a >1-byte value into field.delim — the slow path
+ * silently truncates to a single byte, and diverging here would produce
+ * different results between fast and slow paths for the same table.
+ */
+ @Test
+ public void testCreateSerdeParamsLazySimpleNeverSetsFieldDelimMulti() throws
Exception {
+ Properties tblProps = new Properties();
+ tblProps.setProperty(serdeConstants.FIELD_DELIM, "~|");
+ tblProps.setProperty(serdeConstants.SERIALIZATION_FORMAT, "~|");
+ tblProps.setProperty(serdeConstants.LIST_COLUMNS, "a,b");
+ tblProps.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string:string");
+
+ LazySerDeParameters params = invokeCreateSerdeParams(tblProps, new
LazySimpleSerDe());
+ assertNull(params.getFieldDelimMulti());
+ }
+
+ /**
+ * A MultiDelimit table whose field.delim is only one byte still takes the
+ * specialized single-byte fast path — setFieldDelimMulti drops sub-2-byte
+ * values, so separators[0] remains the source of truth.
+ */
+ @Test
+ public void testCreateSerdeParamsMultiDelimWithSingleByteFallsThrough()
throws Exception {
+ Properties tblProps = new Properties();
+ tblProps.setProperty(serdeConstants.FIELD_DELIM, "|");
+ tblProps.setProperty(serdeConstants.SERIALIZATION_FORMAT, "|");
+ tblProps.setProperty(serdeConstants.LIST_COLUMNS, "a,b");
+ tblProps.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string:string");
+
+ LazySerDeParameters params = invokeCreateSerdeParams(tblProps, new
MultiDelimitSerDe());
+ assertNull(params.getFieldDelimMulti());
+ }
+
+ private static Field reflectField(Class<?> classToReflect, String
fieldNameValueToFetch) {
+ try {
+ Field reflectField = null;
+ Class<?> classForReflect = classToReflect;
+ do {
+ try {
+ reflectField =
classForReflect.getDeclaredField(fieldNameValueToFetch);
+ } catch (NoSuchFieldException e) {
+ classForReflect = classForReflect.getSuperclass();
+ }
+ } while (reflectField == null || classForReflect == null);
+ reflectField.setAccessible(true);
+ return reflectField;
+ } catch (Exception e) {
+ fail("Failed to reflect " + fieldNameValueToFetch + " from " +
classToReflect);
+ }
+ return null;
+ }
+
+ private static void reflectSetValue(Object objToReflect, String
fieldNameToSet, Object valueToSet) {
+ try {
+ Field reflectField = reflectField(objToReflect.getClass(),
fieldNameToSet);
+ reflectField.set(objToReflect, valueToSet);
+ } catch (Exception e) {
+ fail("Failed to reflectively set " + fieldNameToSet + "=" + valueToSet);
+ }
+ }
+
private static void createTestVectors(ColumnVectorBatch cvb,
VectorizedRowBatch vrb) {
for (int i = 0; i < TEST_NUM_COLS; ++i) {
LongColumnVector cv = new LongColumnVector();
@@ -145,6 +212,19 @@ private static VectorDeserializeOrcWriter createOrcWriter(
return orcWriter;
}
+ private static LazySerDeParameters invokeCreateSerdeParams(Properties
tblProps,
+ Deserializer serDe) throws Exception {
+ // createSerdeParams no longer takes a Deserializer — it reads the SerDe
class
+ // from tblProps' SERIALIZATION_LIB (matches createVrbCtx's own routing).
We
+ // set that key here so the test keeps its "give me a MultiDelimit vs a
+ // LazySimple table" API.
+ tblProps.setProperty(serdeConstants.SERIALIZATION_LIB,
serDe.getClass().getName());
+ Method m = VectorDeserializeOrcWriter.class.getDeclaredMethod(
+ "createSerdeParams", Configuration.class, Properties.class);
+ m.setAccessible(true);
+ return (LazySerDeParameters) m.invoke(null, new Configuration(false),
tblProps);
+ }
+
private static EncodedDataConsumer createBlankEncodedDataConsumer() {
return new EncodedDataConsumer(null, 1, null, null) {
@Override
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
index 9da6bf5e3c3..1db95d3b086 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
@@ -60,11 +60,18 @@ public class LazySerDeParameters implements
LazyObjectInspectorParameters {
private Properties tableProperties;
private String serdeName;
- // The list of bytes used for the separators in the column (a nested struct
+ // The list of bytes used for the separators in the column (a nested struct
// such as Array<Array<int>> will use multiple separators).
// The list of separators + escapeChar are the bytes required to be escaped.
private byte[] separators;
+ // Populated only when the top-level field delimiter is longer than one byte
+ // (i.e. by MultiDelimitSerDe). Never populated for LazySimpleSerDe callers,
+ // so existing single-byte behaviour is preserved. The single-byte
`separators[0]`
+ // slot is still set to the first byte of the multi-byte delim so that
+ // needsEscape[] and any legacy consumers of getSeparators() keep working.
+ private byte[] fieldDelimMulti;
+
private Text nullSequence;
private TypeInfo rowTypeInfo;
@@ -196,6 +203,31 @@ public byte[] getSeparators() {
return separators;
}
+ /**
+ * @return the raw (potentially multi-byte) top-level field delimiter, or
+ * {@code null} when the effective delimiter is a single byte (the
+ * value in {@code separators[0]} is authoritative in that case).
+ *
+ * Set by callers that know they're wiring up a multi-byte delimiter
+ * (e.g. LLAP's {@code VectorDeserializeOrcWriter} when the SerDe is
+ * {@code MultiDelimitSerDe}). Left {@code null} for plain LazySimpleSerDe so
+ * its single-byte fast path is preserved verbatim.
+ */
+ public byte[] getFieldDelimMulti() {
+ return fieldDelimMulti;
+ }
+
+ /**
+ * Opt-in switch for the multi-byte top-level delimiter. Only meaningful when
+ * {@code delim.length > 1}; shorter values are ignored (the {@code
separators[0]}
+ * byte remains the source of truth). Never set this for a
+ * LazySimpleSerDe-backed table — its slow path still truncates to a single
+ * byte, so opting the fast path into multi-byte would silently diverge.
+ */
+ public void setFieldDelimMulti(byte[] delim) {
+ this.fieldDelimMulti = (delim != null && delim.length > 1) ? delim : null;
+ }
+
public Text getNullSequence() {
return nullSequence;
}
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
index 747d389db56..1a5a390e709 100644
---
a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
+++
b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
@@ -57,7 +57,7 @@
import com.google.common.base.Preconditions;
-/*
+/**
* Directly deserialize with the caller reading field-by-field the LazySimple
(text)
* serialization format.
*
@@ -71,6 +71,29 @@
* Some type values are by reference to either bytes in the deserialization
buffer or to
* other type specific buffers. So, those references are only valid until the
next time set is
* called.
+ *
+ * <h3>Multi-byte top-level field delimiter</h3>
+ * By default this reader assumes a single-byte top-level FIELD_DELIM and runs
a specialized
+ * byte-at-a-time hot loop against {@code separators[0]} — the LazySimpleSerDe
fast path.
+ *
+ * A multi-byte top-level delimiter (e.g. MultiDelimitSerDe's {@code ~|}) is
supported when, and
+ * only when, the caller opts in via
+ * {@link
org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters#setFieldDelimMulti(byte[])}
on
+ * the {@code LazySerDeParameters} passed to the constructor. In that mode
{@code topLevelParse()}
+ * takes a separate multi-byte scan branch and the specialized single-byte
loop is not entered.
+ *
+ * Restrictions when opting into the multi-byte branch:
+ * <ul>
+ * <li>only the <em>top-level</em> field delimiter is multi-byte — nested
COLLECTION_DELIM /
+ * MAPKEY_DELIM remain single-byte;</li>
+ * <li>ESCAPE_CHAR is rejected at construction (see {@code
MultiDelimitSerDe.parseMultiDelimit}
+ * which itself ignores escape at the top level — enabling both here
would silently
+ * diverge from the slow path);</li>
+ * <li>LAST_COLUMN_TAKES_REST is not honoured on this path (the LLAP router
bails to
+ * DeserializerOrcWriter before we get here).</li>
+ * </ul>
+ * When {@code setFieldDelimMulti} is not called (or the value is shorter than
2 bytes), the
+ * single-byte fast path is unchanged.
*/
public final class LazySimpleDeserializeRead extends DeserializeRead {
public static final Logger LOG =
LoggerFactory.getLogger(LazySimpleDeserializeRead.class.getName());
@@ -198,6 +221,22 @@ public UnionComplexTypeHelper(Field complexField, Field[]
fields) {
private int[] startPositions;
private final byte[] separators;
+ /*
+ * Multi-byte top-level field delimiter (e.g. MultiDelimitSerDe's `~|`).
+ * null in the common single-byte case, in which case `separators[0]` is used
+ * and the row-scan hot loop keeps its per-byte specialization. Non-null only
+ * when the caller (LLAP's VectorDeserializeOrcWriter for MultiDelimitSerDe)
+ * explicitly opted in via LazySerDeParameters.setFieldDelimMulti.
+ */
+ private final byte[] fieldDelimMulti;
+ /*
+ * Byte length "charged" to a top-level separator: 1 for the single-byte fast
+ * path, delim.length for multi-byte. Baked into (a) the Arrays.fill sentinel
+ * for missing/trailing fields and (b) the "length = next.start - this.start
+ * - sepLen" arithmetic in readField and getDetailedReadPositionString, so
+ * both paths share the same downstream code.
+ */
+ private final int topLevelSeparatorLen;
private final boolean isEscaped;
private final byte escapeChar;
private final int[] escapeCounts;
@@ -336,11 +375,23 @@ public LazySimpleDeserializeRead(TypeInfo[] typeInfos,
startPositions = new int[count + 1];
this.separators = lazyParams.getSeparators();
+ this.fieldDelimMulti = lazyParams.getFieldDelimMulti();
+ this.topLevelSeparatorLen = (fieldDelimMulti == null) ? 1 :
fieldDelimMulti.length;
isEscaped = lazyParams.isEscaped();
if (isEscaped) {
escapeChar = lazyParams.getEscapeChar();
escapeCounts = new int[count];
+ // Multi-byte field delimiters with escape sequences are not implemented
+ // in this fast path — the escape/separator interaction would diverge
+ // from MultiDelimitSerDe's own row-based parser, which ignores escapes
+ // for the top-level delim. Callers (VectorDeserializeOrcWriter) route
+ // this combination to the DeserializerOrcWriter slow path instead.
+ if (fieldDelimMulti != null) {
+ throw new RuntimeException(
+ "Multi-byte field delimiter combined with escape.delim is not "
+ + "supported in the vectorized fast path.");
+ }
} else {
escapeChar = (byte) 0;
escapeCounts = null;
@@ -402,7 +453,7 @@ public String getDetailedReadPositionString() {
sb.append(" at field start position ");
sb.append(startPositions[currentTopLevelFieldIndex]);
int currentFieldLength = startPositions[currentTopLevelFieldIndex + 1] -
- startPositions[currentTopLevelFieldIndex] - 1;
+ startPositions[currentTopLevelFieldIndex] - topLevelSeparatorLen;
sb.append(" for field length ");
sb.append(currentFieldLength);
}
@@ -410,97 +461,216 @@ public String getDetailedReadPositionString() {
return sb.toString();
}
+ /**
+ * True if {@code bytes[off..off+delim.length)} equals the top-level field
+ * delimiter. Callers have already verified the first byte matches, so this
+ * only compares the tail — the mismatching-byte case in the multi-byte hot
+ * loop still costs a single load+compare.
+ */
+ private static boolean delimiterTailMatchesAt(byte[] bytes, int off, byte[]
delim, int dlen) {
+ for (int i = 1; i < dlen; i++) {
+ if (bytes[off + i] != delim[i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
/**
* Parse the byte[] and fill each field.
*
* This is an adapted version of the parse method in the LazyStruct class.
* They should parse things the same way.
+ *
+ * <p>Structure: pick one of three parsing algorithms based on the
(isEscaped,
+ * fieldDelimMulti) pair, run it to find each field's start position, then
+ * fill the trailing sentinel and record EOF. Each algorithm is a single-
+ * callsite private method — splitting them out is a readability move; the
+ * combined loops used to hide three flow-controls inside one big if/else.
+ *
+ * <p>The helpers write into {@link #startPositions} and (for the escape
+ * path) {@link #escapeCounts} directly, and stash the parse-stop cursor —
+ * how many fields they consumed, and where in the byte[] they stopped — on
+ * two instance fields {@link #parsedFieldCount} / {@link
#parsedFieldByteEnd}.
+ * Bundling those into a return value would cost a heap allocation on every
+ * row, which we can't afford here.
*/
private void topLevelParse() {
+ if (isEscaped) {
+ parseTopLevelSingleByteWithEscape();
+ } else if (fieldDelimMulti != null) {
+ parseTopLevelMultiByteNoEscape();
+ } else {
+ parseTopLevelSingleByteNoEscape();
+ }
+
+ final int fieldId = parsedFieldCount;
+ final int fieldByteEnd = parsedFieldByteEnd;
+ /*
+ * All fields have been parsed, or the row has been consumed. Fill
+ * startPositions[fieldId..end] with a sentinel so that the shared
+ * "length = next.start - this.start - sepLen" arithmetic used by
+ * readField yields a negative length for missing fields —
+ * uncheckedGetField turns those into SQL NULLs. Charge the actual
+ * separator width (1 byte for the single-byte fast path, delim.length
+ * for the multi-byte MultiDelimit path) so both paths share the same
+ * downstream code.
+ */
+ if (fieldId == fieldCount || fieldByteEnd == end) {
+ Arrays.fill(startPositions, fieldId, startPositions.length,
+ fieldByteEnd + topLevelSeparatorLen);
+ }
+
+ isEndOfInputReached = (fieldByteEnd == end);
+ }
+
+ /*
+ * Parse-stop cursor written by the three parseTopLevelXxx helpers and
consumed
+ * by topLevelParse() for its sentinel/EOF handling. Using fields instead of
a
+ * multi-value return avoids per-row allocation on the hot path.
+ */
+ private int parsedFieldCount;
+ private int parsedFieldByteEnd;
+
+ /**
+ * Single-byte top-level FIELD_DELIM, no escape — the hot path on every row
+ * of a LazySimple table.
+ */
+ private void parseTopLevelSingleByteNoEscape() {
+ final byte separator = separators[0];
int fieldId = 0;
int fieldByteBegin = start;
int fieldByteEnd = start;
-
- final byte separator = this.separators[0];
- final int fieldCount = this.fieldCount;
- final int[] startPositions = this.startPositions;
- final byte[] bytes = this.bytes;
- final int end = this.end;
-
- /*
- * Optimize the loops by pulling special end cases and global decisions
like isEscaped out!
- */
- if (!isEscaped) {
- while (fieldByteEnd < end) {
- if (bytes[fieldByteEnd] == separator) {
- startPositions[fieldId++] = fieldByteBegin;
- if (fieldId == fieldCount) {
- break;
- }
- fieldByteBegin = ++fieldByteEnd;
- } else {
- fieldByteEnd++;
+ while (fieldByteEnd < end) {
+ if (bytes[fieldByteEnd] == separator) {
+ startPositions[fieldId++] = fieldByteBegin;
+ if (fieldId == fieldCount) {
+ break;
}
+ fieldByteBegin = ++fieldByteEnd;
+ } else {
+ fieldByteEnd++;
}
- // End serves as final separator.
- if (fieldByteEnd == end && fieldId < fieldCount) {
+ }
+ // End serves as final separator.
+ if (fieldByteEnd == end && fieldId < fieldCount) {
+ startPositions[fieldId++] = fieldByteBegin;
+ }
+ parsedFieldCount = fieldId;
+ parsedFieldByteEnd = fieldByteEnd;
+ }
+
+ /**
+ * Multi-byte top-level FIELD_DELIM, no escape — MultiDelimitSerDe's fast
+ * path. We keep the "test the first byte, then verify the tail" idiom so
+ * the mismatching-byte case still costs a single load+compare — the tail
+ * memcmp only fires on a first-byte hit.
+ */
+ private void parseTopLevelMultiByteNoEscape() {
+ final byte[] delim = fieldDelimMulti;
+ final int dlen = delim.length;
+ final byte first = delim[0];
+ final int scanEnd = end - dlen; // last index at which a full delim can
start
+
+ int fieldId = 0;
+ int fieldByteBegin = start;
+ int fieldByteEnd = start;
+ while (fieldByteEnd <= scanEnd) {
+ if (bytes[fieldByteEnd] == first && delimiterTailMatchesAt(bytes,
fieldByteEnd, delim, dlen)) {
startPositions[fieldId++] = fieldByteBegin;
- }
- } else {
- final byte escapeChar = this.escapeChar;
- final int endLessOne = end - 1;
- final int[] escapeCounts = this.escapeCounts;
- int escapeCount = 0;
- // Process the bytes that can be escaped (the last one can't be).
- while (fieldByteEnd < endLessOne) {
- if (bytes[fieldByteEnd] == separator) {
- escapeCounts[fieldId] = escapeCount;
- escapeCount = 0;
- startPositions[fieldId++] = fieldByteBegin;
- if (fieldId == fieldCount) {
- break;
- }
- fieldByteBegin = ++fieldByteEnd;
- } else if (bytes[fieldByteEnd] == escapeChar) {
- // Ignore the char after escape_char
- fieldByteEnd += 2;
- escapeCount++;
- } else {
- fieldByteEnd++;
+ if (fieldId == fieldCount) {
+ // Malformed row with more delims than expected: leave fieldByteEnd
+ // where it is (matching single-byte-path semantics) and stop.
+ parsedFieldCount = fieldId;
+ parsedFieldByteEnd = fieldByteEnd;
+ return;
}
+ fieldByteEnd += dlen;
+ fieldByteBegin = fieldByteEnd;
+ } else {
+ fieldByteEnd++;
}
- // Process the last byte if necessary.
- if (fieldByteEnd == endLessOne && fieldId < fieldCount) {
- if (bytes[fieldByteEnd] == separator) {
- escapeCounts[fieldId] = escapeCount;
- escapeCount = 0;
- startPositions[fieldId++] = fieldByteBegin;
- if (fieldId <= fieldCount) {
- fieldByteBegin = ++fieldByteEnd;
- }
- } else {
- fieldByteEnd++;
+ }
+ /*
+ * No trailing delim (single-byte parses "end as final separator" here).
+ * If we still owe a field, the remainder from fieldByteBegin..end is the
+ * last field; fast-forward fieldByteEnd so the Arrays.fill sentinel and
+ * isEndOfInputReached both see the row as fully consumed.
+ */
+ if (fieldId < fieldCount) {
+ fieldByteEnd = end;
+ startPositions[fieldId++] = fieldByteBegin;
+ }
+ parsedFieldCount = fieldId;
+ parsedFieldByteEnd = fieldByteEnd;
+ }
+
+ /**
+ * Single-byte top-level FIELD_DELIM with escape.delim — the escape byte
+ * "swallows" whatever byte follows it (including a delim byte), so we
+ * track how many escapes each field contains for the copy-out path.
+ */
+ private void parseTopLevelSingleByteWithEscape() {
+ final byte separator = separators[0];
+ final int endLessOne = end - 1;
+
+ int fieldId = 0;
+ int fieldByteBegin = start;
+ int fieldByteEnd = start;
+ int escapeCount = 0;
+
+ // Process the bytes that can be escaped (the last one can't be).
+ while (fieldByteEnd < endLessOne) {
+ if (bytes[fieldByteEnd] == separator) {
+ escapeCounts[fieldId] = escapeCount;
+ escapeCount = 0;
+ startPositions[fieldId++] = fieldByteBegin;
+ if (fieldId == fieldCount) {
+ parsedFieldCount = fieldId;
+ parsedFieldByteEnd = fieldByteEnd;
+ return;
}
+ fieldByteBegin = ++fieldByteEnd;
+ } else if (bytes[fieldByteEnd] == escapeChar) {
+ // Ignore the char after escape_char
+ fieldByteEnd += 2;
+ escapeCount++;
+ } else {
+ fieldByteEnd++;
}
- // End serves as final separator.
- if (fieldByteEnd == end && fieldId < fieldCount) {
+ }
+ finalizeTopLevelSingleByteWithEscape(separator, endLessOne, fieldId,
fieldByteBegin,
+ fieldByteEnd, escapeCount);
+ }
+
+ /**
+ * Tail of {@link #parseTopLevelSingleByteWithEscape()} — handles the last
byte
+ * (which can't be escaped) plus the "end-of-row acts as a final separator"
+ * case, and commits the parsed cursor state.
+ */
+ private void finalizeTopLevelSingleByteWithEscape(byte separator, int
endLessOne, int fieldId,
+ int fieldByteBegin, int fieldByteEnd, int escapeCount) {
+ // Process the last byte if necessary.
+ if (fieldByteEnd == endLessOne && fieldId < fieldCount) {
+ if (bytes[fieldByteEnd] == separator) {
escapeCounts[fieldId] = escapeCount;
+ escapeCount = 0;
startPositions[fieldId++] = fieldByteBegin;
+ if (fieldId <= fieldCount) {
+ fieldByteBegin = ++fieldByteEnd;
+ }
+ } else {
+ fieldByteEnd++;
}
}
-
- if (fieldId == fieldCount || fieldByteEnd == end) {
- // All fields have been parsed, or bytes have been parsed.
- // We need to set the startPositions of fields.length to ensure we
- // can use the same formula to calculate the length of each field.
- // For missing fields, their starting positions will all be the same,
- // which will make their lengths to be -1 and uncheckedGetField will
- // return these fields as NULLs.
- Arrays.fill(startPositions, fieldId, startPositions.length, fieldByteEnd
+ 1);
+ // End serves as final separator.
+ if (fieldByteEnd == end && fieldId < fieldCount) {
+ escapeCounts[fieldId] = escapeCount;
+ startPositions[fieldId++] = fieldByteBegin;
}
-
- isEndOfInputReached = (fieldByteEnd == end);
+ parsedFieldCount = fieldId;
+ parsedFieldByteEnd = fieldByteEnd;
}
private int parseComplexField(int start, int end, int level) {
@@ -639,7 +809,7 @@ public boolean readField(int fieldIndex) throws IOException
{
currentTopLevelFieldIndex = fieldIndex;
currentFieldStart = startPositions[fieldIndex];
- currentFieldLength = startPositions[fieldIndex + 1] -
startPositions[fieldIndex] - 1;
+ currentFieldLength = startPositions[fieldIndex + 1] -
startPositions[fieldIndex] - topLevelSeparatorLen;
currentEscapeCount = (isEscaped ? escapeCounts[fieldIndex] : 0);
return doReadField(fields[fieldIndex]);
diff --git
a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java
b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java
index 465a616fa93..f705f0baeb6 100644
---
a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java
+++
b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java
@@ -20,6 +20,8 @@
+import java.lang.reflect.Field;
+import java.nio.charset.StandardCharsets;
import java.util.Properties;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -29,8 +31,12 @@
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.io.Text;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import org.junit.Test;
/**
@@ -86,4 +92,531 @@ public void testEscaping() throws Exception {
assertTrue("The escaped result is incorrect", field.compareTo(escaped) ==
0);
}
+
+ // --- single-byte field delimiter, no escape (LazySimple fast path) --------
+
+ /**
+ * Mixed INT / STRING columns on the single-byte hot loop — the numeric
+ * parsers depend on the length arithmetic between {@code startPositions[i]}
+ * and {@code startPositions[i+1]} charging exactly 1 byte for the delim.
+ */
+ @Test
+ public void testSingleByteNoEscapeMixedTypes() throws Exception {
+ LazySerDeParameters params = singleDelimParams("|");
+ TypeInfo[] typeInfos = new TypeInfo[] {
+ TypeInfoFactory.intTypeInfo,
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.longTypeInfo
+ };
+ LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos,
null, true, params);
+
+ byte[] row = "42|hello|9876543210".getBytes(StandardCharsets.UTF_8);
+ r.set(row, 0, row.length);
+
+ assertTrue(r.readNextField());
+ assertEquals(42, r.currentInt);
+
+ assertArrayEquals("hello".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+
+ assertTrue(r.readNextField());
+ assertEquals(9876543210L, r.currentLong);
+ }
+
+ /**
+ * A row ending exactly on a delim: the last field is empty. Exercises the
+ * "separator hit, then loop terminates because fieldByteEnd == end" arc in
+ * the hot loop and the trailing "end serves as final separator" branch.
+ */
+ @Test
+ public void testSingleByteNoEscapeEmptyLastField() throws Exception {
+ LazySerDeParameters params = singleDelimParams("|");
+ TypeInfo[] typeInfos = new TypeInfo[] {
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo
+ };
+ LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos,
null, true, params);
+
+ byte[] row = "one|".getBytes(StandardCharsets.UTF_8);
+ r.set(row, 0, row.length);
+
+ assertArrayEquals("one".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertTrue("empty last field should be non-null (zero-length)",
r.readNextField());
+ assertEquals(0, r.currentBytesLength);
+ }
+
+ /**
+ * A middle field that's empty (two consecutive delims). Ensures the
+ * "fieldByteBegin = ++fieldByteEnd" step correctly records a zero-length
+ * span in {@code startPositions}, not a spurious NULL.
+ */
+ @Test
+ public void testSingleByteNoEscapeEmptyMiddleField() throws Exception {
+ LazySerDeParameters params = singleDelimParams("|");
+ TypeInfo[] typeInfos = new TypeInfo[] {
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo
+ };
+ LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos,
null, true, params);
+
+ byte[] row = "a||c".getBytes(StandardCharsets.UTF_8);
+ r.set(row, 0, row.length);
+
+ assertArrayEquals("a".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertTrue("empty middle field must be non-null", r.readNextField());
+ assertEquals(0, r.currentBytesLength);
+ assertArrayEquals("c".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ }
+
+ /**
+ * Row has fewer fields than the schema expects. All missing fields must
+ * come back as NULL — this is the sentinel-fill branch of
+ * {@link
org.apache.hadoop.hive.serde2.lazy.fast.LazySimpleDeserializeRead#topLevelParse()}.
+ */
+ @Test
+ public void testSingleByteNoEscapeMissingTrailingFields() throws Exception {
+ LazySerDeParameters params = singleDelimParams("|");
+ TypeInfo[] typeInfos = new TypeInfo[] {
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo
+ };
+ LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos,
null, true, params);
+
+ byte[] row = "only-one".getBytes(StandardCharsets.UTF_8);
+ r.set(row, 0, row.length);
+
+ assertArrayEquals("only-one".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertFalse(r.readNextField());
+ assertFalse(r.readNextField());
+ }
+
+ /**
+ * Row supplies more delims than the schema has columns: the hot loop must
+ * break out on {@code fieldId == fieldCount} without consuming past the
+ * last expected field.
+ */
+ @Test
+ public void testSingleByteNoEscapeExtraDelimsIgnored() throws Exception {
+ LazySerDeParameters params = singleDelimParams("|");
+ TypeInfo[] typeInfos = new TypeInfo[] {
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo
+ };
+ LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos,
null, true, params);
+
+ byte[] row = "a|b|extra|ignored".getBytes(StandardCharsets.UTF_8);
+ r.set(row, 0, row.length);
+
+ assertArrayEquals("a".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertArrayEquals("b".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertFalse("schema has exactly 2 fields — the rest must not surface",
r.readNextField());
+ }
+
+ // --- single-byte field delimiter, with escape (escape hot loop) ----------
+
+ /**
+ * The point of escape: a delim byte preceded by escape.delim must NOT split
+ * fields. Also verifies the {@code escapeCounts} bookkeeping — reading the
+ * field back out requires the external-buffer copy path.
+ */
+ @Test
+ public void testSingleByteEscapeEscapedDelimInsideField() throws Exception {
+ LazySerDeParameters params = escapeParams("|", "\\");
+ TypeInfo[] typeInfos = new TypeInfo[] {
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo
+ };
+ LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos,
null, true, params);
+
+ // "foo\|bar|baz" — the escaped '|' stays inside field 0.
+ byte[] row = "foo\\|bar|baz".getBytes(StandardCharsets.UTF_8);
+ r.set(row, 0, row.length);
+
+ assertTrue(r.readNextField());
+ assertTrue("escaped delim inside field forces the copy-out path",
+ r.currentExternalBufferNeeded);
+ byte[] buf = new byte[r.currentExternalBufferNeededLen];
+ r.copyToExternalBuffer(buf, 0);
+ assertArrayEquals("foo|bar".getBytes(StandardCharsets.UTF_8), buf);
+
+ assertArrayEquals("baz".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ }
+
+ /**
+ * Escape path, empty last field. The endLessOne / trailing-tail branch has
+ * to notice that we're at EOL and record a zero-length last field.
+ */
+ @Test
+ public void testSingleByteEscapeEmptyLastField() throws Exception {
+ LazySerDeParameters params = escapeParams("|", "\\");
+ TypeInfo[] typeInfos = new TypeInfo[] {
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo
+ };
+ LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos,
null, true, params);
+
+ byte[] row = "one|".getBytes(StandardCharsets.UTF_8);
+ r.set(row, 0, row.length);
+
+ assertArrayEquals("one".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertTrue("empty last field on escape path must be non-null
(zero-length)",
+ r.readNextField());
+ assertEquals(0, r.currentBytesLength);
+ }
+
+ /**
+ * Escape path, missing trailing fields → NULL. Symmetric to the no-escape
+ * missing-trailing test; makes sure the sentinel-fill epilogue is common
+ * to both parse helpers.
+ */
+ @Test
+ public void testSingleByteEscapeMissingTrailingFields() throws Exception {
+ LazySerDeParameters params = escapeParams("|", "\\");
+ TypeInfo[] typeInfos = new TypeInfo[] {
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo
+ };
+ LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos,
null, true, params);
+
+ byte[] row = "solo".getBytes(StandardCharsets.UTF_8);
+ r.set(row, 0, row.length);
+
+ assertArrayEquals("solo".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertFalse(r.readNextField());
+ assertFalse(r.readNextField());
+ }
+
+ /**
+ * Escape path with mixed INT / STRING columns and NO escape characters in
+ * the payload. Exercises the "no escape, no delim" plain-byte inner branch
+ * many times in a row and confirms numeric parsing still sees a
+ * zero-escape field boundary.
+ */
+ @Test
+ public void testSingleByteEscapeMixedTypesNoEscapesInPayload() throws
Exception {
+ LazySerDeParameters params = escapeParams("|", "\\");
+ TypeInfo[] typeInfos = new TypeInfo[] {
+ TypeInfoFactory.intTypeInfo,
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.longTypeInfo
+ };
+ LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos,
null, true, params);
+
+ byte[] row = "42|hello|9876543210".getBytes(StandardCharsets.UTF_8);
+ r.set(row, 0, row.length);
+
+ assertTrue(r.readNextField());
+ assertEquals(42, r.currentInt);
+ assertArrayEquals("hello".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertTrue(r.readNextField());
+ assertEquals(9876543210L, r.currentLong);
+ }
+
+ /**
+ * The last byte of the row is a delim: {@code endLessOne} loop stops one
+ * short, then the trailing single-byte branch fires. Confirms both the
+ * "last byte is separator" mini-branch and the following empty-last-field
+ * epilogue produce the right startPositions.
+ */
+ @Test
+ public void testSingleByteEscapeLastByteIsDelim() throws Exception {
+ LazySerDeParameters params = escapeParams("|", "\\");
+ TypeInfo[] typeInfos = new TypeInfo[] {
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo
+ };
+ LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos,
null, true, params);
+
+ byte[] row = "a|b|".getBytes(StandardCharsets.UTF_8);
+ r.set(row, 0, row.length);
+
+ assertArrayEquals("a".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertArrayEquals("b".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertTrue("empty tail after final delim must be non-null (zero-length)",
+ r.readNextField());
+ assertEquals(0, r.currentBytesLength);
+ }
+
+ /**
+ * Escape char at the very last position of the row has nothing to swallow.
+ * The {@code endLessOne} guard is what prevents the "escape swallows next
+ * byte" step from reading past {@code end}; this test pins that behavior.
+ */
+ @Test
+ public void testSingleByteEscapeCharAtEndOfInput() throws Exception {
+ LazySerDeParameters params = escapeParams("|", "\\");
+ TypeInfo[] typeInfos = new TypeInfo[] {TypeInfoFactory.stringTypeInfo };
+ LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos,
null, true, params);
+
+ // Single field whose last byte is the escape char with no follower.
+ byte[] row = "abc\\".getBytes(StandardCharsets.UTF_8);
+ r.set(row, 0, row.length);
+
+ // Must not throw and must not read past end. Whatever surface value the
+ // reader chooses is fine as long as the length is bounded.
+ assertTrue(r.readNextField());
+ if (r.currentExternalBufferNeeded) {
+ byte[] buf = new byte[r.currentExternalBufferNeededLen];
+ r.copyToExternalBuffer(buf, 0);
+ assertTrue("copy-out length must not exceed the row length",
+ buf.length <= row.length);
+ } else {
+ assertTrue("in-place length must not exceed the row length",
+ r.currentBytesLength <= row.length);
+ }
+ }
+
+ // --- multi-byte field delimiter (MultiDelimitSerDe → LLAP fast path) ------
+
+ /**
+ * Three STRING columns separated by "~|" — the delimiter used by the BofA
+ * MultiDelimitSerDe workload that motivated this path.
+ */
+ @Test
+ public void testMultiByteDelimThreeStringColumns() throws Exception {
+ LazySerDeParameters params = multiDelimParams("~|");
+ TypeInfo[] typeInfos = new TypeInfo[] {
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo
+ };
+ LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos,
null, true, params);
+
+ byte[] row = "alpha~|beta~|gamma".getBytes(StandardCharsets.UTF_8);
+ r.set(row, 0, row.length);
+
+ assertArrayEquals("alpha".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertArrayEquals("beta".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertArrayEquals("gamma".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertFalse("row should have exactly 3 fields", r.readNextField());
+ }
+
+ /**
+ * Mixed INT / STRING columns. Verifies that the length arithmetic charges
+ * the full delim.length (not 1) between fields so numeric parsing sees the
+ * correct field boundaries.
+ */
+ @Test
+ public void testMultiByteDelimMixedTypes() throws Exception {
+ LazySerDeParameters params = multiDelimParams("~|");
+ TypeInfo[] typeInfos = new TypeInfo[] {
+ TypeInfoFactory.intTypeInfo,
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.longTypeInfo
+ };
+ LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos,
null, true, params);
+
+ byte[] row = "42~|hello~|9876543210".getBytes(StandardCharsets.UTF_8);
+ r.set(row, 0, row.length);
+
+ assertTrue(r.readNextField());
+ assertEquals(42, r.currentInt);
+
+ assertArrayEquals("hello".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+
+ assertTrue(r.readNextField());
+ assertEquals(9876543210L, r.currentLong);
+ }
+
+ /**
+ * The first byte of the delimiter ("~") appearing standalone inside a field
+ * must NOT trigger a split — the tail-match on delim[1] rescues it.
+ */
+ @Test
+ public void testMultiByteDelimFirstByteInsideField() throws Exception {
+ LazySerDeParameters params = multiDelimParams("~|");
+ TypeInfo[] typeInfos = new TypeInfo[] {
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo
+ };
+ LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos,
null, true, params);
+
+ // A bare '~' followed by non-'|' must stay inside the first field.
+ byte[] row = "foo~bar~|baz".getBytes(StandardCharsets.UTF_8);
+ r.set(row, 0, row.length);
+
+ assertArrayEquals("foo~bar".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertArrayEquals("baz".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ }
+
+ /**
+ * The empty-tail case: last field extends to EOL and has zero length.
+ * startPositions must still tabulate length 0 (not -1), so NULL isn't
+ * spuriously returned.
+ */
+ @Test
+ public void testMultiByteDelimEmptyLastField() throws Exception {
+ LazySerDeParameters params = multiDelimParams("~|");
+ TypeInfo[] typeInfos = new TypeInfo[] {
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo
+ };
+ LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos,
null, true, params);
+
+ byte[] row = "one~|".getBytes(StandardCharsets.UTF_8);
+ r.set(row, 0, row.length);
+
+ assertArrayEquals("one".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertTrue("empty last field should be non-null (zero-length)",
r.readNextField());
+ assertEquals(0, r.currentBytesLength);
+ }
+
+ /**
+ * Missing trailing fields must land in the "startPositions filled with
+ * sentinel" branch and come back as NULL — same behavior as single-byte.
+ */
+ @Test
+ public void testMultiByteDelimMissingTrailingFields() throws Exception {
+ LazySerDeParameters params = multiDelimParams("~|");
+ TypeInfo[] typeInfos = new TypeInfo[] {
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo
+ };
+ LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos,
null, true, params);
+
+ byte[] row = "only-one".getBytes(StandardCharsets.UTF_8);
+ r.set(row, 0, row.length);
+
+ assertArrayEquals("only-one".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ // Missing fields → NULL.
+ assertFalse(r.readNextField());
+ assertFalse(r.readNextField());
+ }
+
+ /**
+ * A three-byte delimiter exercises the "delim length > 2" arithmetic.
+ */
+ @Test
+ public void testMultiByteDelimThreeByteDelimiter() throws Exception {
+ LazySerDeParameters params = multiDelimParams("|~|");
+ TypeInfo[] typeInfos = new TypeInfo[] {
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo
+ };
+ LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos,
null, true, params);
+
+ byte[] row = "a|~|bb|~|ccc".getBytes(StandardCharsets.UTF_8);
+ r.set(row, 0, row.length);
+
+ assertArrayEquals("a".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertArrayEquals("bb".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertArrayEquals("ccc".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ }
+
+ /**
+ * Multi-byte delim combined with escape.delim is not supported in the fast
+ * path — LazySimpleDeserializeRead must reject the combination at
+ * construction time so we never silently diverge from
+ * MultiDelimitSerDe.parseMultiDelimit (which itself ignores escape at the
+ * top level).
+ */
+ @Test
+ public void testMultiByteDelimRejectsEscape() throws Exception {
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.FIELD_DELIM, "~|");
+ props.setProperty(serdeConstants.SERIALIZATION_FORMAT, "~|");
+ props.setProperty(serdeConstants.ESCAPE_CHAR, "\\");
+ LazySerDeParameters params = new LazySerDeParameters(new HiveConf(), props,
+ LazySimpleSerDe.class.getName());
+ params.setFieldDelimMulti("~|".getBytes(StandardCharsets.UTF_8));
+
+ TypeInfo[] typeInfos = new TypeInfo[]{TypeInfoFactory.stringTypeInfo};
+ try {
+ new LazySimpleDeserializeRead(typeInfos, null, true, params);
+ fail("expected RuntimeException for multi-byte delim + escape.delim");
+ } catch (RuntimeException expected) {
+ assertTrue("unexpected message: " + expected.getMessage(),
+ expected.getMessage().contains("Multi-byte field delimiter"));
+ }
+ }
+
+ /**
+ * When the table is a plain LazySimpleSerDe (setFieldDelimMulti never called
+ * on the params), LazySimpleDeserializeRead must select the specialized
+ * single-byte hot loop in {@code topLevelParse()}. The branch is gated on
+ * {@code fieldDelimMulti == null}, so we prove path selection by reflecting
+ * on the reader's internal state after a successful parse:
+ * <ul>
+ * <li>{@code fieldDelimMulti} is {@code null} — the {@code if} in
+ * {@code topLevelParse()} evaluated to the single-byte branch;</li>
+ * <li>{@code topLevelSeparatorLen == 1} — the length arithmetic
+ * (both in the sentinel fill and in {@code readField}) uses the
+ * single-byte constant, not the multi-byte {@code dlen}.</li>
+ * </ul>
+ * A correct parse of the row confirms the branch actually ran.
+ */
+ @Test
+ public void testSingleByteDelimHitsSpecializedHotPath() throws Exception {
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.FIELD_DELIM, "|");
+ props.setProperty(serdeConstants.SERIALIZATION_FORMAT, "|");
+ LazySerDeParameters params = new LazySerDeParameters(new HiveConf(), props,
+ LazySimpleSerDe.class.getName());
+ // Intentionally do NOT call setFieldDelimMulti — this is a LazySimple
table.
+
+ TypeInfo[] typeInfos = new TypeInfo[] {
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo
+ };
+ LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos,
null, true, params);
+
+ byte[] row = "alpha|beta|gamma".getBytes(StandardCharsets.UTF_8);
+ r.set(row, 0, row.length);
+ assertArrayEquals("alpha".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertArrayEquals("beta".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+ assertArrayEquals("gamma".getBytes(StandardCharsets.UTF_8),
readStringField(r));
+
+ // Path-selection proof: the reader's fieldDelimMulti stayed null and its
+ // topLevelSeparatorLen is 1 — the only code path in topLevelParse() that
+ // could have produced the correct parse above.
+ assertNull("fieldDelimMulti must be null on the LazySimple fast path",
+ readPrivateField(r, "fieldDelimMulti"));
+ assertEquals("topLevelSeparatorLen must be 1 on the LazySimple fast path",
+ 1, ((Integer) readPrivateField(r, "topLevelSeparatorLen")).intValue());
+ }
+
+ private static LazySerDeParameters singleDelimParams(String delim) throws
Exception {
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.FIELD_DELIM, delim);
+ props.setProperty(serdeConstants.SERIALIZATION_FORMAT, delim);
+ return new LazySerDeParameters(new HiveConf(), props,
LazySimpleSerDe.class.getName());
+ }
+
+ private static LazySerDeParameters escapeParams(String delim, String escape)
throws Exception {
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.FIELD_DELIM, delim);
+ props.setProperty(serdeConstants.SERIALIZATION_FORMAT, delim);
+ props.setProperty(serdeConstants.ESCAPE_CHAR, escape);
+ return new LazySerDeParameters(new HiveConf(), props,
LazySimpleSerDe.class.getName());
+ }
+
+ private static LazySerDeParameters multiDelimParams(String delim) throws
Exception {
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.FIELD_DELIM, delim);
+ props.setProperty(serdeConstants.SERIALIZATION_FORMAT, delim);
+ LazySerDeParameters p = new LazySerDeParameters(new HiveConf(), props,
+ LazySimpleSerDe.class.getName());
+ p.setFieldDelimMulti(delim.getBytes(StandardCharsets.UTF_8));
+ return p;
+ }
+
+ private static byte[] readStringField(LazySimpleDeserializeRead r) throws
Exception {
+ assertTrue("expected non-null field", r.readNextField());
+ int len = r.currentBytesLength;
+ byte[] out = new byte[len];
+ System.arraycopy(r.currentBytes, r.currentBytesStart, out, 0, len);
+ return out;
+ }
+
+ private static Object readPrivateField(Object target, String name) throws
Exception {
+ Field f = LazySimpleDeserializeRead.class.getDeclaredField(name);
+ f.setAccessible(true);
+ return f.get(target);
+ }
}
