This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new a234ba13cd4 HIVE-28262:Single column use MultiDelimitSerDe parse
column error (#5252)(Liu Weizheng, reviewed by Butao Zhang)
a234ba13cd4 is described below
commit a234ba13cd45149ac7c5f532a7742ea7bad28767
Author: Laughing <[email protected]>
AuthorDate: Wed Jul 3 09:12:54 2024 +0800
HIVE-28262:Single column use MultiDelimitSerDe parse column error
(#5252)(Liu Weizheng, reviewed by Butao Zhang)
(cherry picked from commit b15ce9126dcc596c953fe202e6834cebdfca6080)
---
.../apache/hadoop/hive/serde2/lazy/LazyStruct.java | 4 +-
.../hadoop/hive/serde2/lazy/TestLazyStruct.java | 82 ++++++++++++++++++++++
2 files changed, 84 insertions(+), 2 deletions(-)
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java
b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java
index 2848d348e30..3bd92cc7b5b 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java
@@ -300,7 +300,7 @@ public class LazyStruct extends
LazyNonPrimitive<LazySimpleStructObjectInspector
// first field always starts from 0, even when missing
startPosition[0] = 0;
for (int i = 1; i <= fields.length; i++) {
- if (fields.length > 1 && delimitIndexes[i - 1] != -1) {
+ if (delimitIndexes[i - 1] != -1) {
int start = delimitIndexes[i - 1] + fieldDelimit.length;
startPosition[i] = start - i * diff;
} else {
@@ -313,7 +313,7 @@ public class LazyStruct extends
LazyNonPrimitive<LazySimpleStructObjectInspector
// find all the indexes of the sub byte[]
private int[] findIndexes(byte[] array, byte[] target) {
- if (fields.length <= 1) {
+ if (fields.length < 1) {
return new int[0];
}
int[] indexes = new int[fields.length];
diff --git
a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyStruct.java
b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyStruct.java
new file mode 100644
index 00000000000..a1f9b695f07
--- /dev/null
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyStruct.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazy;
+
+import
org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters;
+import
org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParametersImpl;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * TestLazyStruct.<br/>
+ */
+public class TestLazyStruct {
+
+ @Test
+ public void testParseMultiDelimit() throws Throwable {
+ try {
+ // single field named id
+ List<String> structFieldNames = new ArrayList<>();
+ structFieldNames.add("id");
+ // field type is string
+ List<TypeInfo> fieldTypes = new ArrayList<>();
+ PrimitiveTypeInfo primitiveTypeInfo = new PrimitiveTypeInfo();
+ primitiveTypeInfo.setTypeName("string");
+ fieldTypes.add(primitiveTypeInfo);
+ // separators + escapeChar => "|"
+ byte[] separators = new byte[]{124, 2, 3, 4, 5, 6, 7, 8};
+
+ // sequence =>"\N"
+ Text sequence = new Text();
+ sequence.set(new byte[]{92, 78});
+
+ // create lazy object inspector parameters
+ LazyObjectInspectorParameters lazyObjectInspectorParameters = new
LazyObjectInspectorParametersImpl(false, (byte) '0',
+ false, null, separators, sequence);
+ // create a lazy struct inspector
+ ObjectInspector lazyStructInspector =
LazyFactory.createLazyStructInspector(structFieldNames, fieldTypes,
lazyObjectInspectorParameters);
+ LazyStruct lazyStruct = (LazyStruct)
LazyFactory.createLazyObject(lazyStructInspector);
+
+ // origin row data
+ String rowData = "1|@|";
+ // row field delimiter
+ String fieldDelimiter = "|@|";
+
+ // parse row use multi delimit
+
lazyStruct.parseMultiDelimit(rowData.getBytes(StandardCharsets.UTF_8),
+ fieldDelimiter.getBytes(StandardCharsets.UTF_8));
+
+ // check the first field and second field start position index
+ // before fix result: 0,1
+ // after fix result: 0,2
+ Assert.assertArrayEquals(new int[]{0, 2},
lazyStruct.startPosition);
+ } catch (Throwable e) {
+ e.printStackTrace();
+ throw e;
+ }
+
+ }
+}
\ No newline at end of file