This is an automated email from the ASF dual-hosted git repository.
kasakrisz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 9173850b9d8 HIVE-24599: Add support for vectorized two-parameter trim
functions (#6616)
9173850b9d8 is described below
commit 9173850b9d8d51b253a0763115a6e5720b040640
Author: MANYA MEHROTRA <[email protected]>
AuthorDate: Mon Aug 3 13:59:21 2026 +0530
HIVE-24599: Add support for vectorized two-parameter trim functions (#6616)
Co-authored-by: ManyaMehrotra <[email protected]>
---
.../exec/vector/expressions/StringLTrimColCol.java | 44 +++
.../vector/expressions/StringLTrimColScalar.java | 19 +-
.../vector/expressions/StringLTrimScalarCol.java | 43 +++
.../exec/vector/expressions/StringRTrimColCol.java | 44 +++
.../vector/expressions/StringRTrimColScalar.java | 19 +-
.../vector/expressions/StringRTrimScalarCol.java | 43 +++
.../exec/vector/expressions/StringTrimColCol.java | 44 +++
.../vector/expressions/StringTrimColColBase.java | 106 +++++++
.../vector/expressions/StringTrimColScalar.java | 27 +-
.../expressions/StringTrimColScalarBase.java | 70 ++++-
.../vector/expressions/StringTrimScalarCol.java | 43 +++
.../expressions/StringTrimScalarColBase.java | 145 +++++++++
.../hive/ql/udf/generic/GenericUDFLTrim.java | 8 +-
.../hive/ql/udf/generic/GenericUDFRTrim.java | 8 +-
.../hadoop/hive/ql/udf/generic/GenericUDFTrim.java | 8 +-
.../ql/exec/vector/TestVectorizationContext.java | 65 +++++
.../test/queries/clientpositive/udf_ltrim_vector.q | 25 +-
.../test/queries/clientpositive/udf_rtrim_vector.q | 25 +-
.../test/queries/clientpositive/udf_trim_vector.q | 25 +-
.../clientpositive/llap/udf_ltrim_vector.q.out | 324 ++++++++++++++++++++-
.../clientpositive/llap/udf_rtrim_vector.q.out | 324 ++++++++++++++++++++-
.../clientpositive/llap/udf_trim_vector.q.out | 324 ++++++++++++++++++++-
22 files changed, 1704 insertions(+), 79 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringLTrimColCol.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringLTrimColCol.java
new file mode 100644
index 00000000000..24189033366
--- /dev/null
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringLTrimColCol.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+
+/**
+ * Vectorized LTRIM with a string column and a trim-characters column.
+ */
+public class StringLTrimColCol extends StringTrimColColBase {
+ private static final long serialVersionUID = 1L;
+
+ public StringLTrimColCol(int strCol, int trimCharsCol, int outputColumnNum) {
+ super(strCol, trimCharsCol, outputColumnNum);
+ }
+
+ public StringLTrimColCol() {
+ super();
+ }
+
+ @Override
+ protected void func(BytesColumnVector outV, BytesColumnVector strV, int
strIndex,
+ BytesColumnVector trimV, int trimIndex, int outIndex) {
+ StringTrimColScalarBase.trimLeft(outV, strV.vector[strIndex],
strV.start[strIndex],
+ strV.length[strIndex], trimV.vector[trimIndex], trimV.start[trimIndex],
+ trimV.length[trimIndex], outIndex);
+ }
+}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringLTrimColScalar.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringLTrimColScalar.java
index e510ef23f96..ec005f04cb9 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringLTrimColScalar.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringLTrimColScalar.java
@@ -38,21 +38,8 @@ public StringLTrimColScalar() {
*/
protected void func(BytesColumnVector outV, byte[][] vector, int[] start,
int[] length,
int batchIndex) {
- byte[] bytes = vector[batchIndex];
- final int startIndex = start[batchIndex];
-
- // Skip past blank characters.
- final int end = startIndex + length[batchIndex];
- int index = startIndex;
- while(index < end && shouldTrim(bytes[index])) {
- index++;
- }
-
- final int resultLength = end - index;
- if (resultLength == 0) {
- outV.setVal(batchIndex, EMPTY_BYTES, 0, 0);
- return;
- }
- outV.setVal(batchIndex, bytes, index, resultLength);
+ byte[] trimChars = getTrimChars();
+ trimLeft(outV, vector[batchIndex], start[batchIndex], length[batchIndex],
+ trimChars, 0, trimChars.length, batchIndex);
}
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringLTrimScalarCol.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringLTrimScalarCol.java
new file mode 100644
index 00000000000..f8e5a28fbee
--- /dev/null
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringLTrimScalarCol.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+
+/**
+ * Vectorized LTRIM with a scalar string and a trim-characters column.
+ */
+public class StringLTrimScalarCol extends StringTrimScalarColBase {
+ private static final long serialVersionUID = 1L;
+
+ public StringLTrimScalarCol(byte[] stringToTrim, int trimCharsCol, int
outputColumnNum) {
+ super(stringToTrim, trimCharsCol, outputColumnNum);
+ }
+
+ public StringLTrimScalarCol() {
+ super();
+ }
+
+ @Override
+ protected void func(BytesColumnVector outV, BytesColumnVector trimV, int
trimIndex,
+ int outIndex) {
+ StringTrimColScalarBase.trimLeft(outV, stringToTrim, 0,
stringToTrim.length,
+ trimV.vector[trimIndex], trimV.start[trimIndex],
trimV.length[trimIndex], outIndex);
+ }
+}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringRTrimColCol.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringRTrimColCol.java
new file mode 100644
index 00000000000..5e1d8c29506
--- /dev/null
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringRTrimColCol.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+
+/**
+ * Vectorized RTRIM with a string column and a trim-characters column.
+ */
+public class StringRTrimColCol extends StringTrimColColBase {
+ private static final long serialVersionUID = 1L;
+
+ public StringRTrimColCol(int strCol, int trimCharsCol, int outputColumnNum) {
+ super(strCol, trimCharsCol, outputColumnNum);
+ }
+
+ public StringRTrimColCol() {
+ super();
+ }
+
+ @Override
+ protected void func(BytesColumnVector outV, BytesColumnVector strV, int
strIndex,
+ BytesColumnVector trimV, int trimIndex, int outIndex) {
+ StringTrimColScalarBase.trimRight(outV, strV.vector[strIndex],
strV.start[strIndex],
+ strV.length[strIndex], trimV.vector[trimIndex], trimV.start[trimIndex],
+ trimV.length[trimIndex], outIndex);
+ }
+}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringRTrimColScalar.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringRTrimColScalar.java
index 61f079a4d0d..3e57dd20dfa 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringRTrimColScalar.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringRTrimColScalar.java
@@ -38,21 +38,8 @@ public StringRTrimColScalar() {
*/
protected void func(BytesColumnVector outV, byte[][] vector, int[] start,
int[] length,
int batchIndex) {
-
- byte[] bytes = vector[batchIndex];
- final int startIndex = start[batchIndex];
-
- // Skip trailing blank characters.
- int index = startIndex + length[batchIndex] - 1;
- while(index >= startIndex && shouldTrim(bytes[index])) {
- index--;
- }
-
- final int resultLength = index - startIndex + 1;
- if (resultLength == 0) {
- outV.setVal(batchIndex, EMPTY_BYTES, 0, 0);
- return;
- }
- outV.setVal(batchIndex, bytes, startIndex, resultLength);
+ byte[] trimChars = getTrimChars();
+ trimRight(outV, vector[batchIndex], start[batchIndex], length[batchIndex],
+ trimChars, 0, trimChars.length, batchIndex);
}
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringRTrimScalarCol.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringRTrimScalarCol.java
new file mode 100644
index 00000000000..1d218f96888
--- /dev/null
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringRTrimScalarCol.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+
+/**
+ * Vectorized RTRIM with a scalar string and a trim-characters column.
+ */
+public class StringRTrimScalarCol extends StringTrimScalarColBase {
+ private static final long serialVersionUID = 1L;
+
+ public StringRTrimScalarCol(byte[] stringToTrim, int trimCharsCol, int
outputColumnNum) {
+ super(stringToTrim, trimCharsCol, outputColumnNum);
+ }
+
+ public StringRTrimScalarCol() {
+ super();
+ }
+
+ @Override
+ protected void func(BytesColumnVector outV, BytesColumnVector trimV, int
trimIndex,
+ int outIndex) {
+ StringTrimColScalarBase.trimRight(outV, stringToTrim, 0,
stringToTrim.length,
+ trimV.vector[trimIndex], trimV.start[trimIndex],
trimV.length[trimIndex], outIndex);
+ }
+}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimColCol.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimColCol.java
new file mode 100644
index 00000000000..a4edc1872bc
--- /dev/null
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimColCol.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+
+/**
+ * Vectorized TRIM with a string column and a trim-characters column.
+ */
+public class StringTrimColCol extends StringTrimColColBase {
+ private static final long serialVersionUID = 1L;
+
+ public StringTrimColCol(int strCol, int trimCharsCol, int outputColumnNum) {
+ super(strCol, trimCharsCol, outputColumnNum);
+ }
+
+ public StringTrimColCol() {
+ super();
+ }
+
+ @Override
+ protected void func(BytesColumnVector outV, BytesColumnVector strV, int
strIndex,
+ BytesColumnVector trimV, int trimIndex, int outIndex) {
+ StringTrimColScalarBase.trimBoth(outV, strV.vector[strIndex],
strV.start[strIndex],
+ strV.length[strIndex], trimV.vector[trimIndex], trimV.start[trimIndex],
+ trimV.length[trimIndex], outIndex);
+ }
+}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimColColBase.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimColColBase.java
new file mode 100644
index 00000000000..b7a6b016df4
--- /dev/null
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimColColBase.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+
+/**
+ * Base class for vectorized two-parameter trim functions with a string column
and a
+ * trim-characters column. Null/repeating/selected handling is delegated to
+ * {@link NullUtil#propagateNullsColCol}.
+ */
+abstract public class StringTrimColColBase extends VectorExpression {
+ private static final long serialVersionUID = 1L;
+
+ public StringTrimColColBase(int strCol, int trimCharsCol, int
outputColumnNum) {
+ super(strCol, trimCharsCol, outputColumnNum);
+ }
+
+ public StringTrimColColBase() {
+ super();
+ }
+
+ abstract protected void func(BytesColumnVector outV, BytesColumnVector strV,
int strIndex,
+ BytesColumnVector trimV, int trimIndex, int outIndex);
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) throws HiveException {
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ BytesColumnVector strV = (BytesColumnVector) batch.cols[inputColumnNum[0]];
+ BytesColumnVector trimV = (BytesColumnVector)
batch.cols[inputColumnNum[1]];
+ BytesColumnVector outV = (BytesColumnVector) batch.cols[outputColumnNum];
+ int[] sel = batch.selected;
+ int n = batch.size;
+ if (n == 0) {
+ return;
+ }
+
+ outV.initBuffer();
+ NullUtil.propagateNullsColCol(strV, trimV, outV, sel, n,
batch.selectedInUse);
+
+ if (strV.isRepeating && trimV.isRepeating) {
+ if (!outV.isNull[0]) {
+ func(outV, strV, 0, trimV, 0, 0);
+ }
+ outV.isRepeating = true;
+ return;
+ }
+
+ if (batch.selectedInUse) {
+ for (int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!outV.isNull[i]) {
+ func(outV, strV, i, trimV, trimV.isRepeating ? 0 : i, i);
+ }
+ }
+ } else {
+ for (int i = 0; i != n; i++) {
+ if (!outV.isNull[i]) {
+ func(outV, strV, i, trimV, trimV.isRepeating ? 0 : i, i);
+ }
+ }
+ }
+ }
+
+ @Override
+ public String vectorExpressionParameters() {
+ return getColumnParamString(0, inputColumnNum[0]) + ", "
+ + getColumnParamString(1, inputColumnNum[1]);
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ return (new VectorExpressionDescriptor.Builder())
+ .setMode(VectorExpressionDescriptor.Mode.PROJECTION)
+ .setNumArguments(2)
+ .setArgumentTypes(
+ VectorExpressionDescriptor.ArgumentType.STRING_FAMILY,
+ VectorExpressionDescriptor.ArgumentType.STRING_FAMILY)
+ .setInputExpressionTypes(
+ VectorExpressionDescriptor.InputExpressionType.COLUMN,
+ VectorExpressionDescriptor.InputExpressionType.COLUMN)
+ .build();
+ }
+}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimColScalar.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimColScalar.java
index 4e7b2990a61..7f68b863a4d 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimColScalar.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimColScalar.java
@@ -39,29 +39,8 @@ public StringTrimColScalar() {
*/
protected void func(BytesColumnVector outV, byte[][] vector, int[] start,
int[] length,
int batchIndex) {
-
- byte[] bytes = vector[batchIndex];
- final int startIndex = start[batchIndex];
- final int end = startIndex + length[batchIndex];
- int leftIndex = startIndex;
- while(leftIndex < end && shouldTrim(bytes[leftIndex])) {
- leftIndex++;
- }
- if (leftIndex == end) {
- outV.setVal(batchIndex, EMPTY_BYTES, 0, 0);
- return;
- }
-
- // Have at least 1 non-blank; Skip trailing blank characters.
- int rightIndex = end - 1;
- final int rightLimit = leftIndex + 1;
- while(rightIndex >= rightLimit && shouldTrim(bytes[rightIndex])) {
- rightIndex--;
- }
- final int resultLength = rightIndex - leftIndex + 1;
- if (resultLength <= 0) {
- throw new RuntimeException("Not expected");
- }
- outV.setVal(batchIndex, bytes, leftIndex, resultLength);
+ byte[] trimChars = getTrimChars();
+ trimBoth(outV, vector[batchIndex], start[batchIndex], length[batchIndex],
+ trimChars, 0, trimChars.length, batchIndex);
}
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimColScalarBase.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimColScalarBase.java
index e9ee4e44b10..6a062411850 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimColScalarBase.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimColScalarBase.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions;
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
/**
@@ -43,14 +44,79 @@ public StringTrimColScalarBase() {
}
protected boolean shouldTrim(int character) {
- for (int i = 0; i < trimChars.length; ++i) {
- if (trimChars[i] == character) {
+ return shouldTrimByte((byte) character, trimChars, 0, trimChars.length);
+ }
+
+ protected byte[] getTrimChars() {
+ return trimChars;
+ }
+
+ static boolean shouldTrimByte(byte character, byte[] trimBytes, int
trimStart, int trimLen) {
+ final int trimEnd = trimStart + trimLen;
+ for (int i = trimStart; i < trimEnd; ++i) {
+ if (trimBytes[i] == character) {
return true;
}
}
return false;
}
+ static void trimBoth(BytesColumnVector outV, byte[] bytes, int startIndex,
int length,
+ byte[] trimBytes, int trimStart, int trimLen, int batchIndex) {
+ final int end = startIndex + length;
+ int leftIndex = startIndex;
+ while (leftIndex < end && shouldTrimByte(bytes[leftIndex], trimBytes,
trimStart, trimLen)) {
+ leftIndex++;
+ }
+ if (leftIndex == end) {
+ outV.setVal(batchIndex, EMPTY_BYTES, 0, 0);
+ return;
+ }
+
+ int rightIndex = end - 1;
+ final int rightLimit = leftIndex + 1;
+ while (rightIndex >= rightLimit && shouldTrimByte(bytes[rightIndex],
trimBytes, trimStart, trimLen)) {
+ rightIndex--;
+ }
+ final int resultLength = rightIndex - leftIndex + 1;
+ if (resultLength <= 0) {
+ throw new RuntimeException("Not expected");
+ }
+ outV.setVal(batchIndex, bytes, leftIndex, resultLength);
+ }
+
+ static void trimLeft(BytesColumnVector outV, byte[] bytes, int startIndex,
int length,
+ byte[] trimBytes, int trimStart, int trimLen, int batchIndex) {
+ final int end = startIndex + length;
+ int index = startIndex;
+ while (index < end && shouldTrimByte(bytes[index], trimBytes, trimStart,
trimLen)) {
+ index++;
+ }
+
+ final int resultLength = end - index;
+ if (resultLength == 0) {
+ outV.setVal(batchIndex, EMPTY_BYTES, 0, 0);
+ return;
+ }
+ outV.setVal(batchIndex, bytes, index, resultLength);
+ }
+
+ static void trimRight(BytesColumnVector outV, byte[] bytes, int startIndex,
int length,
+ byte[] trimBytes, int trimStart, int trimLen, int batchIndex) {
+ final int start = startIndex;
+ int index = startIndex + length - 1;
+ while (index >= start && shouldTrimByte(bytes[index], trimBytes,
trimStart, trimLen)) {
+ index--;
+ }
+
+ final int resultLength = index - start + 1;
+ if (resultLength == 0) {
+ outV.setVal(batchIndex, EMPTY_BYTES, 0, 0);
+ return;
+ }
+ outV.setVal(batchIndex, bytes, start, resultLength);
+ }
+
@Override
public VectorExpressionDescriptor.Descriptor getDescriptor() {
VectorExpressionDescriptor.Builder b = new
VectorExpressionDescriptor.Builder();
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimScalarCol.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimScalarCol.java
new file mode 100644
index 00000000000..d0f6cd891e9
--- /dev/null
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimScalarCol.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+
+/**
+ * Vectorized TRIM with a scalar string and a trim-characters column.
+ */
+public class StringTrimScalarCol extends StringTrimScalarColBase {
+ private static final long serialVersionUID = 1L;
+
+ public StringTrimScalarCol(byte[] stringToTrim, int trimCharsCol, int
outputColumnNum) {
+ super(stringToTrim, trimCharsCol, outputColumnNum);
+ }
+
+ public StringTrimScalarCol() {
+ super();
+ }
+
+ @Override
+ protected void func(BytesColumnVector outV, BytesColumnVector trimV, int
trimIndex,
+ int outIndex) {
+ StringTrimColScalarBase.trimBoth(outV, stringToTrim, 0,
stringToTrim.length,
+ trimV.vector[trimIndex], trimV.start[trimIndex],
trimV.length[trimIndex], outIndex);
+ }
+}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimScalarColBase.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimScalarColBase.java
new file mode 100644
index 00000000000..e1c39661708
--- /dev/null
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringTrimScalarColBase.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import java.util.Arrays;
+
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+
+/**
+ * Base class for vectorized two-parameter trim functions with a scalar string
and a
+ * trim-characters column.
+ */
+abstract public class StringTrimScalarColBase extends VectorExpression {
+ private static final long serialVersionUID = 1L;
+
+ protected final byte[] stringToTrim;
+
+ public StringTrimScalarColBase(byte[] stringToTrim, int trimCharsCol, int
outputColumnNum) {
+ super(trimCharsCol, outputColumnNum);
+ this.stringToTrim = stringToTrim;
+ }
+
+ public StringTrimScalarColBase() {
+ super();
+ stringToTrim = null;
+ }
+
+ abstract protected void func(BytesColumnVector outV, BytesColumnVector
trimV, int trimIndex,
+ int outIndex);
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) throws HiveException {
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ BytesColumnVector trimV = (BytesColumnVector)
batch.cols[inputColumnNum[0]];
+ BytesColumnVector outV = (BytesColumnVector) batch.cols[outputColumnNum];
+ int[] sel = batch.selected;
+ int n = batch.size;
+ boolean[] trimIsNull = trimV.isNull;
+ boolean[] outputIsNull = outV.isNull;
+
+ if (n == 0) {
+ return;
+ }
+
+ outV.initBuffer();
+ outV.isRepeating = false;
+
+ if (trimV.isRepeating) {
+ if (trimV.noNulls || !trimIsNull[0]) {
+ outputIsNull[0] = false;
+ func(outV, trimV, 0, 0);
+ } else {
+ outputIsNull[0] = true;
+ outV.noNulls = false;
+ }
+ outV.isRepeating = true;
+ return;
+ }
+
+ if (trimV.noNulls) {
+ if (batch.selectedInUse) {
+ if (!outV.noNulls) {
+ for (int j = 0; j != n; j++) {
+ final int i = sel[j];
+ outputIsNull[i] = false;
+ func(outV, trimV, i, i);
+ }
+ } else {
+ for (int j = 0; j != n; j++) {
+ final int i = sel[j];
+ func(outV, trimV, i, i);
+ }
+ }
+ } else {
+ if (!outV.noNulls) {
+ Arrays.fill(outputIsNull, false);
+ outV.noNulls = true;
+ }
+ for (int i = 0; i != n; i++) {
+ func(outV, trimV, i, i);
+ }
+ }
+ } else {
+ outV.noNulls = false;
+ if (batch.selectedInUse) {
+ for (int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputIsNull[i] = trimIsNull[i];
+ if (!trimIsNull[i]) {
+ func(outV, trimV, i, i);
+ }
+ }
+ } else {
+ System.arraycopy(trimIsNull, 0, outputIsNull, 0, n);
+ for (int i = 0; i != n; i++) {
+ if (!trimIsNull[i]) {
+ func(outV, trimV, i, i);
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public String vectorExpressionParameters() {
+ return "val " + displayUtf8Bytes(stringToTrim) + ", "
+ + getColumnParamString(0, inputColumnNum[0]);
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ return (new VectorExpressionDescriptor.Builder())
+ .setMode(VectorExpressionDescriptor.Mode.PROJECTION)
+ .setNumArguments(2)
+ .setArgumentTypes(
+ VectorExpressionDescriptor.ArgumentType.STRING_FAMILY,
+ VectorExpressionDescriptor.ArgumentType.STRING_FAMILY)
+ .setInputExpressionTypes(
+ VectorExpressionDescriptor.InputExpressionType.SCALAR,
+ VectorExpressionDescriptor.InputExpressionType.COLUMN)
+ .build();
+ }
+}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLTrim.java
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLTrim.java
index 1f0bb9b37eb..92425ed44bc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLTrim.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLTrim.java
@@ -22,7 +22,9 @@
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions;
import org.apache.hadoop.hive.ql.exec.vector.expressions.StringLTrimCol;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.StringLTrimColCol;
import org.apache.hadoop.hive.ql.exec.vector.expressions.StringLTrimColScalar;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.StringLTrimScalarCol;
/**
* UDFLTrim.
@@ -33,7 +35,11 @@
"leading pad characters from str ", extended = "Example:\n"
+ " > SELECT _FUNC_(' facebook') FROM src LIMIT 1;\n" + " 'facebook'\n"
+ " > SELECT _FUNC_('xyzzxyfacebook', 'zyx') FROM src LIMIT 1;\n" + "
'facebook'")
-@VectorizedExpressions({ StringLTrimCol.class, StringLTrimColScalar.class })
+@VectorizedExpressions({
+ StringLTrimCol.class,
+ StringLTrimColScalar.class,
+ StringLTrimColCol.class,
+ StringLTrimScalarCol.class })
public class GenericUDFLTrim extends GenericUDFBaseTrim {
public GenericUDFLTrim() {
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRTrim.java
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRTrim.java
index 4e6b1f3528a..25eecdaef0a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRTrim.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRTrim.java
@@ -22,7 +22,9 @@
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions;
import org.apache.hadoop.hive.ql.exec.vector.expressions.StringRTrimCol;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.StringRTrimColCol;
import org.apache.hadoop.hive.ql.exec.vector.expressions.StringRTrimColScalar;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.StringRTrimScalarCol;
/**
* UDFRTrim.
@@ -33,7 +35,11 @@
extended = "Example:\n"
+ " > SELECT _FUNC_('facebook ') FROM src LIMIT 1;\n" + " 'facebook'\n"
+ " > SELECT _FUNC_('facebookxyzzyx', 'xzy') FROM src LIMIT 1;\n" + "
'facebook'")
-@VectorizedExpressions({ StringRTrimCol.class, StringRTrimColScalar.class })
+@VectorizedExpressions({
+ StringRTrimCol.class,
+ StringRTrimColScalar.class,
+ StringRTrimColCol.class,
+ StringRTrimScalarCol.class })
public class GenericUDFRTrim extends GenericUDFBaseTrim {
public GenericUDFRTrim() {
super("rtrim");
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTrim.java
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTrim.java
index 4b9608a2008..a742db9410b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTrim.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTrim.java
@@ -22,7 +22,9 @@
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions;
import org.apache.hadoop.hive.ql.exec.vector.expressions.StringTrimCol;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.StringTrimColCol;
import org.apache.hadoop.hive.ql.exec.vector.expressions.StringTrimColScalar;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.StringTrimScalarCol;
/**
* UDFTrim.
@@ -37,7 +39,11 @@
+ " > SELECT _FUNC_(' ' FROM ' Facebook ');\n" + " 'Facebook'\n"
+ " > SELECT _FUNC_(LEADING ' ' FROM ' Facebook ');\n" + " 'Facebook
'\n"
+ " > SELECT _FUNC_('xyfacebookyyx', 'xy');\n" + " 'facebook'")
-@VectorizedExpressions({ StringTrimCol.class, StringTrimColScalar.class})
+@VectorizedExpressions({
+ StringTrimCol.class,
+ StringTrimColScalar.class,
+ StringTrimColCol.class,
+ StringTrimScalarCol.class})
public class GenericUDFTrim extends GenericUDFBaseTrim {
public GenericUDFTrim() {
super("trim");
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
index d6ac3758891..4ce57b779e6 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
@@ -22,6 +22,7 @@
import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.hive.common.type.HiveChar;
@@ -68,7 +69,14 @@
import org.apache.hadoop.hive.ql.exec.vector.expressions.SelectColumnIsNull;
import org.apache.hadoop.hive.ql.exec.vector.expressions.SelectColumnIsTrue;
import org.apache.hadoop.hive.ql.exec.vector.expressions.StringColumnInList;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.StringLTrimColCol;
import org.apache.hadoop.hive.ql.exec.vector.expressions.StringLTrimColScalar;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.StringLTrimScalarCol;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.StringRTrimColCol;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.StringRTrimScalarCol;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.StringTrimColCol;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.StringTrimColScalar;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.StringTrimScalarCol;
import org.apache.hadoop.hive.ql.exec.vector.expressions.StringLower;
import org.apache.hadoop.hive.ql.exec.vector.expressions.StringUpper;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
@@ -132,6 +140,8 @@
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFInBloomFilter;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLTrim;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFRTrim;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTrim;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLower;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
@@ -970,6 +980,61 @@ public void testUnaryStringExpressions() throws
HiveException {
assertEquals(3, ve.getOutputColumnNum());
}
+ @Test
+ public void testTwoParameterTrimVectorExpressions() throws HiveException {
+ List<String> columns = new ArrayList<String>();
+ columns.add("b");
+ columns.add("a");
+ VectorizationContext vc = new VectorizationContext("name", columns);
+
+ ExprNodeColumnDesc col0 = new ExprNodeColumnDesc(String.class, "a",
"table", false);
+ ExprNodeColumnDesc col1 = new ExprNodeColumnDesc(String.class, "b",
"table", false);
+ ExprNodeConstantDesc trimScalar = new ExprNodeConstantDesc("xy");
+ ExprNodeConstantDesc stringScalar = new
ExprNodeConstantDesc("xyfacebookyyx");
+
+ ExprNodeGenericFuncDesc trimColScalar = new ExprNodeGenericFuncDesc();
+ trimColScalar.setTypeInfo(TypeInfoFactory.stringTypeInfo);
+ trimColScalar.setGenericUDF(new GenericUDFTrim());
+ trimColScalar.setChildren(Arrays.asList(col0, trimScalar));
+ assertEquals(StringTrimColScalar.class,
vc.getVectorExpression(trimColScalar).getClass());
+
+ ExprNodeGenericFuncDesc trimColCol = new ExprNodeGenericFuncDesc();
+ trimColCol.setTypeInfo(TypeInfoFactory.stringTypeInfo);
+ trimColCol.setGenericUDF(new GenericUDFTrim());
+ trimColCol.setChildren(Arrays.asList(col0, col1));
+ assertEquals(StringTrimColCol.class,
vc.getVectorExpression(trimColCol).getClass());
+
+ ExprNodeGenericFuncDesc trimScalarCol = new ExprNodeGenericFuncDesc();
+ trimScalarCol.setTypeInfo(TypeInfoFactory.stringTypeInfo);
+ trimScalarCol.setGenericUDF(new GenericUDFTrim());
+ trimScalarCol.setChildren(Arrays.asList(stringScalar, col1));
+ assertEquals(StringTrimScalarCol.class,
vc.getVectorExpression(trimScalarCol).getClass());
+
+ ExprNodeGenericFuncDesc ltrimColCol = new ExprNodeGenericFuncDesc();
+ ltrimColCol.setTypeInfo(TypeInfoFactory.stringTypeInfo);
+ ltrimColCol.setGenericUDF(new GenericUDFLTrim());
+ ltrimColCol.setChildren(Arrays.asList(col0, col1));
+ assertEquals(StringLTrimColCol.class,
vc.getVectorExpression(ltrimColCol).getClass());
+
+ ExprNodeGenericFuncDesc ltrimScalarCol = new ExprNodeGenericFuncDesc();
+ ltrimScalarCol.setTypeInfo(TypeInfoFactory.stringTypeInfo);
+ ltrimScalarCol.setGenericUDF(new GenericUDFLTrim());
+ ltrimScalarCol.setChildren(Arrays.asList(stringScalar, col1));
+ assertEquals(StringLTrimScalarCol.class,
vc.getVectorExpression(ltrimScalarCol).getClass());
+
+ ExprNodeGenericFuncDesc rtrimColCol = new ExprNodeGenericFuncDesc();
+ rtrimColCol.setTypeInfo(TypeInfoFactory.stringTypeInfo);
+ rtrimColCol.setGenericUDF(new GenericUDFRTrim());
+ rtrimColCol.setChildren(Arrays.asList(col0, col1));
+ assertEquals(StringRTrimColCol.class,
vc.getVectorExpression(rtrimColCol).getClass());
+
+ ExprNodeGenericFuncDesc rtrimScalarCol = new ExprNodeGenericFuncDesc();
+ rtrimScalarCol.setTypeInfo(TypeInfoFactory.stringTypeInfo);
+ rtrimScalarCol.setGenericUDF(new GenericUDFRTrim());
+ rtrimScalarCol.setChildren(Arrays.asList(stringScalar, col1));
+ assertEquals(StringRTrimScalarCol.class,
vc.getVectorExpression(rtrimScalarCol).getClass());
+ }
+
@Test
public void testMathFunctions() throws HiveException {
ExprNodeGenericFuncDesc mathFuncExpr = new ExprNodeGenericFuncDesc();
diff --git a/ql/src/test/queries/clientpositive/udf_ltrim_vector.q
b/ql/src/test/queries/clientpositive/udf_ltrim_vector.q
index 1a6d9479e8d..4ddb6fe3349 100644
--- a/ql/src/test/queries/clientpositive/udf_ltrim_vector.q
+++ b/ql/src/test/queries/clientpositive/udf_ltrim_vector.q
@@ -1,6 +1,6 @@
-create table t1 (col0 string);
+create table t1 (col0 string, col1 string);
-insert into t1(col0) values ('xyfacebookyyx'),(' tech ');
+insert into t1(col0, col1) values ('xyfacebookyyx', 'xy'),(' tech ', ' ');
explain vectorization expression
select ltrim(col0) from t1 group by col0;
@@ -13,4 +13,25 @@ select ltrim(col0, 'xy') from t1 group by col0;
select ltrim(col0, 'xy') from t1 group by col0;
+
+explain vectorization expression
+select ltrim(col0, col1) from t1 group by col0, col1;
+
+select ltrim(col0, col1) from t1 group by col0, col1;
+
+
+explain vectorization expression
+select ltrim('xyfacebookyyx', col1) from t1 group by col1;
+
+select ltrim('xyfacebookyyx', col1) from t1 group by col1;
+
+
+insert into t1(col0, col1) values (null, 'xy'),('foo', null);
+
+select ltrim(col0, col1) from t1 where col0 is null;
+select ltrim(col0, col1) from t1 where col1 is null;
+
+select ltrim(col0, null) from t1 group by col0;
+select ltrim(null, col1) from t1 group by col1;
+
drop table t1;
diff --git a/ql/src/test/queries/clientpositive/udf_rtrim_vector.q
b/ql/src/test/queries/clientpositive/udf_rtrim_vector.q
index e10f3726d43..02bced5ef4f 100644
--- a/ql/src/test/queries/clientpositive/udf_rtrim_vector.q
+++ b/ql/src/test/queries/clientpositive/udf_rtrim_vector.q
@@ -1,6 +1,6 @@
-create table t1 (col0 string);
+create table t1 (col0 string, col1 string);
-insert into t1(col0) values ('xyfacebookyyx'),(' tech ');
+insert into t1(col0, col1) values ('xyfacebookyyx', 'xy'),(' tech ', ' ');
explain vectorization expression
select rtrim(col0) from t1 group by col0;
@@ -13,4 +13,25 @@ select rtrim(col0, 'xy') from t1 group by col0;
select rtrim(col0, 'xy') from t1 group by col0;
+
+explain vectorization expression
+select rtrim(col0, col1) from t1 group by col0, col1;
+
+select rtrim(col0, col1) from t1 group by col0, col1;
+
+
+explain vectorization expression
+select rtrim('xyfacebookyyx', col1) from t1 group by col1;
+
+select rtrim('xyfacebookyyx', col1) from t1 group by col1;
+
+
+insert into t1(col0, col1) values (null, 'xy'),('foo', null);
+
+select rtrim(col0, col1) from t1 where col0 is null;
+select rtrim(col0, col1) from t1 where col1 is null;
+
+select rtrim(col0, null) from t1 group by col0;
+select rtrim(null, col1) from t1 group by col1;
+
drop table t1;
diff --git a/ql/src/test/queries/clientpositive/udf_trim_vector.q
b/ql/src/test/queries/clientpositive/udf_trim_vector.q
index 7028d8aaeb8..b883d27cf12 100644
--- a/ql/src/test/queries/clientpositive/udf_trim_vector.q
+++ b/ql/src/test/queries/clientpositive/udf_trim_vector.q
@@ -1,6 +1,6 @@
-create table t1 (col0 string);
+create table t1 (col0 string, col1 string);
-insert into t1(col0) values ('xyfacebookyyx'),(' tech ');
+insert into t1(col0, col1) values ('xyfacebookyyx', 'xy'),(' tech ', ' ');
explain vectorization expression
select trim(col0) from t1 group by col0;
@@ -13,4 +13,25 @@ select trim(col0, 'xy') from t1 group by col0;
select trim(col0, 'xy') from t1 group by col0;
+
+explain vectorization expression
+select trim(col0, col1) from t1 group by col0, col1;
+
+select trim(col0, col1) from t1 group by col0, col1;
+
+
+explain vectorization expression
+select trim('xyfacebookyyx', col1) from t1 group by col1;
+
+select trim('xyfacebookyyx', col1) from t1 group by col1;
+
+
+insert into t1(col0, col1) values (null, 'xy'),('foo', null);
+
+select trim(col0, col1) from t1 where col0 is null;
+select trim(col0, col1) from t1 where col1 is null;
+
+select trim(col0, null) from t1 group by col0;
+select trim(null, col1) from t1 group by col1;
+
drop table t1;
diff --git a/ql/src/test/results/clientpositive/llap/udf_ltrim_vector.q.out
b/ql/src/test/results/clientpositive/llap/udf_ltrim_vector.q.out
index d3bf066f254..3b5db011c37 100644
--- a/ql/src/test/results/clientpositive/llap/udf_ltrim_vector.q.out
+++ b/ql/src/test/results/clientpositive/llap/udf_ltrim_vector.q.out
@@ -1,20 +1,21 @@
-PREHOOK: query: create table t1 (col0 string)
+PREHOOK: query: create table t1 (col0 string, col1 string)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@t1
-POSTHOOK: query: create table t1 (col0 string)
+POSTHOOK: query: create table t1 (col0 string, col1 string)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@t1
-PREHOOK: query: insert into t1(col0) values ('xyfacebookyyx'),(' tech ')
+PREHOOK: query: insert into t1(col0, col1) values ('xyfacebookyyx', 'xy'),('
tech ', ' ')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@t1
-POSTHOOK: query: insert into t1(col0) values ('xyfacebookyyx'),(' tech ')
+POSTHOOK: query: insert into t1(col0, col1) values ('xyfacebookyyx', 'xy'),('
tech ', ' ')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@t1
POSTHOOK: Lineage: t1.col0 SCRIPT []
+POSTHOOK: Lineage: t1.col1 SCRIPT []
PREHOOK: query: explain vectorization expression
select ltrim(col0) from t1 group by col0
PREHOOK: type: QUERY
@@ -279,6 +280,321 @@ POSTHOOK: Input: default@t1
#### A masked pattern was here ####
tech
facebookyyx
+PREHOOK: query: explain vectorization expression
+select ltrim(col0, col1) from t1 group by col0, col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization expression
+select ltrim(col0, col1) from t1 group by col0, col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 2 Data size: 364 Basic stats: COMPLETE
Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: col0 (type: string), col1 (type: string)
+ outputColumnNames: col0, col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1]
+ Statistics: Num rows: 2 Data size: 364 Basic stats:
COMPLETE Column stats: COMPLETE
+ Group By Operator
+ Group By Vectorization:
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 0:string, col 1:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: []
+ keys: col0 (type: string), col1 (type: string)
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 364 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type:
string)
+ null sort order: zz
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string),
_col1 (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 2 Data size: 364 Basic stats:
COMPLETE Column stats: COMPLETE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vector.serde.deserialize IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ Group By Vectorization:
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string, col 1:string
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: []
+ keys: KEY._col0 (type: string), KEY._col1 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 364 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: ltrim(_col0, _col1) (type: string)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2]
+ selectExpressions: StringLTrimColCol(col 0:string, col
1:string) -> 2:string
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 368 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select ltrim(col0, col1) from t1 group by col0, col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select ltrim(col0, col1) from t1 group by col0, col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+tech
+facebookyyx
+PREHOOK: query: explain vectorization expression
+select ltrim('xyfacebookyyx', col1) from t1 group by col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization expression
+select ltrim('xyfacebookyyx', col1) from t1 group by col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE
Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: col1 (type: string)
+ outputColumnNames: col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1]
+ Statistics: Num rows: 2 Data size: 172 Basic stats:
COMPLETE Column stats: COMPLETE
+ Group By Operator
+ Group By Vectorization:
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: []
+ keys: col1 (type: string)
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 172 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkStringOperator
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 2 Data size: 172 Basic stats:
COMPLETE Column stats: COMPLETE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vector.serde.deserialize IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ Group By Vectorization:
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: []
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: ltrim('xyfacebookyyx', _col0) (type: string)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1]
+ selectExpressions: StringLTrimScalarCol(val
xyfacebookyyx, col 0:string) -> 1:string
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 368 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select ltrim('xyfacebookyyx', col1) from t1 group by col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select ltrim('xyfacebookyyx', col1) from t1 group by col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+xyfacebookyyx
+facebookyyx
+PREHOOK: query: insert into t1(col0, col1) values (null, 'xy'),('foo', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t1
+POSTHOOK: query: insert into t1(col0, col1) values (null, 'xy'),('foo', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t1
+POSTHOOK: Lineage: t1.col0 SCRIPT []
+POSTHOOK: Lineage: t1.col1 SCRIPT []
+PREHOOK: query: select ltrim(col0, col1) from t1 where col0 is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select ltrim(col0, col1) from t1 where col0 is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+NULL
+PREHOOK: query: select ltrim(col0, col1) from t1 where col1 is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select ltrim(col0, col1) from t1 where col1 is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+NULL
+PREHOOK: query: select ltrim(col0, null) from t1 group by col0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select ltrim(col0, null) from t1 group by col0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+NULL
+NULL
+NULL
+NULL
+PREHOOK: query: select ltrim(null, col1) from t1 group by col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select ltrim(null, col1) from t1 group by col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+NULL
+NULL
+NULL
PREHOOK: query: drop table t1
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@t1
diff --git a/ql/src/test/results/clientpositive/llap/udf_rtrim_vector.q.out
b/ql/src/test/results/clientpositive/llap/udf_rtrim_vector.q.out
index d5f20d0770c..46d6d0d97a3 100644
--- a/ql/src/test/results/clientpositive/llap/udf_rtrim_vector.q.out
+++ b/ql/src/test/results/clientpositive/llap/udf_rtrim_vector.q.out
@@ -1,20 +1,21 @@
-PREHOOK: query: create table t1 (col0 string)
+PREHOOK: query: create table t1 (col0 string, col1 string)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@t1
-POSTHOOK: query: create table t1 (col0 string)
+POSTHOOK: query: create table t1 (col0 string, col1 string)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@t1
-PREHOOK: query: insert into t1(col0) values ('xyfacebookyyx'),(' tech ')
+PREHOOK: query: insert into t1(col0, col1) values ('xyfacebookyyx', 'xy'),('
tech ', ' ')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@t1
-POSTHOOK: query: insert into t1(col0) values ('xyfacebookyyx'),(' tech ')
+POSTHOOK: query: insert into t1(col0, col1) values ('xyfacebookyyx', 'xy'),('
tech ', ' ')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@t1
POSTHOOK: Lineage: t1.col0 SCRIPT []
+POSTHOOK: Lineage: t1.col1 SCRIPT []
PREHOOK: query: explain vectorization expression
select rtrim(col0) from t1 group by col0
PREHOOK: type: QUERY
@@ -279,6 +280,321 @@ POSTHOOK: Input: default@t1
#### A masked pattern was here ####
tech
xyfacebook
+PREHOOK: query: explain vectorization expression
+select rtrim(col0, col1) from t1 group by col0, col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization expression
+select rtrim(col0, col1) from t1 group by col0, col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 2 Data size: 364 Basic stats: COMPLETE
Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: col0 (type: string), col1 (type: string)
+ outputColumnNames: col0, col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1]
+ Statistics: Num rows: 2 Data size: 364 Basic stats:
COMPLETE Column stats: COMPLETE
+ Group By Operator
+ Group By Vectorization:
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 0:string, col 1:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: []
+ keys: col0 (type: string), col1 (type: string)
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 364 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type:
string)
+ null sort order: zz
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string),
_col1 (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 2 Data size: 364 Basic stats:
COMPLETE Column stats: COMPLETE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vector.serde.deserialize IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ Group By Vectorization:
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string, col 1:string
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: []
+ keys: KEY._col0 (type: string), KEY._col1 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 364 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: rtrim(_col0, _col1) (type: string)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2]
+ selectExpressions: StringRTrimColCol(col 0:string, col
1:string) -> 2:string
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 368 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select rtrim(col0, col1) from t1 group by col0, col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select rtrim(col0, col1) from t1 group by col0, col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+ tech
+xyfacebook
+PREHOOK: query: explain vectorization expression
+select rtrim('xyfacebookyyx', col1) from t1 group by col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization expression
+select rtrim('xyfacebookyyx', col1) from t1 group by col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE
Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: col1 (type: string)
+ outputColumnNames: col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1]
+ Statistics: Num rows: 2 Data size: 172 Basic stats:
COMPLETE Column stats: COMPLETE
+ Group By Operator
+ Group By Vectorization:
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: []
+ keys: col1 (type: string)
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 172 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkStringOperator
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 2 Data size: 172 Basic stats:
COMPLETE Column stats: COMPLETE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vector.serde.deserialize IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ Group By Vectorization:
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: []
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: rtrim('xyfacebookyyx', _col0) (type: string)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1]
+ selectExpressions: StringRTrimScalarCol(val
xyfacebookyyx, col 0:string) -> 1:string
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 368 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select rtrim('xyfacebookyyx', col1) from t1 group by col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select rtrim('xyfacebookyyx', col1) from t1 group by col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+xyfacebookyyx
+xyfacebook
+PREHOOK: query: insert into t1(col0, col1) values (null, 'xy'),('foo', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t1
+POSTHOOK: query: insert into t1(col0, col1) values (null, 'xy'),('foo', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t1
+POSTHOOK: Lineage: t1.col0 SCRIPT []
+POSTHOOK: Lineage: t1.col1 SCRIPT []
+PREHOOK: query: select rtrim(col0, col1) from t1 where col0 is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select rtrim(col0, col1) from t1 where col0 is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+NULL
+PREHOOK: query: select rtrim(col0, col1) from t1 where col1 is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select rtrim(col0, col1) from t1 where col1 is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+NULL
+PREHOOK: query: select rtrim(col0, null) from t1 group by col0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select rtrim(col0, null) from t1 group by col0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+NULL
+NULL
+NULL
+NULL
+PREHOOK: query: select rtrim(null, col1) from t1 group by col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select rtrim(null, col1) from t1 group by col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+NULL
+NULL
+NULL
PREHOOK: query: drop table t1
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@t1
diff --git a/ql/src/test/results/clientpositive/llap/udf_trim_vector.q.out
b/ql/src/test/results/clientpositive/llap/udf_trim_vector.q.out
index e5028efa15e..7e9d8100b95 100644
--- a/ql/src/test/results/clientpositive/llap/udf_trim_vector.q.out
+++ b/ql/src/test/results/clientpositive/llap/udf_trim_vector.q.out
@@ -1,20 +1,21 @@
-PREHOOK: query: create table t1 (col0 string)
+PREHOOK: query: create table t1 (col0 string, col1 string)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@t1
-POSTHOOK: query: create table t1 (col0 string)
+POSTHOOK: query: create table t1 (col0 string, col1 string)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@t1
-PREHOOK: query: insert into t1(col0) values ('xyfacebookyyx'),(' tech ')
+PREHOOK: query: insert into t1(col0, col1) values ('xyfacebookyyx', 'xy'),('
tech ', ' ')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@t1
-POSTHOOK: query: insert into t1(col0) values ('xyfacebookyyx'),(' tech ')
+POSTHOOK: query: insert into t1(col0, col1) values ('xyfacebookyyx', 'xy'),('
tech ', ' ')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@t1
POSTHOOK: Lineage: t1.col0 SCRIPT []
+POSTHOOK: Lineage: t1.col1 SCRIPT []
PREHOOK: query: explain vectorization expression
select trim(col0) from t1 group by col0
PREHOOK: type: QUERY
@@ -279,6 +280,321 @@ POSTHOOK: Input: default@t1
#### A masked pattern was here ####
tech
facebook
+PREHOOK: query: explain vectorization expression
+select trim(col0, col1) from t1 group by col0, col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization expression
+select trim(col0, col1) from t1 group by col0, col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 2 Data size: 364 Basic stats: COMPLETE
Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: col0 (type: string), col1 (type: string)
+ outputColumnNames: col0, col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1]
+ Statistics: Num rows: 2 Data size: 364 Basic stats:
COMPLETE Column stats: COMPLETE
+ Group By Operator
+ Group By Vectorization:
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 0:string, col 1:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: []
+ keys: col0 (type: string), col1 (type: string)
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 364 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type:
string)
+ null sort order: zz
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string),
_col1 (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 2 Data size: 364 Basic stats:
COMPLETE Column stats: COMPLETE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vector.serde.deserialize IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ Group By Vectorization:
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string, col 1:string
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: []
+ keys: KEY._col0 (type: string), KEY._col1 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 364 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: trim(_col0, _col1) (type: string)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2]
+ selectExpressions: StringTrimColCol(col 0:string, col
1:string) -> 2:string
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 368 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select trim(col0, col1) from t1 group by col0, col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select trim(col0, col1) from t1 group by col0, col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+tech
+facebook
+PREHOOK: query: explain vectorization expression
+select trim('xyfacebookyyx', col1) from t1 group by col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization expression
+select trim('xyfacebookyyx', col1) from t1 group by col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE
Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: col1 (type: string)
+ outputColumnNames: col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1]
+ Statistics: Num rows: 2 Data size: 172 Basic stats:
COMPLETE Column stats: COMPLETE
+ Group By Operator
+ Group By Vectorization:
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: []
+ keys: col1 (type: string)
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 172 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkStringOperator
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 2 Data size: 172 Basic stats:
COMPLETE Column stats: COMPLETE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vector.serde.deserialize IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ Group By Vectorization:
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: []
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: trim('xyfacebookyyx', _col0) (type: string)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1]
+ selectExpressions: StringTrimScalarCol(val
xyfacebookyyx, col 0:string) -> 1:string
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 368 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select trim('xyfacebookyyx', col1) from t1 group by col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select trim('xyfacebookyyx', col1) from t1 group by col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+xyfacebookyyx
+facebook
+PREHOOK: query: insert into t1(col0, col1) values (null, 'xy'),('foo', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t1
+POSTHOOK: query: insert into t1(col0, col1) values (null, 'xy'),('foo', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t1
+POSTHOOK: Lineage: t1.col0 SCRIPT []
+POSTHOOK: Lineage: t1.col1 SCRIPT []
+PREHOOK: query: select trim(col0, col1) from t1 where col0 is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select trim(col0, col1) from t1 where col0 is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+NULL
+PREHOOK: query: select trim(col0, col1) from t1 where col1 is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select trim(col0, col1) from t1 where col1 is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+NULL
+PREHOOK: query: select trim(col0, null) from t1 group by col0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select trim(col0, null) from t1 group by col0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+NULL
+NULL
+NULL
+NULL
+PREHOOK: query: select trim(null, col1) from t1 group by col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select trim(null, col1) from t1 group by col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+NULL
+NULL
+NULL
PREHOOK: query: drop table t1
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@t1