This is an automated email from the ASF dual-hosted git repository.
lgbo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 61ab633d21 [GLUTEN-9682][Flink] Support double in vector-to-row
conversion (#9683)
61ab633d21 is described below
commit 61ab633d21fc752af48d03429b5a093a5d643ecc
Author: lgbo <[email protected]>
AuthorDate: Tue May 27 09:56:08 2025 +0800
[GLUTEN-9682][Flink] Support double in vector-to-row conversion (#9683)
---
.../apache/gluten/util/LogicalTypeConverter.java | 3 +
.../gluten/vectorized/ArrowVectorAccessor.java | 142 +++++++++++++++++++++
.../gluten/vectorized/ArrowVectorWriter.java | 26 +++-
.../vectorized/FlinkRowToVLVectorConvertor.java | 37 +++---
.../table/runtime/stream/custom/ScanTest.java | 33 +++--
5 files changed, 206 insertions(+), 35 deletions(-)
diff --git
a/gluten-flink/runtime/src/main/java/org/apache/gluten/util/LogicalTypeConverter.java
b/gluten-flink/runtime/src/main/java/org/apache/gluten/util/LogicalTypeConverter.java
index e2a1ccc2e1..e342e5d94b 100644
---
a/gluten-flink/runtime/src/main/java/org/apache/gluten/util/LogicalTypeConverter.java
+++
b/gluten-flink/runtime/src/main/java/org/apache/gluten/util/LogicalTypeConverter.java
@@ -22,6 +22,7 @@ import org.apache.flink.table.types.logical.BigIntType;
import org.apache.flink.table.types.logical.BooleanType;
import org.apache.flink.table.types.logical.DayTimeIntervalType;
import org.apache.flink.table.types.logical.DecimalType;
+import org.apache.flink.table.types.logical.DoubleType;
import org.apache.flink.table.types.logical.IntType;
import org.apache.flink.table.types.logical.LogicalType;
import org.apache.flink.table.types.logical.RowType;
@@ -49,6 +50,8 @@ public class LogicalTypeConverter {
return new IntegerType();
} else if (logicalType instanceof BigIntType) {
return new io.github.zhztheplayer.velox4j.type.BigIntType();
+ } else if (logicalType instanceof DoubleType) {
+ return new io.github.zhztheplayer.velox4j.type.DoubleType();
} else if (logicalType instanceof VarCharType) {
return new io.github.zhztheplayer.velox4j.type.VarCharType();
} else if (logicalType instanceof TimestampType) {
diff --git
a/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/ArrowVectorAccessor.java
b/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/ArrowVectorAccessor.java
new file mode 100644
index 0000000000..704d1ee08e
--- /dev/null
+++
b/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/ArrowVectorAccessor.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gluten.vectorized;
+
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.BitVector;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.Float8Vector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.complex.StructVector;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.binary.BinaryStringData;
+
+import io.github.zhztheplayer.velox4j.type.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/*
+ * This module is used to convert column vector to flink generic rows.
+ * BinaryRowData is not supported here.
+ */
+public abstract class ArrowVectorAccessor {
+ public static ArrowVectorAccessor create(FieldVector vector) {
+ if (vector instanceof BitVector) {
+ return new BooleanVectorAccessor(vector);
+ } else if (vector instanceof IntVector) {
+ return new IntVectorAccessor(vector);
+ } else if (vector instanceof BigIntVector) {
+ return new BigIntVectorAccessor(vector);
+ } else if (vector instanceof Float8Vector) {
+ return new DoubleVectorAccessor(vector);
+ } else if (vector instanceof VarCharVector) {
+ return new VarCharVectorAccessor(vector);
+ } else if (vector instanceof StructVector) {
+ return new StructVectorAccessor(vector);
+ } else {
+ throw new UnsupportedOperationException("ArrowVectorAccessor.
Unsupported type: " + vector.getClass().getName());
+ }
+ }
+
+ // A general method to extract values from the vector.
+ public Object get(int rowIndex) {
+ throw new UnsupportedOperationException("get not supported");
+ }
+}
+
+class BooleanVectorAccessor extends ArrowVectorAccessor {
+ private final BitVector vector;
+ public BooleanVectorAccessor(FieldVector vector) {
+ this.vector = (BitVector) vector;
+ }
+ @Override
+ public Object get(int rowIndex) {
+ return vector.get(rowIndex) != 0;
+ }
+}
+
+class IntVectorAccessor extends ArrowVectorAccessor {
+ private final IntVector vector;
+ public IntVectorAccessor(FieldVector vector) {
+ this.vector = (IntVector) vector;
+ }
+ @Override
+ public Object get(int rowIndex) {
+ return vector.get(rowIndex);
+ }
+}
+
+class BigIntVectorAccessor extends ArrowVectorAccessor {
+ private final BigIntVector vector;
+ public BigIntVectorAccessor(FieldVector vector) {
+ this.vector = (BigIntVector) vector;
+ }
+ @Override
+ public Object get(int rowIndex) {
+ return vector.get(rowIndex);
+ }
+}
+
+class DoubleVectorAccessor extends ArrowVectorAccessor {
+ private final Float8Vector vector;
+ public DoubleVectorAccessor(FieldVector vector) {
+ this.vector = (Float8Vector) vector;
+ }
+ @Override
+ public Object get(int rowIndex) {
+ return vector.get(rowIndex);
+ }
+}
+
+class VarCharVectorAccessor extends ArrowVectorAccessor {
+ private final VarCharVector vector;
+ public VarCharVectorAccessor(FieldVector vector) {
+ this.vector = (VarCharVector) vector;
+ }
+ @Override
+ public Object get(int rowIndex) {
+ return BinaryStringData.fromBytes(vector.get(rowIndex));
+ }
+}
+
+class StructVectorAccessor extends ArrowVectorAccessor {
+ private final StructVector vector;
+ private final List<ArrowVectorAccessor> fieldAccessors;
+ private int fieldCount = 0;
+ public StructVectorAccessor(FieldVector vector) {
+ this.vector = (StructVector) vector;
+ // size() returns the number of child vectors
+ this.fieldCount = this.vector.size();
+ this.fieldAccessors = new ArrayList<>(fieldCount);
+ for (int i = 0; i < this.fieldCount; i++) {
+ FieldVector fieldVector =
(FieldVector)this.vector.getChildByOrdinal(i);
+ this.fieldAccessors.add(i,
ArrowVectorAccessor.create(fieldVector));
+ }
+ }
+ @Override
+ public Object get(int rowIndex) {
+ int fieldCount = vector.size();
+ Object[] fieldValues = new Object[fieldCount];
+ for (int i = 0; i < fieldCount; i++) {
+ fieldValues[i] = fieldAccessors.get(i).get(rowIndex);
+ }
+ return GenericRowData.of(fieldValues);
+ }
+}
diff --git
a/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/ArrowVectorWriter.java
b/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/ArrowVectorWriter.java
index 9f3f41369b..35f6c78ee4 100644
---
a/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/ArrowVectorWriter.java
+++
b/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/ArrowVectorWriter.java
@@ -22,6 +22,7 @@ import io.github.zhztheplayer.velox4j.type.*;
import org.apache.arrow.flatbuf.Int;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.complex.StructVector;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
import org.apache.arrow.vector.types.TimeUnit;
import org.apache.arrow.vector.types.pojo.*;
import org.apache.arrow.vector.*;
@@ -49,6 +50,8 @@ public abstract class ArrowVectorWriter {
return new BooleanVectorWriter(fieldType, allocator, vector);
} else if (fieldType instanceof BigIntType) {
return new BigIntVectorWriter(fieldType, allocator, vector);
+ } else if (fieldType instanceof DoubleType) {
+ return new Float8VectorWriter(fieldType, allocator, vector);
} else if (fieldType instanceof VarCharType) {
return new VarCharVectorWriter(fieldType, allocator, vector);
} else if (fieldType instanceof TimestampType) {
@@ -56,7 +59,7 @@ public abstract class ArrowVectorWriter {
} else if (fieldType instanceof RowType) {
return new StructVectorWriter(fieldType, allocator, vector);
} else {
- throw new UnsupportedOperationException("Unsupported type: " +
fieldType);
+ throw new UnsupportedOperationException("ArrowVectorWriter.
Unsupported type: " + fieldType);
}
}
public void write(int fieldIndex, RowData rowData) {
@@ -91,13 +94,15 @@ class FieldVectorCreator {
return field.createVector(allocator);
}
- private static ArrowType toArroyType(Type dataType, String timeZoneId) {
+ private static ArrowType toArrowType(Type dataType, String timeZoneId) {
if (dataType instanceof BooleanType) {
return ArrowType.Bool.INSTANCE;
} else if (dataType instanceof IntegerType) {
return new ArrowType.Int(8 * 4, true);
} else if (dataType instanceof BigIntType) {
return new ArrowType.Int(8 * 8, true);
+ } else if (dataType instanceof DoubleType) {
+ return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
} else if (dataType instanceof VarCharType) {
return ArrowType.Utf8.INSTANCE;
} else if (dataType instanceof TimestampType) {
@@ -132,7 +137,7 @@ class FieldVectorCreator {
return new Field(name, strcutType, subFields);
} else {
// TODO: support nullable
- ArrowType arrowType = toArroyType(dataType, timeZoneId);
+ ArrowType arrowType = toArrowType(dataType, timeZoneId);
FieldType fieldType = new FieldType(nullable, arrowType, null);
return new Field(name, fieldType, new ArrayList<>());
}
@@ -185,6 +190,21 @@ class BigIntVectorWriter extends ArrowVectorWriter {
}
}
+class Float8VectorWriter extends ArrowVectorWriter {
+ private final Float8Vector float8Vector;
+
+ public Float8VectorWriter(Type fieldType, BufferAllocator allocator,
FieldVector vector) {
+ super(vector);
+ this.float8Vector = (Float8Vector) vector;
+ }
+
+ @Override
+ public void write(int fieldIndex, RowData rowData) {
+ float8Vector.setSafe(valueCount, rowData.getDouble(fieldIndex));
+ valueCount++;
+ }
+}
+
class VarCharVectorWriter extends ArrowVectorWriter {
private final VarCharVector varCharVector;
diff --git
a/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/FlinkRowToVLVectorConvertor.java
b/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/FlinkRowToVLVectorConvertor.java
index 631681eda0..1c15d6fd90 100644
---
a/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/FlinkRowToVLVectorConvertor.java
+++
b/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/FlinkRowToVLVectorConvertor.java
@@ -22,6 +22,7 @@ import io.github.zhztheplayer.velox4j.data.RowVector;
import io.github.zhztheplayer.velox4j.session.Session;
import io.github.zhztheplayer.velox4j.type.BigIntType;
import io.github.zhztheplayer.velox4j.type.BooleanType;
+import io.github.zhztheplayer.velox4j.type.DoubleType;
import io.github.zhztheplayer.velox4j.type.IntegerType;
import io.github.zhztheplayer.velox4j.type.RowType;
import io.github.zhztheplayer.velox4j.type.TimestampType;
@@ -32,12 +33,14 @@ import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.BitVector;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.Float8Vector;
import org.apache.arrow.vector.TimeStampMilliVector;
import org.apache.arrow.vector.VarCharVector;
import org.apache.arrow.vector.complex.StructVector;
import org.apache.arrow.vector.table.Table;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.gluten.vectorized.ArrowVectorAccessor;
import org.apache.gluten.vectorized.ArrowVectorWriter;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.RowData;
@@ -81,35 +84,20 @@ public class FlinkRowToVLVectorConvertor {
// TODO: support more types
BaseVector loadedVector = null;
FieldVector structVector = null;
+
try{
loadedVector = rowVector.loadedVector();
// The result is StructVector
- structVector = Arrow.toArrowVector(
- allocator,
- loadedVector);
+ structVector = Arrow.toArrowVector(allocator,loadedVector);
final List<FieldVector> fieldVectors =
structVector.getChildrenFromFields();
+ List<ArrowVectorAccessor> accessors =
buildArrowVectorAccessors(fieldVectors);
List<RowData> rowDatas = new ArrayList<>(rowVector.getSize());
for (int j = 0; j < rowVector.getSize(); j++) {
- List<Object> fieldValues = new ArrayList<>(rowType.size());
+ Object[] fieldValues = new Object[rowType.size()];
for (int i = 0; i < rowType.size(); i++) {
- Type fieldType = rowType.getChildren().get(i);
- if (fieldType instanceof BooleanType) {
- // BitVector returns are integer, need to convert to
boolean
- fieldValues.add(i, ((BitVector)
fieldVectors.get(i)).get(j) != 0);
- } else if (fieldType instanceof IntegerType) {
- fieldValues.add(i, ((IntVector)
fieldVectors.get(i)).get(j));
- } else if (fieldType instanceof BigIntType) {
- fieldValues.add(i, ((BigIntVector)
fieldVectors.get(i)).get(j));
- } else if (fieldType instanceof VarCharType) {
- fieldValues.add(
- i,
- BinaryStringData.fromBytes(
- ((VarCharVector)
fieldVectors.get(i)).get(j)));
- } else {
- throw new RuntimeException("Unsupported field type: "
+ fieldType);
- }
+ fieldValues[i] = accessors.get(i).get(j);
}
- rowDatas.add(GenericRowData.of(fieldValues.toArray()));
+ rowDatas.add(GenericRowData.of(fieldValues));
}
return rowDatas;
} finally {
@@ -124,4 +112,11 @@ public class FlinkRowToVLVectorConvertor {
}
}
+ private static List<ArrowVectorAccessor>
buildArrowVectorAccessors(List<FieldVector> vectors) {
+ List<ArrowVectorAccessor> accessors = new ArrayList<>(vectors.size());
+ for (int i = 0; i < vectors.size(); ++i) {
+ accessors.add(i, ArrowVectorAccessor.create(vectors.get(i)));
+ }
+ return accessors;
+ }
}
diff --git
a/gluten-flink/ut/src/test/java/org/apache/gluten/table/runtime/stream/custom/ScanTest.java
b/gluten-flink/ut/src/test/java/org/apache/gluten/table/runtime/stream/custom/ScanTest.java
index b0dd716e77..9f083b16cd 100644
---
a/gluten-flink/ut/src/test/java/org/apache/gluten/table/runtime/stream/custom/ScanTest.java
+++
b/gluten-flink/ut/src/test/java/org/apache/gluten/table/runtime/stream/custom/ScanTest.java
@@ -38,30 +38,41 @@ class ScanTest extends GlutenStreamingTestBase {
@BeforeEach
public void before() throws Exception {
super.before();
+
+ }
+
+ @Test
+ void testFilter() {
List<Row> rows =
Arrays.asList(Row.of(1, 1L, "1"),
Row.of(2, 2L, "2"),
Row.of(3, 3L, "3"));
createSimpleBoundedValuesTable("MyTable", "a int, b bigint, c string",
rows);
+ String query = "select a, b as b,c, a > 2 from MyTable where a > 0";
+ LOG.info("execution plan: {}", explainExecutionPlan(query));
+ runAndCheck(query, Arrays.asList("+I[1, 1, 1, false]", "+I[2, 2, 2,
false]", "+I[3, 3, 3, true]"));
+ }
- List<Row> structRows =
+ @Test
+ void testStructScan() {
+ List<Row> rows =
Arrays.asList(Row.of(1, Row.of(2L, "abc")),
Row.of(2, Row.of(6L, "def")),
Row.of(3, Row.of(8L, "ghi")));
- createSimpleBoundedValuesTable("MyTable2", "a int, b ROW<d bigint, e
string>", structRows);
- }
+ createSimpleBoundedValuesTable("strctTbl", "a int, b ROW<x bigint, y
string>", rows);
+ String query1 = "select a, b.x, b.y from strctTbl where a > 0";
+ runAndCheck(query1, Arrays.asList("+I[1, 2, abc]", "+I[2, 6, def]",
"+I[3, 8, ghi]"));
- @Test
- void testFilter() {
- String query = "select a, b as b,c, a > 2 from MyTable where a > 0";
- LOG.info("execution plan: {}", explainExecutionPlan(query));
- runAndCheck(query, Arrays.asList("+I[1, 1, 1, false]", "+I[2, 2, 2,
false]", "+I[3, 3, 3, true]"));
+ String query2 = "select a, b from strctTbl where a > 1";
+ runAndCheck(query2, Arrays.asList("+I[2, +I[6, def]]", "+I[3, +I[8,
ghi]]"));
}
@Test
- void testFilterWithStruct() {
- String query = "select a, b.d, b.e from MyTable2 where a > 0";
- runAndCheck(query, Arrays.asList("+I[1, 2, abc]", "+I[2, 6, def]",
"+I[3, 8, ghi]"));
+ void testDoubleScan() {
+ List<Row> rows = Arrays.asList(Row.of(1, 1.0), Row.of(2, 2.0));
+ createSimpleBoundedValuesTable("floatTbl", "a int, b double", rows);
+ String query = "select a, b from floatTbl where a > 0";
+ runAndCheck(query, Arrays.asList("+I[1, 1.0]", "+I[2, 2.0]"));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]