This is an automated email from the ASF dual-hosted git repository.

lgbo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 61ab633d21 [GLUTEN-9682][Flink] Support double in vector-to-row 
conversion (#9683)
61ab633d21 is described below

commit 61ab633d21fc752af48d03429b5a093a5d643ecc
Author: lgbo <[email protected]>
AuthorDate: Tue May 27 09:56:08 2025 +0800

    [GLUTEN-9682][Flink] Support double in vector-to-row conversion (#9683)
---
 .../apache/gluten/util/LogicalTypeConverter.java   |   3 +
 .../gluten/vectorized/ArrowVectorAccessor.java     | 142 +++++++++++++++++++++
 .../gluten/vectorized/ArrowVectorWriter.java       |  26 +++-
 .../vectorized/FlinkRowToVLVectorConvertor.java    |  37 +++---
 .../table/runtime/stream/custom/ScanTest.java      |  33 +++--
 5 files changed, 206 insertions(+), 35 deletions(-)

diff --git 
a/gluten-flink/runtime/src/main/java/org/apache/gluten/util/LogicalTypeConverter.java
 
b/gluten-flink/runtime/src/main/java/org/apache/gluten/util/LogicalTypeConverter.java
index e2a1ccc2e1..e342e5d94b 100644
--- 
a/gluten-flink/runtime/src/main/java/org/apache/gluten/util/LogicalTypeConverter.java
+++ 
b/gluten-flink/runtime/src/main/java/org/apache/gluten/util/LogicalTypeConverter.java
@@ -22,6 +22,7 @@ import org.apache.flink.table.types.logical.BigIntType;
 import org.apache.flink.table.types.logical.BooleanType;
 import org.apache.flink.table.types.logical.DayTimeIntervalType;
 import org.apache.flink.table.types.logical.DecimalType;
+import org.apache.flink.table.types.logical.DoubleType;
 import org.apache.flink.table.types.logical.IntType;
 import org.apache.flink.table.types.logical.LogicalType;
 import org.apache.flink.table.types.logical.RowType;
@@ -49,6 +50,8 @@ public class LogicalTypeConverter {
             return new IntegerType();
         } else if (logicalType instanceof BigIntType) {
             return new io.github.zhztheplayer.velox4j.type.BigIntType();
+        } else if (logicalType instanceof DoubleType) {
+            return new io.github.zhztheplayer.velox4j.type.DoubleType();
         } else if (logicalType instanceof VarCharType) {
             return new io.github.zhztheplayer.velox4j.type.VarCharType();
         } else if (logicalType instanceof TimestampType) {
diff --git 
a/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/ArrowVectorAccessor.java
 
b/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/ArrowVectorAccessor.java
new file mode 100644
index 0000000000..704d1ee08e
--- /dev/null
+++ 
b/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/ArrowVectorAccessor.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gluten.vectorized;
+
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.BitVector;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.Float8Vector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.complex.StructVector;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.binary.BinaryStringData;
+
+import io.github.zhztheplayer.velox4j.type.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/*
+ * This module is used to convert column vector to flink generic rows.
+ * BinaryRowData is not supported here.
+ */
+public abstract class ArrowVectorAccessor {
+    public static ArrowVectorAccessor create(FieldVector vector) {
+        if (vector instanceof BitVector) {
+            return new BooleanVectorAccessor(vector);
+        } else if (vector instanceof IntVector) {
+            return new IntVectorAccessor(vector);
+        } else if (vector instanceof BigIntVector) {
+            return new BigIntVectorAccessor(vector);
+        } else if (vector instanceof Float8Vector) {
+            return new DoubleVectorAccessor(vector);
+        } else if (vector instanceof VarCharVector) {
+            return new VarCharVectorAccessor(vector);
+        } else if (vector instanceof StructVector) {
+            return new StructVectorAccessor(vector);
+        } else {
+            throw new UnsupportedOperationException("ArrowVectorAccessor. 
Unsupported type: " + vector.getClass().getName());
+        }
+    }
+
+    // A general method to extract values from the vector.
+    public Object get(int rowIndex) {
+        throw new UnsupportedOperationException("get not supported");
+    }
+}
+
+class BooleanVectorAccessor extends ArrowVectorAccessor {
+    private final BitVector vector;
+    public BooleanVectorAccessor(FieldVector vector) {
+        this.vector = (BitVector) vector;
+    }
+    @Override
+    public Object get(int rowIndex) {
+        return vector.get(rowIndex) != 0;
+    }
+}
+
+class IntVectorAccessor extends ArrowVectorAccessor {
+    private final IntVector vector;
+    public IntVectorAccessor(FieldVector vector) {
+        this.vector = (IntVector) vector;
+    }
+    @Override
+    public Object get(int rowIndex) {
+        return vector.get(rowIndex);
+    }
+}
+
+class BigIntVectorAccessor extends ArrowVectorAccessor {
+    private final BigIntVector vector;
+    public BigIntVectorAccessor(FieldVector vector) {
+        this.vector = (BigIntVector) vector;
+    }
+    @Override
+    public Object get(int rowIndex) {
+        return vector.get(rowIndex);
+    }
+}
+
+class DoubleVectorAccessor extends ArrowVectorAccessor {
+    private final Float8Vector vector;
+    public DoubleVectorAccessor(FieldVector vector) {
+        this.vector = (Float8Vector) vector;
+    }
+    @Override
+    public Object get(int rowIndex) {
+        return vector.get(rowIndex);
+    }
+}
+
+class VarCharVectorAccessor extends ArrowVectorAccessor {
+    private final VarCharVector vector;
+    public VarCharVectorAccessor(FieldVector vector) {
+        this.vector = (VarCharVector) vector;
+    }
+    @Override
+    public Object get(int rowIndex) {
+        return BinaryStringData.fromBytes(vector.get(rowIndex));
+    }
+}
+
+class StructVectorAccessor extends ArrowVectorAccessor {
+    private final StructVector vector;
+    private final List<ArrowVectorAccessor> fieldAccessors;
+    private int fieldCount = 0;
+    public StructVectorAccessor(FieldVector vector) {
+        this.vector = (StructVector) vector;
+        // size() returns the number of child vectors
+        this.fieldCount = this.vector.size();
+        this.fieldAccessors = new ArrayList<>(fieldCount);
+        for (int i = 0; i < this.fieldCount; i++) {
+            FieldVector fieldVector = 
(FieldVector)this.vector.getChildByOrdinal(i);
+            this.fieldAccessors.add(i, 
ArrowVectorAccessor.create(fieldVector));
+        }
+    }
+    @Override
+    public Object get(int rowIndex) {
+        int fieldCount = vector.size();
+        Object[] fieldValues = new Object[fieldCount];
+        for (int i = 0; i < fieldCount; i++) {
+            fieldValues[i] = fieldAccessors.get(i).get(rowIndex);
+        }
+        return GenericRowData.of(fieldValues);
+    }
+}
diff --git 
a/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/ArrowVectorWriter.java
 
b/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/ArrowVectorWriter.java
index 9f3f41369b..35f6c78ee4 100644
--- 
a/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/ArrowVectorWriter.java
+++ 
b/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/ArrowVectorWriter.java
@@ -22,6 +22,7 @@ import io.github.zhztheplayer.velox4j.type.*;
 import org.apache.arrow.flatbuf.Int;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.complex.StructVector;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
 import org.apache.arrow.vector.types.TimeUnit;
 import org.apache.arrow.vector.types.pojo.*;
 import org.apache.arrow.vector.*;
@@ -49,6 +50,8 @@ public abstract class ArrowVectorWriter {
             return new BooleanVectorWriter(fieldType, allocator, vector);
         } else if (fieldType instanceof BigIntType) {
             return new BigIntVectorWriter(fieldType, allocator, vector);
+        } else if (fieldType instanceof DoubleType) {
+            return new Float8VectorWriter(fieldType, allocator, vector);
         } else if (fieldType instanceof VarCharType) {
             return new VarCharVectorWriter(fieldType, allocator, vector);
         } else if (fieldType instanceof TimestampType) {
@@ -56,7 +59,7 @@ public abstract class ArrowVectorWriter {
         } else if (fieldType instanceof RowType) {
             return new StructVectorWriter(fieldType, allocator, vector);
         } else {
-            throw new UnsupportedOperationException("Unsupported type: " + 
fieldType);
+            throw new UnsupportedOperationException("ArrowVectorWriter. 
Unsupported type: " + fieldType);
         }
     }
     public void write(int fieldIndex, RowData rowData) {
@@ -91,13 +94,15 @@ class FieldVectorCreator {
         return field.createVector(allocator);
     }
 
-    private static ArrowType toArroyType(Type dataType, String timeZoneId) {
+    private static ArrowType toArrowType(Type dataType, String timeZoneId) {
         if (dataType instanceof BooleanType) {
             return ArrowType.Bool.INSTANCE;
         } else if (dataType instanceof IntegerType) {
             return new ArrowType.Int(8 * 4, true);
         } else if (dataType instanceof BigIntType) {
             return new ArrowType.Int(8 * 8, true);
+        } else if (dataType instanceof DoubleType) {
+            return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
         } else if (dataType instanceof VarCharType) {
             return ArrowType.Utf8.INSTANCE;
         } else if (dataType instanceof TimestampType) {
@@ -132,7 +137,7 @@ class FieldVectorCreator {
             return new Field(name, strcutType, subFields);
         } else {
             // TODO: support nullable
-            ArrowType arrowType = toArroyType(dataType, timeZoneId);
+            ArrowType arrowType = toArrowType(dataType, timeZoneId);
             FieldType fieldType = new FieldType(nullable, arrowType, null);
             return new Field(name, fieldType, new ArrayList<>());
         }
@@ -185,6 +190,21 @@ class BigIntVectorWriter extends ArrowVectorWriter {
     }
 }
 
+class Float8VectorWriter extends ArrowVectorWriter {
+    private final Float8Vector float8Vector;
+
+    public Float8VectorWriter(Type fieldType, BufferAllocator allocator, 
FieldVector vector) {
+        super(vector);
+        this.float8Vector = (Float8Vector) vector;
+    }
+
+    @Override
+    public void write(int fieldIndex, RowData rowData) {
+        float8Vector.setSafe(valueCount, rowData.getDouble(fieldIndex));
+        valueCount++;
+    }
+}
+
 class VarCharVectorWriter extends ArrowVectorWriter {
     private final VarCharVector varCharVector;
 
diff --git 
a/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/FlinkRowToVLVectorConvertor.java
 
b/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/FlinkRowToVLVectorConvertor.java
index 631681eda0..1c15d6fd90 100644
--- 
a/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/FlinkRowToVLVectorConvertor.java
+++ 
b/gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/FlinkRowToVLVectorConvertor.java
@@ -22,6 +22,7 @@ import io.github.zhztheplayer.velox4j.data.RowVector;
 import io.github.zhztheplayer.velox4j.session.Session;
 import io.github.zhztheplayer.velox4j.type.BigIntType;
 import io.github.zhztheplayer.velox4j.type.BooleanType;
+import io.github.zhztheplayer.velox4j.type.DoubleType;
 import io.github.zhztheplayer.velox4j.type.IntegerType;
 import io.github.zhztheplayer.velox4j.type.RowType;
 import io.github.zhztheplayer.velox4j.type.TimestampType;
@@ -32,12 +33,14 @@ import org.apache.arrow.vector.BigIntVector;
 import org.apache.arrow.vector.BitVector;
 import org.apache.arrow.vector.IntVector;
 import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.Float8Vector;
 import org.apache.arrow.vector.TimeStampMilliVector;
 import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.complex.StructVector;
 import org.apache.arrow.vector.table.Table;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.gluten.vectorized.ArrowVectorAccessor;
 import org.apache.gluten.vectorized.ArrowVectorWriter;
 import org.apache.flink.table.data.GenericRowData;
 import org.apache.flink.table.data.RowData;
@@ -81,35 +84,20 @@ public class FlinkRowToVLVectorConvertor {
         // TODO: support more types
         BaseVector loadedVector = null;
         FieldVector structVector = null;
+
         try{
             loadedVector = rowVector.loadedVector();
             // The result is StructVector
-            structVector = Arrow.toArrowVector(
-                allocator,
-                loadedVector);
+            structVector = Arrow.toArrowVector(allocator,loadedVector);
             final List<FieldVector> fieldVectors = 
structVector.getChildrenFromFields();
+            List<ArrowVectorAccessor> accessors = 
buildArrowVectorAccessors(fieldVectors);
             List<RowData> rowDatas = new ArrayList<>(rowVector.getSize());
             for (int j = 0; j < rowVector.getSize(); j++) {
-                List<Object> fieldValues = new ArrayList<>(rowType.size());
+                Object[] fieldValues = new Object[rowType.size()];
                 for (int i = 0; i < rowType.size(); i++) {
-                    Type fieldType = rowType.getChildren().get(i);
-                    if (fieldType instanceof BooleanType) {
-                        // BitVector returns are integer, need to convert to 
boolean
-                        fieldValues.add(i, ((BitVector) 
fieldVectors.get(i)).get(j) != 0);
-                    } else if (fieldType instanceof IntegerType) {
-                        fieldValues.add(i, ((IntVector) 
fieldVectors.get(i)).get(j));
-                    } else if (fieldType instanceof BigIntType) {
-                        fieldValues.add(i, ((BigIntVector) 
fieldVectors.get(i)).get(j));
-                    } else if (fieldType instanceof VarCharType) {
-                        fieldValues.add(
-                            i,
-                            BinaryStringData.fromBytes(
-                                    ((VarCharVector) 
fieldVectors.get(i)).get(j)));
-                    } else {
-                        throw new RuntimeException("Unsupported field type: " 
+ fieldType);
-                    }
+                    fieldValues[i] = accessors.get(i).get(j);
                 }
-                rowDatas.add(GenericRowData.of(fieldValues.toArray()));
+                rowDatas.add(GenericRowData.of(fieldValues));
             }
             return rowDatas;
         } finally {
@@ -124,4 +112,11 @@ public class FlinkRowToVLVectorConvertor {
         }
     }
 
+    private static List<ArrowVectorAccessor> 
buildArrowVectorAccessors(List<FieldVector> vectors) {
+        List<ArrowVectorAccessor> accessors = new ArrayList<>(vectors.size());
+        for (int i = 0; i < vectors.size(); ++i) {
+            accessors.add(i, ArrowVectorAccessor.create(vectors.get(i)));
+        }
+        return accessors;
+    }
 }
diff --git 
a/gluten-flink/ut/src/test/java/org/apache/gluten/table/runtime/stream/custom/ScanTest.java
 
b/gluten-flink/ut/src/test/java/org/apache/gluten/table/runtime/stream/custom/ScanTest.java
index b0dd716e77..9f083b16cd 100644
--- 
a/gluten-flink/ut/src/test/java/org/apache/gluten/table/runtime/stream/custom/ScanTest.java
+++ 
b/gluten-flink/ut/src/test/java/org/apache/gluten/table/runtime/stream/custom/ScanTest.java
@@ -38,30 +38,41 @@ class ScanTest extends GlutenStreamingTestBase {
     @BeforeEach
     public void before() throws Exception {
         super.before();
+
+    }
+
+    @Test
+    void testFilter() {
         List<Row> rows =
                 Arrays.asList(Row.of(1, 1L, "1"),
                     Row.of(2, 2L, "2"),
                     Row.of(3, 3L, "3"));
         createSimpleBoundedValuesTable("MyTable", "a int, b bigint, c string", 
rows);
+        String query = "select a, b as b,c, a > 2 from MyTable where a > 0";
+        LOG.info("execution plan: {}", explainExecutionPlan(query));
+        runAndCheck(query, Arrays.asList("+I[1, 1, 1, false]", "+I[2, 2, 2, 
false]", "+I[3, 3, 3, true]"));
+    }
 
-        List<Row> structRows =
+    @Test
+    void testStructScan() {
+        List<Row> rows =
                 Arrays.asList(Row.of(1, Row.of(2L, "abc")),
                     Row.of(2,  Row.of(6L, "def")),
                     Row.of(3,  Row.of(8L, "ghi")));
-        createSimpleBoundedValuesTable("MyTable2", "a int, b ROW<d bigint, e 
string>", structRows);
-    }
+        createSimpleBoundedValuesTable("strctTbl", "a int, b ROW<x bigint, y 
string>", rows);
+        String query1 = "select a, b.x, b.y from strctTbl where a > 0";
+        runAndCheck(query1, Arrays.asList("+I[1, 2, abc]", "+I[2, 6, def]", 
"+I[3, 8, ghi]"));
 
-    @Test
-    void testFilter() {
-        String query = "select a, b as b,c, a > 2 from MyTable where a > 0";
-        LOG.info("execution plan: {}", explainExecutionPlan(query));
-        runAndCheck(query, Arrays.asList("+I[1, 1, 1, false]", "+I[2, 2, 2, 
false]", "+I[3, 3, 3, true]"));
+        String query2 = "select a, b from strctTbl where a > 1";
+        runAndCheck(query2, Arrays.asList("+I[2, +I[6, def]]", "+I[3, +I[8, 
ghi]]"));
     }
 
     @Test
-    void testFilterWithStruct() {
-        String query = "select a, b.d, b.e from MyTable2 where a > 0";
-        runAndCheck(query, Arrays.asList("+I[1, 2, abc]", "+I[2, 6, def]", 
"+I[3, 8, ghi]"));
+    void testDoubleScan() {
+        List<Row> rows = Arrays.asList(Row.of(1, 1.0), Row.of(2, 2.0));
+        createSimpleBoundedValuesTable("floatTbl", "a int, b double", rows);
+        String query = "select a, b from floatTbl where a > 0";
+        runAndCheck(query, Arrays.asList("+I[1, 1.0]", "+I[2, 2.0]"));
     }
 }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to