This is an automated email from the ASF dual-hosted git repository.
zhehu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/main by this push:
new 0c9d879e1a [CALCITE-6760] Support time data type in Arrow adapter
0c9d879e1a is described below
commit 0c9d879e1ad8f1555f6a9f76e1d1e0c030d354ba
Author: Cancai Cai <[email protected]>
AuthorDate: Thu Jan 2 22:46:25 2025 +0800
[CALCITE-6760] Support time data type in Arrow adapter
---
.../adapter/arrow/ArrowFieldTypeFactory.java | 2 ++
.../adapter/arrow/ArrowAdapterDataTypesTest.java | 35 +++++++++++++++-------
.../calcite/adapter/arrow/ArrowAdapterTest.java | 18 +++++------
.../calcite/adapter/arrow/ArrowDataTest.java | 19 ++++++++++++
4 files changed, 55 insertions(+), 19 deletions(-)
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFieldTypeFactory.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFieldTypeFactory.java
index 30af9736d3..ad993c813d 100644
---
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFieldTypeFactory.java
+++
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFieldTypeFactory.java
@@ -80,6 +80,8 @@ public class ArrowFieldTypeFactory {
return typeFactory.createSqlType(SqlTypeName.DECIMAL,
((ArrowType.Decimal) arrowType).getPrecision(),
((ArrowType.Decimal) arrowType).getScale());
+ case Time:
+ return typeFactory.createSqlType(SqlTypeName.TIME);
default:
throw new IllegalArgumentException("Unsupported type: " + arrowType);
}
diff --git
a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterDataTypesTest.java
b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterDataTypesTest.java
index 5d3414532d..46e4404937 100644
---
a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterDataTypesTest.java
+++
b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterDataTypesTest.java
@@ -68,7 +68,7 @@ public class ArrowAdapterDataTypesTest {
String sql = "select \"tinyIntField\" from arrowdatatype";
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(tinyIntField=[$0])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result = "tinyIntField=0\ntinyIntField=1\n";
CalciteAssert.that()
.with(arrow)
@@ -82,7 +82,7 @@ public class ArrowAdapterDataTypesTest {
String sql = "select \"smallIntField\" from arrowdatatype";
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(smallIntField=[$1])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result = "smallIntField=0\nsmallIntField=1\n";
CalciteAssert.that()
.with(arrow)
@@ -96,7 +96,7 @@ public class ArrowAdapterDataTypesTest {
String sql = "select \"intField\" from arrowdatatype";
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(intField=[$2])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result = "intField=0\nintField=1\n";
CalciteAssert.that()
.with(arrow)
@@ -110,7 +110,7 @@ public class ArrowAdapterDataTypesTest {
String sql = "select \"longField\" from arrowdatatype";
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(longField=[$5])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result = "longField=0\nlongField=1\n";
CalciteAssert.that()
.with(arrow)
@@ -124,7 +124,7 @@ public class ArrowAdapterDataTypesTest {
String sql = "select \"floatField\" from arrowdatatype";
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(floatField=[$4])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result = "floatField=0.0\nfloatField=1.0\n";
CalciteAssert.that()
.with(arrow)
@@ -138,7 +138,7 @@ public class ArrowAdapterDataTypesTest {
String sql = "select \"doubleField\" from arrowdatatype";
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(doubleField=[$6])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result = "doubleField=0.0\ndoubleField=1.0\n";
CalciteAssert.that()
.with(arrow)
@@ -152,7 +152,7 @@ public class ArrowAdapterDataTypesTest {
String sql = "select \"decimalField\" from arrowdatatype";
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(decimalField=[$8])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result = "decimalField=0.00\ndecimalField=1.00\n";
CalciteAssert.that()
.with(arrow)
@@ -166,7 +166,7 @@ public class ArrowAdapterDataTypesTest {
String sql = "select \"dateField\" from arrowdatatype";
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(dateField=[$9])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result = "dateField=1970-01-01\n"
+ "dateField=1970-01-02\n";
CalciteAssert.that()
@@ -181,7 +181,7 @@ public class ArrowAdapterDataTypesTest {
String sql = "select \"booleanField\" from arrowdatatype";
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(booleanField=[$7])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result =
"booleanField=null\nbooleanField=true\nbooleanField=false\n";
CalciteAssert.that()
.with(arrow)
@@ -198,7 +198,7 @@ public class ArrowAdapterDataTypesTest {
String sql = "select \"decimalField2\" from arrowdatatype";
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(decimalField2=[$10])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result = "decimalField2=20.000\ndecimalField2=21.000\n";
CalciteAssert.that()
.with(arrow)
@@ -207,4 +207,19 @@ public class ArrowAdapterDataTypesTest {
.returns(result)
.explainContains(plan);
}
+
+ @Test void testTimeProject() {
+ String sql = "select \"timeField\" from arrowdatatype";
+ String plan = "PLAN=ArrowToEnumerableConverter\n"
+ + " ArrowProject(timeField=[$11])\n"
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
+ String result = "timeField=00:00:00\n"
+ + "timeField=00:00:01\n";
+ CalciteAssert.that()
+ .with(arrow)
+ .query(sql)
+ .limit(2)
+ .returns(result)
+ .explainContains(plan);
+ }
}
diff --git
a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java
b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java
index 2858208450..6027e24488 100644
--- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java
+++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java
@@ -860,7 +860,7 @@ class ArrowAdapterTest {
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(booleanField=[$7])\n"
+ " ArrowFilter(condition=[$7])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result = "booleanField=true\nbooleanField=true\n";
CalciteAssert.that()
@@ -878,7 +878,7 @@ class ArrowAdapterTest {
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(intField=[$2])\n"
+ " ArrowFilter(condition=[>($2, 10)])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result = "intField=11\nintField=12\n";
CalciteAssert.that()
@@ -896,7 +896,7 @@ class ArrowAdapterTest {
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(booleanField=[$7])\n"
+ " ArrowFilter(condition=[NOT($7)])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result = "booleanField=false\nbooleanField=false\n";
CalciteAssert.that()
@@ -915,7 +915,7 @@ class ArrowAdapterTest {
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(booleanField=[$7])\n"
+ " ArrowFilter(condition=[IS NOT TRUE($7)])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result = "booleanField=null\nbooleanField=false\n";
CalciteAssert.that()
@@ -933,7 +933,7 @@ class ArrowAdapterTest {
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(booleanField=[$7])\n"
+ " ArrowFilter(condition=[IS NOT FALSE($7)])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result = "booleanField=null\nbooleanField=true\n";
CalciteAssert.that()
@@ -951,7 +951,7 @@ class ArrowAdapterTest {
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(booleanField=[$7])\n"
+ " ArrowFilter(condition=[IS NULL($7)])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result = "booleanField=null\n";
CalciteAssert.that()
@@ -972,7 +972,7 @@ class ArrowAdapterTest {
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(decimalField=[$8])\n"
+ " ArrowFilter(condition=[=($8, 1.00)])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result = "decimalField=1.00\n";
CalciteAssert.that()
@@ -989,7 +989,7 @@ class ArrowAdapterTest {
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(doubleField=[$6])\n"
+ " ArrowFilter(condition=[=($6, 1.0E0)])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result = "doubleField=1.0\n";
CalciteAssert.that()
@@ -1006,7 +1006,7 @@ class ArrowAdapterTest {
String plan = "PLAN=ArrowToEnumerableConverter\n"
+ " ArrowProject(stringField=[$3])\n"
+ " ArrowFilter(condition=[=($3, '1')])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10]])\n\n";
+ + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1,
2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n";
String result = "stringField=1\n";
CalciteAssert.that()
diff --git
a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java
b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java
index 9557c69af2..0d92da040e 100644
--- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java
+++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java
@@ -31,12 +31,14 @@ import org.apache.arrow.vector.Float8Vector;
import org.apache.arrow.vector.FloatingPointVector;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.SmallIntVector;
+import org.apache.arrow.vector.TimeSecVector;
import org.apache.arrow.vector.TinyIntVector;
import org.apache.arrow.vector.VarCharVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowFileWriter;
import org.apache.arrow.vector.types.DateUnit;
import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.apache.arrow.vector.types.TimeUnit;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
@@ -76,6 +78,7 @@ public class ArrowDataTest {
private double doubleValue;
private boolean booleanValue;
private BigDecimal decimalValue;
+ private int timeValue;
public ArrowDataTest() {
this.batchSize = 20;
@@ -89,6 +92,7 @@ public class ArrowDataTest {
this.doubleValue = 0;
this.booleanValue = false;
this.decimalValue = BigDecimal.ZERO;
+ this.timeValue = 0;
}
private Schema makeArrowDateTypeSchema() {
@@ -106,6 +110,7 @@ public class ArrowDataTest {
FieldType decimalType = FieldType.nullable(new ArrowType.Decimal(12, 2,
128));
FieldType decimalType2 = FieldType.nullable(new ArrowType.Decimal(12, 3,
128));
FieldType dateType = FieldType.nullable(new ArrowType.Date(DateUnit.DAY));
+ FieldType timeType = FieldType.nullable(new
ArrowType.Time(TimeUnit.SECOND, 32));
childrenBuilder.add(new Field("tinyIntField", tinyIntType, null));
childrenBuilder.add(new Field("smallIntField", smallIntType, null));
@@ -118,6 +123,7 @@ public class ArrowDataTest {
childrenBuilder.add(new Field("decimalField", decimalType, null));
childrenBuilder.add(new Field("dateField", dateType, null));
childrenBuilder.add(new Field("decimalField2", decimalType2, null));
+ childrenBuilder.add(new Field("timeField", timeType, null));
return new Schema(childrenBuilder.build(), null);
}
@@ -273,6 +279,9 @@ public class ArrowDataTest {
case "dateField":
dateField(vector, numRows);
break;
+ case "timeField":
+ timeField(vector, numRows);
+ break;
default:
throw new IllegalStateException("Not supported type yet: " +
vector.getMinorType());
}
@@ -411,4 +420,14 @@ public class ArrowDataTest {
}
fieldVector.setValueCount(rowCount);
}
+
+ private void timeField(FieldVector fieldVector, int rowCount) {
+ TimeSecVector timeVector = (TimeSecVector) fieldVector;
+ timeVector.setInitialCapacity(rowCount);
+ timeVector.allocateNew();
+ for (int i = 0; i < rowCount; i++) {
+ timeVector.set(i, i * 1000);
+ }
+ fieldVector.setValueCount(rowCount);
+ }
}
