hsiang-c commented on code in PR #4752:
URL: https://github.com/apache/datafusion-comet/pull/4752#discussion_r3554356850
##########
spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala:
##########
@@ -4310,32 +4310,235 @@ class CometIcebergNativeSuite
}
}
- test("CometScanRule should report unsupported metadata columns") {
+ test("metadata columns - _pos returns row position within each file") {
+ assume(icebergAvailable, "Iceberg not available in classpath")
+
withTempIcebergDir { warehouseDir =>
withSQLConf(
"spark.sql.catalog.test_cat" ->
"org.apache.iceberg.spark.SparkCatalog",
"spark.sql.catalog.test_cat.type" -> "hadoop",
"spark.sql.catalog.test_cat.warehouse" -> warehouseDir.getAbsolutePath,
CometConf.COMET_ENABLED.key -> "true",
CometConf.COMET_EXEC_ENABLED.key -> "true",
- CometConf.COMET_ICEBERG_NATIVE_ENABLED.key -> "true",
- CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "true") {
+ CometConf.COMET_ICEBERG_NATIVE_ENABLED.key -> "true") {
+
+ val table = "test_cat.db.pos_test"
+ try {
+ spark.sql(s"""
+ CREATE TABLE $table (id INT, name STRING) USING iceberg
+ """)
+
+ spark.sql(s"""
+ INSERT INTO $table VALUES (1, 'Alice'), (2, 'Bob'), (3, 'Charlie')
+ """)
+
+ checkIcebergNativeScan(s"SELECT id, _pos FROM $table ORDER BY id")
+
+ // _pos should be 0-indexed per file
+ val result = spark.sql(s"SELECT id, _pos FROM $table ORDER BY
id").collect()
+ assert(result(0).getLong(1) == 0L)
Review Comment:
Should be all zero according to my local test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
