malinjawi commented on code in PR #12198:
URL: https://github.com/apache/gluten/pull/12198#discussion_r3362293152
##########
backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala:
##########
@@ -179,6 +214,93 @@ class VeloxIteratorApi extends IteratorApi with Logging {
NativePlanEvaluator.injectWriteFilesTempPath(path, fileName)
}
+ private def normalizeDeltaSplitMetadata(
+ partitionColumnCount: Int,
+ partitionFiles: Seq[PartitionedFile]):
Option[NormalizedDeltaSplitMetadata] = {
+ try {
+ // scalastyle:off classforname
+ val moduleClass = Class.forName(deltaScanInfoClassName)
+ // scalastyle:on classforname
+ val module = moduleClass.getField("MODULE$").get(null)
+ val extractAllMethod = moduleClass.getMethod(
+ "extractAllFromJava",
+ classOf[SparkSession],
+ classOf[Int],
+ classOf[java.util.List[_]])
+ val scanInfos = extractAllMethod
+ .invoke(module, activeSparkSession, Int.box(partitionColumnCount),
partitionFiles.asJava)
+ .asInstanceOf[java.util.List[_]]
+ .asScala
+ .toSeq
+ val splitMetadata = scanInfos.map(toDeltaSplitMetadata)
+ if (splitMetadata.exists(_._2.hasDeletionVector())) {
+ Some((splitMetadata.map(_._1), splitMetadata.map(_._2)))
+ } else {
+ None
+ }
+ } catch {
+ case _: ClassNotFoundException | _: NoSuchMethodException =>
+ None
+ }
+ }
+
+ private def toDeltaSplitMetadata(
+ scanInfo: Any): (java.util.Map[String, Object], DeltaFileReadOptions) = {
+ val metadata = scanInfo
+ .getClass
+ .getMethod("normalizedOtherMetadataColumns")
+ .invoke(scanInfo)
+ .asInstanceOf[scala.collection.Map[String, Object]]
+ .asJava
+ val deletionVectorInfo =
scanInfo.getClass.getMethod("deletionVectorInfo").invoke(scanInfo)
+ val rowIndexFilterType = deletionVectorInfo
+ .getClass
+ .getMethod("rowIndexFilterType")
+ .invoke(deletionVectorInfo)
+ .toString
+ val hasDeletionVector = deletionVectorInfo
+ .getClass
+ .getMethod("hasDeletionVector")
+ .invoke(deletionVectorInfo)
+ .asInstanceOf[Boolean]
+ val cardinality = deletionVectorInfo
+ .getClass
+ .getMethod("cardinality")
+ .invoke(deletionVectorInfo)
+ .asInstanceOf[JLong]
+ .longValue()
+ val serializedDeletionVector = deletionVectorInfo
+ .getClass
+ .getMethod("serializedDeletionVector")
+ .invoke(deletionVectorInfo)
+ .asInstanceOf[Array[Byte]]
+
+ (
+ metadata,
+ new DeltaFileReadOptions(
+ toDeltaRowIndexFilterType(rowIndexFilterType),
+ hasDeletionVector,
+ cardinality,
+ serializedDeletionVector))
+ }
Review Comment:
@zhztheplayer I moved this reflection out of `VeloxIteratorApi` into
`DeltaSplitMetadataExtractor`.
Some reflection I think is still needed here because `VeloxIteratorApi` is
in the common Velox source set, while `DeltaDeletionVectorScanInfo` is only
compiled in Delta-specific source profiles. The extractor now isolates that
bridge and caches the class/method lookups, so the iterator path only calls the
utility and does not carry the reflection block inline.
Let me know if you know a workaround?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
