dongjoon-hyun commented on code in PR #56334:
URL: https://github.com/apache/spark/pull/56334#discussion_r3562386421
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnStats.scala:
##########
@@ -382,8 +405,28 @@ private[columnar] final class ObjectColumnStats(dataType:
DataType) extends Colu
override def gatherStats(row: InternalRow, ordinal: Int): Unit = {
if (!row.isNullAt(ordinal)) {
- val size = columnType.actualSize(row, ordinal)
- sizeInBytes += size
+ // Check if this is a columnar complex type that doesn't support
getSizeInBytes
+ val isColumnarComplexType = columnType match {
+ case _: ARRAY =>
+ row.getArray(ordinal).isInstanceOf[ColumnarArray]
+ case _: MAP =>
+ row.getMap(ordinal).isInstanceOf[ColumnarMap]
+ case struct: STRUCT =>
+ row.getStruct(ordinal,
struct.dataType.fields.length).isInstanceOf[ColumnarRow]
+ case _ =>
+ false
+ }
+
+ if (!isColumnarComplexType) {
+ // Normal path: calculate size for unsafe types
+ // (UnsafeArrayData/UnsafeMapData/UnsafeRow)
+ val size = columnType.actualSize(row, ordinal)
+ sizeInBytes += size
+ }
+ // else: Skip size calculation for columnar complex types
+ // (ColumnarArray/ColumnarMap/ColumnarRow). These are views into
ColumnVectors
+ // and don't expose getSizeInBytes()
+
count += 1
Review Comment:
BTW, I didn't check the line width. So, please consider the logic only,
@viirya .
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
