Github user viirya commented on a diff in the pull request:
https://github.com/apache/spark/pull/20935#discussion_r178425525
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnarTestUtils.scala
---
@@ -54,12 +54,22 @@ object ColumnarTestUtils {
case COMPACT_DECIMAL(precision, scale) => Decimal(Random.nextLong()
% 100, precision, scale)
case LARGE_DECIMAL(precision, scale) => Decimal(Random.nextLong(),
precision, scale)
case STRUCT(_) =>
- new
GenericInternalRow(Array[Any](UTF8String.fromString(Random.nextString(10))))
+ val schema = StructType(Array(StructField("test", StringType)))
+ val converter = UnsafeProjection.create(schema)
+
converter(InternalRow(Array(UTF8String.fromString(Random.nextString(10))): _*))
case ARRAY(_) =>
- new GenericArrayData(Array[Any](Random.nextInt(),
Random.nextInt()))
+ UnsafeArrayData.fromPrimitiveArray(Array(Random.nextInt(),
Random.nextInt()))
case MAP(_) =>
- ArrayBasedMapData(
- Map(Random.nextInt() ->
UTF8String.fromString(Random.nextString(Random.nextInt(32)))))
+ val unsafeConverter =
+ UnsafeProjection.create(Array[DataType](MapType(IntegerType,
StringType)))
+ val row = new GenericInternalRow(1)
+ def toUnsafeMap(map: ArrayBasedMapData): UnsafeMapData = {
+ row.update(0, map)
+ val unsafeRow = unsafeConverter.apply(row)
+ unsafeRow.getMap(0).copy
+ }
+ toUnsafeMap(ArrayBasedMapData(
+ Map(Random.nextInt() ->
UTF8String.fromString(Random.nextString(Random.nextInt(32))))))
--- End diff --
Seems above changes to data generation are unnecessary too?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
