Github user ueshin commented on a diff in the pull request:
https://github.com/apache/spark/pull/21912#discussion_r209854249
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayData.scala
---
@@ -34,6 +36,37 @@ object ArrayData {
case a: Array[Double] => UnsafeArrayData.fromPrimitiveArray(a)
case other => new GenericArrayData(other)
}
+
+
+ /**
+ * Allocate [[UnsafeArrayData]] or [[GenericArrayData]] based on given
parameters.
+ *
+ * @param elementSize a size of an element in bytes
+ * @param numElements the number of elements the array should contain
+ * @param isPrimitiveType whether the type of an element is primitive
type
+ * @param additionalErrorMessage string to include in the error message
+ */
+ def allocateArrayData(
+ elementSize: Int,
+ numElements : Long,
+ isPrimitiveType: Boolean,
+ additionalErrorMessage: String) : ArrayData = {
+ val arraySize =
UnsafeArrayData.calculateSizeOfUnderlyingByteArray(numElements, elementSize)
+ if (isPrimitiveType &&
!UnsafeArrayData.shouldUseGenericArrayData(elementSize, numElements)) {
+ val arrayBytes = new Array[Byte](arraySize.toInt)
+ val arrayName = new UnsafeArrayData()
+ Platform.putLong(arrayBytes, Platform.BYTE_ARRAY_OFFSET, numElements)
+ arrayName.pointTo(arrayBytes, Platform.BYTE_ARRAY_OFFSET,
arraySize.toInt)
+ arrayName
+ } else if (numElements <=
ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH.toLong) {
+ new GenericArrayData(new Array[Any](numElements.toInt))
+ } else {
+ throw new RuntimeException(s"Cannot create array with $numElements "
+
+ "elements of data due to exceeding the limit " +
+ s"${ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH} elements for
ArrayData." +
+ s"$additionalErrorMessage")
--- End diff --
nit: no need to surround with `s""`?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
