This is an automated email from the ASF dual-hosted git repository.
hvanhovell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new db2625c70a8 [SPARK-43168][SQL] Remove get PhysicalDataType method from
Datatype class
db2625c70a8 is described below
commit db2625c70a8c3aff64e6a9466981c8dd49a4ca51
Author: Rui Wang <rui.w...@databricks.com>
AuthorDate: Mon Apr 17 22:16:25 2023 -0400
[SPARK-43168][SQL] Remove get PhysicalDataType method from Datatype class
### What changes were proposed in this pull request?
DataType is public API while we can leave PhysicalDataType as internal
API/implementation thus we can remove PhysicalDataType from DataType. So
DataType does not need to have a class dependency on PhysicalDataType.
### Why are the changes needed?
Simplify DataType.
### Does this PR introduce _any_ user-facing change?
NO
### How was this patch tested?
UT
Closes #40826 from amaliujia/catalyst_datatype_refactor_8.
Authored-by: Rui Wang <rui.w...@databricks.com>
Signed-off-by: Herman van Hovell <her...@databricks.com>
---
.../spark/sql/catalyst/expressions/SpecializedGettersReader.java | 2 +-
.../java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java | 4 ++--
.../main/java/org/apache/spark/sql/vectorized/ColumnarBatchRow.java | 2 +-
.../src/main/java/org/apache/spark/sql/vectorized/ColumnarRow.java | 2 +-
.../src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala | 2 +-
.../spark/sql/catalyst/expressions/InterpretedUnsafeProjection.scala | 2 +-
.../spark/sql/catalyst/expressions/codegen/CodeGenerator.scala | 4 ++--
.../scala/org/apache/spark/sql/catalyst/expressions/literals.scala | 2 +-
.../org/apache/spark/sql/catalyst/expressions/namedExpressions.scala | 2 +-
.../scala/org/apache/spark/sql/catalyst/types/PhysicalDataType.scala | 5 ++++-
.../src/main/scala/org/apache/spark/sql/types/ArrayType.scala | 4 ----
.../src/main/scala/org/apache/spark/sql/types/BinaryType.scala | 3 ---
.../src/main/scala/org/apache/spark/sql/types/BooleanType.scala | 3 ---
.../src/main/scala/org/apache/spark/sql/types/ByteType.scala | 3 ---
.../main/scala/org/apache/spark/sql/types/CalendarIntervalType.scala | 3 ---
.../src/main/scala/org/apache/spark/sql/types/CharType.scala | 2 --
.../src/main/scala/org/apache/spark/sql/types/DataType.scala | 4 +---
.../src/main/scala/org/apache/spark/sql/types/DateType.scala | 3 ---
.../main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala | 3 ---
.../src/main/scala/org/apache/spark/sql/types/DecimalType.scala | 4 ----
.../src/main/scala/org/apache/spark/sql/types/DoubleType.scala | 3 ---
.../src/main/scala/org/apache/spark/sql/types/FloatType.scala | 3 ---
.../src/main/scala/org/apache/spark/sql/types/IntegerType.scala | 3 ---
.../src/main/scala/org/apache/spark/sql/types/LongType.scala | 3 ---
sql/catalyst/src/main/scala/org/apache/spark/sql/types/MapType.scala | 4 ----
.../src/main/scala/org/apache/spark/sql/types/NullType.scala | 3 ---
.../src/main/scala/org/apache/spark/sql/types/ShortType.scala | 3 ---
.../src/main/scala/org/apache/spark/sql/types/StringType.scala | 3 ---
.../src/main/scala/org/apache/spark/sql/types/StructType.scala | 4 +---
.../src/main/scala/org/apache/spark/sql/types/TimestampNTZType.scala | 3 ---
.../src/main/scala/org/apache/spark/sql/types/TimestampType.scala | 3 ---
.../src/main/scala/org/apache/spark/sql/types/VarcharType.scala | 3 ---
.../scala/org/apache/spark/sql/types/YearMonthIntervalType.scala | 3 ---
.../org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java | 2 +-
34 files changed, 18 insertions(+), 84 deletions(-)
diff --git
a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java
b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java
index c5a7d34281f..be50350b106 100644
---
a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java
+++
b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java
@@ -29,7 +29,7 @@ public final class SpecializedGettersReader {
DataType dataType,
boolean handleNull,
boolean handleUserDefinedType) {
- PhysicalDataType physicalDataType = dataType.physicalDataType();
+ PhysicalDataType physicalDataType = PhysicalDataType.apply(dataType);
if (handleNull && (obj.isNullAt(ordinal) || physicalDataType instanceof
PhysicalNullType)) {
return null;
}
diff --git
a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
index a3097cd6770..d2433292fc7 100644
---
a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
+++
b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
@@ -73,7 +73,7 @@ public final class UnsafeRow extends InternalRow implements
Externalizable, Kryo
if (dt instanceof UserDefinedType) {
return isFixedLength(((UserDefinedType<?>) dt).sqlType());
}
- PhysicalDataType pdt = dt.physicalDataType();
+ PhysicalDataType pdt = PhysicalDataType.apply(dt);
if (pdt instanceof PhysicalDecimalType) {
return ((DecimalType) dt).precision() <= Decimal.MAX_LONG_DIGITS();
} else {
@@ -88,7 +88,7 @@ public final class UnsafeRow extends InternalRow implements
Externalizable, Kryo
if (dt instanceof UserDefinedType) {
return isMutable(((UserDefinedType<?>) dt).sqlType());
}
- PhysicalDataType pdt = dt.physicalDataType();
+ PhysicalDataType pdt = PhysicalDataType.apply(dt);
return pdt instanceof PhysicalPrimitiveType || pdt instanceof
PhysicalDecimalType ||
pdt instanceof PhysicalCalendarIntervalType;
}
diff --git
a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarBatchRow.java
b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarBatchRow.java
index 05922a0cd5d..d30155eb2b4 100644
---
a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarBatchRow.java
+++
b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarBatchRow.java
@@ -49,7 +49,7 @@ public final class ColumnarBatchRow extends InternalRow {
row.setNullAt(i);
} else {
DataType dt = columns[i].dataType();
- PhysicalDataType pdt = dt.physicalDataType();
+ PhysicalDataType pdt = PhysicalDataType.apply(dt);
if (pdt instanceof PhysicalBooleanType) {
row.setBoolean(i, getBoolean(i));
} else if (pdt instanceof PhysicalByteType) {
diff --git
a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarRow.java
b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarRow.java
index 9c2b1833348..c4fbc2ff642 100644
---
a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarRow.java
+++
b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarRow.java
@@ -56,7 +56,7 @@ public final class ColumnarRow extends InternalRow {
row.setNullAt(i);
} else {
DataType dt = data.getChild(i).dataType();
- PhysicalDataType pdt = dt.physicalDataType();
+ PhysicalDataType pdt = PhysicalDataType.apply(dt);
if (pdt instanceof PhysicalBooleanType) {
row.setBoolean(i, getBoolean(i));
} else if (pdt instanceof PhysicalByteType) {
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala
index a44dca7dda9..02b42f11cc0 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala
@@ -131,7 +131,7 @@ object InternalRow {
def getAccessor(dt: DataType, nullable: Boolean = true):
(SpecializedGetters, Int) => Any = {
val getValueNullSafe: (SpecializedGetters, Int) => Any = dt match {
case u: UserDefinedType[_] => getAccessor(u.sqlType, nullable)
- case _ => dt.physicalDataType match {
+ case _ => PhysicalDataType(dt) match {
case PhysicalBooleanType => (input, ordinal) =>
input.getBoolean(ordinal)
case PhysicalByteType => (input, ordinal) => input.getByte(ordinal)
case PhysicalShortType => (input, ordinal) => input.getShort(ordinal)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedUnsafeProjection.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedUnsafeProjection.scala
index 90a90444695..a8f21a18ca7 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedUnsafeProjection.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedUnsafeProjection.scala
@@ -136,7 +136,7 @@ object InterpretedUnsafeProjection {
// Create the basic writer.
val unsafeWriter: (SpecializedGetters, Int) => Unit = dt match {
case udt: UserDefinedType[_] => generateFieldWriter(writer, udt.sqlType,
nullable)
- case _ => dt.physicalDataType match {
+ case _ => PhysicalDataType(dt) match {
case PhysicalBooleanType => (v, i) => writer.write(i, v.getBoolean(i))
case PhysicalByteType => (v, i) => writer.write(i, v.getByte(i))
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
index 12103ceef6e..5651a30515f 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
@@ -1628,7 +1628,7 @@ object CodeGenerator extends Logging {
dataType match {
case udt: UserDefinedType[_] => getValue(input, udt.sqlType, ordinal)
case _ if isPrimitiveType(jt) =>
s"$input.get${primitiveTypeName(jt)}($ordinal)"
- case _ => dataType.physicalDataType match {
+ case _ => PhysicalDataType(dataType) match {
case _: PhysicalArrayType => s"$input.getArray($ordinal)"
case PhysicalBinaryType => s"$input.getBinary($ordinal)"
case PhysicalCalendarIntervalType => s"$input.getInterval($ordinal)"
@@ -1909,7 +1909,7 @@ object CodeGenerator extends Logging {
case udt: UserDefinedType[_] => javaType(udt.sqlType)
case ObjectType(cls) if cls.isArray =>
s"${javaType(ObjectType(cls.getComponentType))}[]"
case ObjectType(cls) => cls.getName
- case _ => dt.physicalDataType match {
+ case _ => PhysicalDataType(dt) match {
case _: PhysicalArrayType => "ArrayData"
case PhysicalBinaryType => "byte[]"
case PhysicalBooleanType => JAVA_BOOLEAN
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
index e8ac858eb11..dd41cefb724 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
@@ -208,7 +208,7 @@ object Literal {
case _ if v == null => true
case ObjectType(cls) => cls.isInstance(v)
case udt: UserDefinedType[_] => doValidate(v, udt.sqlType)
- case dt => dataType.physicalDataType match {
+ case dt => PhysicalDataType(dataType) match {
case PhysicalArrayType(et, _) =>
v.isInstanceOf[ArrayData] && {
val ar = v.asInstanceOf[ArrayData]
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
index 048f379f8a3..4f4fbc07b16 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
@@ -563,7 +563,7 @@ object FileSourceMetadataAttribute {
* supported (including most primitive types). Notably unsupported types
include [[ObjectType]],
* [[UserDefinedType]], and the complex types ([[StructType]], [[MapType]],
[[ArrayType]]).
*/
- def isSupportedType(dataType: DataType): Boolean = dataType.physicalDataType
match {
+ def isSupportedType(dataType: DataType): Boolean =
PhysicalDataType(dataType) match {
// PhysicalPrimitiveType covers: Boolean, Byte, Double, Float, Integer,
Long, Null, Short
case _: PhysicalPrimitiveType | _: PhysicalDecimalType => true
case PhysicalBinaryType | PhysicalStringType |
PhysicalCalendarIntervalType => true
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/PhysicalDataType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/PhysicalDataType.scala
index b6e0cd88f08..d392557e650 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/PhysicalDataType.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/PhysicalDataType.scala
@@ -23,7 +23,7 @@ import scala.reflect.runtime.universe.typeTag
import org.apache.spark.sql.catalyst.expressions.{Ascending, BoundReference,
InterpretedOrdering, SortOrder}
import org.apache.spark.sql.catalyst.util.{ArrayData, SQLOrderingUtil}
import org.apache.spark.sql.errors.QueryExecutionErrors
-import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType,
ByteExactNumeric, ByteType, DataType, DateType, DayTimeIntervalType, Decimal,
DecimalExactNumeric, DecimalType, DoubleExactNumeric, DoubleType,
FloatExactNumeric, FloatType, FractionalType, IntegerExactNumeric, IntegerType,
IntegralType, LongExactNumeric, LongType, MapType, NullType, NumericType,
ShortExactNumeric, ShortType, StringType, StructField, StructType,
TimestampNTZType, TimestampType, YearMonthIntervalType}
+import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType,
ByteExactNumeric, ByteType, CalendarIntervalType, CharType, DataType, DateType,
DayTimeIntervalType, Decimal, DecimalExactNumeric, DecimalType,
DoubleExactNumeric, DoubleType, FloatExactNumeric, FloatType, FractionalType,
IntegerExactNumeric, IntegerType, IntegralType, LongExactNumeric, LongType,
MapType, NullType, NumericType, ShortExactNumeric, ShortType, StringType,
StructField, StructType, TimestampNTZType, Timest [...]
import org.apache.spark.unsafe.types.{ByteArray, UTF8String}
sealed abstract class PhysicalDataType {
@@ -39,6 +39,8 @@ object PhysicalDataType {
case ShortType => PhysicalShortType
case IntegerType => PhysicalIntegerType
case LongType => PhysicalLongType
+ case VarcharType(_) => PhysicalStringType
+ case CharType(_) => PhysicalStringType
case StringType => PhysicalStringType
case FloatType => PhysicalFloatType
case DoubleType => PhysicalDoubleType
@@ -47,6 +49,7 @@ object PhysicalDataType {
case BinaryType => PhysicalBinaryType
case TimestampType => PhysicalLongType
case TimestampNTZType => PhysicalLongType
+ case CalendarIntervalType => PhysicalCalendarIntervalType
case DayTimeIntervalType(_, _) => PhysicalLongType
case YearMonthIntervalType(_, _) => PhysicalIntegerType
case DateType => PhysicalIntegerType
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala
index 553ac4e15a5..d6f5c3bdf43 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql.types
import org.json4s.JsonDSL._
import org.apache.spark.annotation.Stable
-import org.apache.spark.sql.catalyst.types.{PhysicalArrayType,
PhysicalDataType}
import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat
/**
@@ -88,9 +87,6 @@ case class ArrayType(elementType: DataType, containsNull:
Boolean) extends DataT
*/
override def defaultSize: Int = 1 * elementType.defaultSize
- private[sql] override def physicalDataType: PhysicalDataType =
- PhysicalArrayType(elementType, containsNull)
-
override def simpleString: String = s"array<${elementType.simpleString}>"
override def catalogString: String = s"array<${elementType.catalogString}>"
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BinaryType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BinaryType.scala
index 25f29e12dce..c280f66f943 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BinaryType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BinaryType.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.types
import org.apache.spark.annotation.Stable
-import org.apache.spark.sql.catalyst.types.{PhysicalBinaryType,
PhysicalDataType}
/**
* The data type representing `Array[Byte]` values.
@@ -31,8 +30,6 @@ class BinaryType private() extends AtomicType {
*/
override def defaultSize: Int = 100
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalBinaryType
-
private[spark] override def asNullable: BinaryType = this
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BooleanType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BooleanType.scala
index ddcc2fa75bd..836c41a996a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BooleanType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BooleanType.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.types
import org.apache.spark.annotation.Stable
-import org.apache.spark.sql.catalyst.types.{PhysicalBooleanType,
PhysicalDataType}
/**
* The data type representing `Boolean` values. Please use the singleton
`DataTypes.BooleanType`.
@@ -32,8 +31,6 @@ class BooleanType private() extends AtomicType {
*/
override def defaultSize: Int = 1
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalBooleanType
-
private[spark] override def asNullable: BooleanType = this
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ByteType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ByteType.scala
index d525a9072ed..546ac02f263 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ByteType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ByteType.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.types
import org.apache.spark.annotation.Stable
-import org.apache.spark.sql.catalyst.types.{PhysicalByteType, PhysicalDataType}
/**
* The data type representing `Byte` values. Please use the singleton
`DataTypes.ByteType`.
@@ -32,8 +31,6 @@ class ByteType private() extends IntegralType {
*/
override def defaultSize: Int = 1
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalByteType
-
override def simpleString: String = "tinyint"
private[spark] override def asNullable: ByteType = this
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CalendarIntervalType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CalendarIntervalType.scala
index 7f1c51ef23d..d506a1521e1 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CalendarIntervalType.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CalendarIntervalType.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.types
import org.apache.spark.annotation.Stable
-import org.apache.spark.sql.catalyst.types.{PhysicalCalendarIntervalType,
PhysicalDataType}
/**
* The data type representing calendar intervals. The calendar interval is
stored internally in
@@ -38,8 +37,6 @@ class CalendarIntervalType private() extends DataType {
override def defaultSize: Int = 16
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalCalendarIntervalType
-
override def typeName: String = "interval"
private[spark] override def asNullable: CalendarIntervalType = this
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CharType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CharType.scala
index 1a721588344..5e30ff6e52a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CharType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CharType.scala
@@ -18,14 +18,12 @@
package org.apache.spark.sql.types
import org.apache.spark.annotation.Experimental
-import org.apache.spark.sql.catalyst.types.{PhysicalDataType,
PhysicalStringType}
@Experimental
case class CharType(length: Int) extends AtomicType {
require(length >= 0, "The length of char type cannot be negative.")
override def defaultSize: Int = length
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalStringType
override def typeName: String = s"char($length)"
override def toString: String = s"CharType($length)"
private[spark] override def asNullable: CharType = this
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
index 5e2974c2645..fb9241a637d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
@@ -32,7 +32,7 @@ import org.apache.spark.annotation.Stable
import org.apache.spark.sql.catalyst.analysis.Resolver
import org.apache.spark.sql.catalyst.expressions.{Cast, Expression}
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
-import org.apache.spark.sql.catalyst.types._
+import org.apache.spark.sql.catalyst.types.DataTypeUtils
import
org.apache.spark.sql.catalyst.util.DataTypeJsonUtils.{DataTypeJsonDeserializer,
DataTypeJsonSerializer}
import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat
import org.apache.spark.sql.errors.QueryCompilationErrors
@@ -107,8 +107,6 @@ abstract class DataType extends AbstractDataType {
override private[sql] def acceptsType(other: DataType): Boolean =
DataTypeUtils.sameType(this, other)
-
- private[sql] def physicalDataType: PhysicalDataType =
UninitializedPhysicalType
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DateType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DateType.scala
index 91d40163618..d37ebbcdad7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DateType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DateType.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.types
import org.apache.spark.annotation.Stable
-import org.apache.spark.sql.catalyst.types.{PhysicalDataType,
PhysicalIntegerType}
/**
* The date type represents a valid date in the proleptic Gregorian calendar.
@@ -34,8 +33,6 @@ class DateType private() extends DatetimeType {
*/
override def defaultSize: Int = 4
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalIntegerType
-
private[spark] override def asNullable: DateType = this
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala
index 78c68ae28b1..aaea89e9a6e 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.types
import org.apache.spark.annotation.Unstable
-import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalLongType}
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.types.DayTimeIntervalType.fieldToString
@@ -47,8 +46,6 @@ case class DayTimeIntervalType(startField: Byte, endField:
Byte) extends AnsiInt
*/
override def defaultSize: Int = 8
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalLongType
-
private[spark] override def asNullable: DayTimeIntervalType = this
override val typeName: String = {
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
index e4e067b9684..b48bbcad578 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
@@ -23,7 +23,6 @@ import scala.annotation.tailrec
import org.apache.spark.annotation.Stable
import org.apache.spark.sql.catalyst.expressions.{Expression, Literal}
-import org.apache.spark.sql.catalyst.types.{PhysicalDataType,
PhysicalDecimalType}
import org.apache.spark.sql.errors.{QueryCompilationErrors,
QueryExecutionErrors}
import org.apache.spark.sql.internal.SQLConf
@@ -101,9 +100,6 @@ case class DecimalType(precision: Int, scale: Int) extends
FractionalType {
*/
override def defaultSize: Int = if (precision <= Decimal.MAX_LONG_DIGITS) 8
else 16
- private[sql] override def physicalDataType: PhysicalDataType =
- PhysicalDecimalType(precision, scale)
-
override def simpleString: String = s"decimal($precision,$scale)"
private[spark] override def asNullable: DecimalType = this
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
index b1f9f58f7bd..d18c7b98af2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql.types
import scala.util.Try
import org.apache.spark.annotation.Stable
-import org.apache.spark.sql.catalyst.types.{PhysicalDataType,
PhysicalDoubleType}
/**
* The data type representing `Double` values. Please use the singleton
`DataTypes.DoubleType`.
@@ -34,8 +33,6 @@ class DoubleType private() extends FractionalType {
*/
override def defaultSize: Int = 8
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalDoubleType
-
private[spark] override def asNullable: DoubleType = this
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala
index 59022b99939..978384eebfe 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql.types
import scala.util.Try
import org.apache.spark.annotation.Stable
-import org.apache.spark.sql.catalyst.types.{PhysicalDataType,
PhysicalFloatType}
/**
* The data type representing `Float` values. Please use the singleton
`DataTypes.FloatType`.
@@ -34,8 +33,6 @@ class FloatType private() extends FractionalType {
*/
override def defaultSize: Int = 4
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalFloatType
-
private[spark] override def asNullable: FloatType = this
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/IntegerType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/IntegerType.scala
index de2c6f3fa20..b26a555c9b5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/IntegerType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/IntegerType.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.types
import org.apache.spark.annotation.Stable
-import org.apache.spark.sql.catalyst.types.{PhysicalDataType,
PhysicalIntegerType}
/**
* The data type representing `Int` values. Please use the singleton
`DataTypes.IntegerType`.
@@ -32,8 +31,6 @@ class IntegerType private() extends IntegralType {
*/
override def defaultSize: Int = 4
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalIntegerType
-
override def simpleString: String = "int"
private[spark] override def asNullable: IntegerType = this
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/LongType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/LongType.scala
index 320c7005caf..87ebacfe9ce 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/LongType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/LongType.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.types
import org.apache.spark.annotation.Stable
-import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalLongType}
/**
* The data type representing `Long` values. Please use the singleton
`DataTypes.LongType`.
@@ -32,8 +31,6 @@ class LongType private() extends IntegralType {
*/
override def defaultSize: Int = 8
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalLongType
-
override def simpleString: String = "bigint"
private[spark] override def asNullable: LongType = this
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/MapType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/MapType.scala
index 48a8e77250a..2e5c7f731dc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/MapType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/MapType.scala
@@ -21,7 +21,6 @@ import org.json4s.JsonAST.JValue
import org.json4s.JsonDSL._
import org.apache.spark.annotation.Stable
-import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalMapType}
import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat
/**
@@ -68,9 +67,6 @@ case class MapType(
*/
override def defaultSize: Int = 1 * (keyType.defaultSize +
valueType.defaultSize)
- private[sql] override def physicalDataType: PhysicalDataType =
- PhysicalMapType(keyType, valueType, valueContainsNull)
-
override def simpleString: String =
s"map<${keyType.simpleString},${valueType.simpleString}>"
override def catalogString: String =
s"map<${keyType.catalogString},${valueType.catalogString}>"
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/NullType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/NullType.scala
index e5e8c4b6a7c..d211fac70c6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/NullType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/NullType.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.types
import org.apache.spark.annotation.Stable
-import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalNullType}
/**
* The data type representing `NULL` values. Please use the singleton
`DataTypes.NullType`.
@@ -32,8 +31,6 @@ class NullType private() extends DataType {
// Defined with a private constructor so the companion object is the only
possible instantiation.
override def defaultSize: Int = 1
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalNullType
-
private[spark] override def asNullable: NullType = this
override def typeName: String = "void"
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ShortType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ShortType.scala
index 0ec50b3df0b..66696793e62 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ShortType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ShortType.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.types
import org.apache.spark.annotation.Stable
-import org.apache.spark.sql.catalyst.types.{PhysicalDataType,
PhysicalShortType}
/**
* The data type representing `Short` values. Please use the singleton
`DataTypes.ShortType`.
@@ -32,8 +31,6 @@ class ShortType private() extends IntegralType {
*/
override def defaultSize: Int = 2
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalShortType
-
override def simpleString: String = "smallint"
private[spark] override def asNullable: ShortType = this
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StringType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StringType.scala
index a4954836f9e..5985238a863 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StringType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StringType.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.types
import org.apache.spark.annotation.Stable
-import org.apache.spark.sql.catalyst.types.{PhysicalDataType,
PhysicalStringType}
/**
* The data type representing `String` values. Please use the singleton
`DataTypes.StringType`.
@@ -32,8 +31,6 @@ class StringType private() extends AtomicType {
*/
override def defaultSize: Int = 20
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalStringType
-
private[spark] override def asNullable: StringType = this
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
index db6859fd0c7..895e4bbfee9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.analysis.Resolver
import org.apache.spark.sql.catalyst.expressions.{Attribute,
AttributeReference}
import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser,
LegacyTypeStringParser}
import org.apache.spark.sql.catalyst.trees.Origin
-import org.apache.spark.sql.catalyst.types.{DataTypeUtils, PhysicalDataType,
PhysicalStructType}
+import org.apache.spark.sql.catalyst.types.DataTypeUtils
import org.apache.spark.sql.catalyst.util.{truncatedString, StringUtils}
import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns._
import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat
@@ -431,8 +431,6 @@ case class StructType(fields: Array[StructField]) extends
DataType with Seq[Stru
*/
override def defaultSize: Int = fields.map(_.dataType.defaultSize).sum
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalStructType(fields)
-
override def simpleString: String = {
val fieldTypes = fields.view.map(field =>
s"${field.name}:${field.dataType.simpleString}").toSeq
truncatedString(
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampNTZType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampNTZType.scala
index 731549c3015..9968d75dd25 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampNTZType.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampNTZType.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.types
import org.apache.spark.annotation.Unstable
-import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalLongType}
/**
* The timestamp without time zone type represents a local time in microsecond
precision,
@@ -36,8 +35,6 @@ class TimestampNTZType private() extends DatetimeType {
*/
override def defaultSize: Int = 8
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalLongType
-
override def typeName: String = "timestamp_ntz"
private[spark] override def asNullable: TimestampNTZType = this
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampType.scala
index f077f112e55..1185e4a9e32 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampType.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.types
import org.apache.spark.annotation.Stable
-import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalLongType}
/**
* The timestamp type represents a time instant in microsecond precision.
@@ -36,8 +35,6 @@ class TimestampType private() extends DatetimeType {
*/
override def defaultSize: Int = 8
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalLongType
-
private[spark] override def asNullable: TimestampType = this
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/VarcharType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/VarcharType.scala
index 9c282a9020c..3d21e2e6580 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/VarcharType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/VarcharType.scala
@@ -17,14 +17,11 @@
package org.apache.spark.sql.types
import org.apache.spark.annotation.Experimental
-import org.apache.spark.sql.catalyst.types.{PhysicalDataType,
PhysicalStringType}
@Experimental
case class VarcharType(length: Int) extends AtomicType {
require(length >= 0, "The length of varchar type cannot be negative.")
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalStringType
-
override def defaultSize: Int = length
override def typeName: String = s"varchar($length)"
override def toString: String = s"VarcharType($length)"
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/YearMonthIntervalType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/YearMonthIntervalType.scala
index 691df915670..4cb3fda3e99 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/YearMonthIntervalType.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/YearMonthIntervalType.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.types
import org.apache.spark.annotation.Unstable
-import org.apache.spark.sql.catalyst.types.{PhysicalDataType,
PhysicalIntegerType}
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.types.YearMonthIntervalType.fieldToString
@@ -45,8 +44,6 @@ case class YearMonthIntervalType(startField: Byte, endField:
Byte) extends AnsiI
*/
override def defaultSize: Int = 4
- private[sql] override def physicalDataType: PhysicalDataType =
PhysicalIntegerType
-
private[spark] override def asNullable: YearMonthIntervalType = this
override val typeName: String = {
diff --git
a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java
b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java
index f89c10155a7..6ab81cf4048 100644
---
a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java
+++
b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java
@@ -51,7 +51,7 @@ public class ColumnVectorUtils {
*/
public static void populate(ConstantColumnVector col, InternalRow row, int
fieldIdx) {
DataType t = col.dataType();
- PhysicalDataType pdt = t.physicalDataType();
+ PhysicalDataType pdt = PhysicalDataType.apply(t);
if (row.isNullAt(fieldIdx)) {
col.setNull();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
