This is an automated email from the ASF dual-hosted git repository. hvanhovell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new db2625c70a8 [SPARK-43168][SQL] Remove get PhysicalDataType method from Datatype class db2625c70a8 is described below commit db2625c70a8c3aff64e6a9466981c8dd49a4ca51 Author: Rui Wang <rui.w...@databricks.com> AuthorDate: Mon Apr 17 22:16:25 2023 -0400 [SPARK-43168][SQL] Remove get PhysicalDataType method from Datatype class ### What changes were proposed in this pull request? DataType is public API while we can leave PhysicalDataType as internal API/implementation thus we can remove PhysicalDataType from DataType. So DataType does not need to have a class dependency on PhysicalDataType. ### Why are the changes needed? Simplify DataType. ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? UT Closes #40826 from amaliujia/catalyst_datatype_refactor_8. Authored-by: Rui Wang <rui.w...@databricks.com> Signed-off-by: Herman van Hovell <her...@databricks.com> --- .../spark/sql/catalyst/expressions/SpecializedGettersReader.java | 2 +- .../java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java | 4 ++-- .../main/java/org/apache/spark/sql/vectorized/ColumnarBatchRow.java | 2 +- .../src/main/java/org/apache/spark/sql/vectorized/ColumnarRow.java | 2 +- .../src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala | 2 +- .../spark/sql/catalyst/expressions/InterpretedUnsafeProjection.scala | 2 +- .../spark/sql/catalyst/expressions/codegen/CodeGenerator.scala | 4 ++-- .../scala/org/apache/spark/sql/catalyst/expressions/literals.scala | 2 +- .../org/apache/spark/sql/catalyst/expressions/namedExpressions.scala | 2 +- .../scala/org/apache/spark/sql/catalyst/types/PhysicalDataType.scala | 5 ++++- .../src/main/scala/org/apache/spark/sql/types/ArrayType.scala | 4 ---- .../src/main/scala/org/apache/spark/sql/types/BinaryType.scala | 3 --- .../src/main/scala/org/apache/spark/sql/types/BooleanType.scala | 3 --- .../src/main/scala/org/apache/spark/sql/types/ByteType.scala | 3 --- .../main/scala/org/apache/spark/sql/types/CalendarIntervalType.scala | 3 --- .../src/main/scala/org/apache/spark/sql/types/CharType.scala | 2 -- .../src/main/scala/org/apache/spark/sql/types/DataType.scala | 4 +--- .../src/main/scala/org/apache/spark/sql/types/DateType.scala | 3 --- .../main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala | 3 --- .../src/main/scala/org/apache/spark/sql/types/DecimalType.scala | 4 ---- .../src/main/scala/org/apache/spark/sql/types/DoubleType.scala | 3 --- .../src/main/scala/org/apache/spark/sql/types/FloatType.scala | 3 --- .../src/main/scala/org/apache/spark/sql/types/IntegerType.scala | 3 --- .../src/main/scala/org/apache/spark/sql/types/LongType.scala | 3 --- sql/catalyst/src/main/scala/org/apache/spark/sql/types/MapType.scala | 4 ---- .../src/main/scala/org/apache/spark/sql/types/NullType.scala | 3 --- .../src/main/scala/org/apache/spark/sql/types/ShortType.scala | 3 --- .../src/main/scala/org/apache/spark/sql/types/StringType.scala | 3 --- .../src/main/scala/org/apache/spark/sql/types/StructType.scala | 4 +--- .../src/main/scala/org/apache/spark/sql/types/TimestampNTZType.scala | 3 --- .../src/main/scala/org/apache/spark/sql/types/TimestampType.scala | 3 --- .../src/main/scala/org/apache/spark/sql/types/VarcharType.scala | 3 --- .../scala/org/apache/spark/sql/types/YearMonthIntervalType.scala | 3 --- .../org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java | 2 +- 34 files changed, 18 insertions(+), 84 deletions(-) diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java index c5a7d34281f..be50350b106 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java @@ -29,7 +29,7 @@ public final class SpecializedGettersReader { DataType dataType, boolean handleNull, boolean handleUserDefinedType) { - PhysicalDataType physicalDataType = dataType.physicalDataType(); + PhysicalDataType physicalDataType = PhysicalDataType.apply(dataType); if (handleNull && (obj.isNullAt(ordinal) || physicalDataType instanceof PhysicalNullType)) { return null; } diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java index a3097cd6770..d2433292fc7 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java @@ -73,7 +73,7 @@ public final class UnsafeRow extends InternalRow implements Externalizable, Kryo if (dt instanceof UserDefinedType) { return isFixedLength(((UserDefinedType<?>) dt).sqlType()); } - PhysicalDataType pdt = dt.physicalDataType(); + PhysicalDataType pdt = PhysicalDataType.apply(dt); if (pdt instanceof PhysicalDecimalType) { return ((DecimalType) dt).precision() <= Decimal.MAX_LONG_DIGITS(); } else { @@ -88,7 +88,7 @@ public final class UnsafeRow extends InternalRow implements Externalizable, Kryo if (dt instanceof UserDefinedType) { return isMutable(((UserDefinedType<?>) dt).sqlType()); } - PhysicalDataType pdt = dt.physicalDataType(); + PhysicalDataType pdt = PhysicalDataType.apply(dt); return pdt instanceof PhysicalPrimitiveType || pdt instanceof PhysicalDecimalType || pdt instanceof PhysicalCalendarIntervalType; } diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarBatchRow.java b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarBatchRow.java index 05922a0cd5d..d30155eb2b4 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarBatchRow.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarBatchRow.java @@ -49,7 +49,7 @@ public final class ColumnarBatchRow extends InternalRow { row.setNullAt(i); } else { DataType dt = columns[i].dataType(); - PhysicalDataType pdt = dt.physicalDataType(); + PhysicalDataType pdt = PhysicalDataType.apply(dt); if (pdt instanceof PhysicalBooleanType) { row.setBoolean(i, getBoolean(i)); } else if (pdt instanceof PhysicalByteType) { diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarRow.java b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarRow.java index 9c2b1833348..c4fbc2ff642 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarRow.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarRow.java @@ -56,7 +56,7 @@ public final class ColumnarRow extends InternalRow { row.setNullAt(i); } else { DataType dt = data.getChild(i).dataType(); - PhysicalDataType pdt = dt.physicalDataType(); + PhysicalDataType pdt = PhysicalDataType.apply(dt); if (pdt instanceof PhysicalBooleanType) { row.setBoolean(i, getBoolean(i)); } else if (pdt instanceof PhysicalByteType) { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala index a44dca7dda9..02b42f11cc0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala @@ -131,7 +131,7 @@ object InternalRow { def getAccessor(dt: DataType, nullable: Boolean = true): (SpecializedGetters, Int) => Any = { val getValueNullSafe: (SpecializedGetters, Int) => Any = dt match { case u: UserDefinedType[_] => getAccessor(u.sqlType, nullable) - case _ => dt.physicalDataType match { + case _ => PhysicalDataType(dt) match { case PhysicalBooleanType => (input, ordinal) => input.getBoolean(ordinal) case PhysicalByteType => (input, ordinal) => input.getByte(ordinal) case PhysicalShortType => (input, ordinal) => input.getShort(ordinal) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedUnsafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedUnsafeProjection.scala index 90a90444695..a8f21a18ca7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedUnsafeProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InterpretedUnsafeProjection.scala @@ -136,7 +136,7 @@ object InterpretedUnsafeProjection { // Create the basic writer. val unsafeWriter: (SpecializedGetters, Int) => Unit = dt match { case udt: UserDefinedType[_] => generateFieldWriter(writer, udt.sqlType, nullable) - case _ => dt.physicalDataType match { + case _ => PhysicalDataType(dt) match { case PhysicalBooleanType => (v, i) => writer.write(i, v.getBoolean(i)) case PhysicalByteType => (v, i) => writer.write(i, v.getByte(i)) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala index 12103ceef6e..5651a30515f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala @@ -1628,7 +1628,7 @@ object CodeGenerator extends Logging { dataType match { case udt: UserDefinedType[_] => getValue(input, udt.sqlType, ordinal) case _ if isPrimitiveType(jt) => s"$input.get${primitiveTypeName(jt)}($ordinal)" - case _ => dataType.physicalDataType match { + case _ => PhysicalDataType(dataType) match { case _: PhysicalArrayType => s"$input.getArray($ordinal)" case PhysicalBinaryType => s"$input.getBinary($ordinal)" case PhysicalCalendarIntervalType => s"$input.getInterval($ordinal)" @@ -1909,7 +1909,7 @@ object CodeGenerator extends Logging { case udt: UserDefinedType[_] => javaType(udt.sqlType) case ObjectType(cls) if cls.isArray => s"${javaType(ObjectType(cls.getComponentType))}[]" case ObjectType(cls) => cls.getName - case _ => dt.physicalDataType match { + case _ => PhysicalDataType(dt) match { case _: PhysicalArrayType => "ArrayData" case PhysicalBinaryType => "byte[]" case PhysicalBooleanType => JAVA_BOOLEAN diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala index e8ac858eb11..dd41cefb724 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala @@ -208,7 +208,7 @@ object Literal { case _ if v == null => true case ObjectType(cls) => cls.isInstance(v) case udt: UserDefinedType[_] => doValidate(v, udt.sqlType) - case dt => dataType.physicalDataType match { + case dt => PhysicalDataType(dataType) match { case PhysicalArrayType(et, _) => v.isInstanceOf[ArrayData] && { val ar = v.asInstanceOf[ArrayData] diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala index 048f379f8a3..4f4fbc07b16 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala @@ -563,7 +563,7 @@ object FileSourceMetadataAttribute { * supported (including most primitive types). Notably unsupported types include [[ObjectType]], * [[UserDefinedType]], and the complex types ([[StructType]], [[MapType]], [[ArrayType]]). */ - def isSupportedType(dataType: DataType): Boolean = dataType.physicalDataType match { + def isSupportedType(dataType: DataType): Boolean = PhysicalDataType(dataType) match { // PhysicalPrimitiveType covers: Boolean, Byte, Double, Float, Integer, Long, Null, Short case _: PhysicalPrimitiveType | _: PhysicalDecimalType => true case PhysicalBinaryType | PhysicalStringType | PhysicalCalendarIntervalType => true diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/PhysicalDataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/PhysicalDataType.scala index b6e0cd88f08..d392557e650 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/PhysicalDataType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/PhysicalDataType.scala @@ -23,7 +23,7 @@ import scala.reflect.runtime.universe.typeTag import org.apache.spark.sql.catalyst.expressions.{Ascending, BoundReference, InterpretedOrdering, SortOrder} import org.apache.spark.sql.catalyst.util.{ArrayData, SQLOrderingUtil} import org.apache.spark.sql.errors.QueryExecutionErrors -import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteExactNumeric, ByteType, DataType, DateType, DayTimeIntervalType, Decimal, DecimalExactNumeric, DecimalType, DoubleExactNumeric, DoubleType, FloatExactNumeric, FloatType, FractionalType, IntegerExactNumeric, IntegerType, IntegralType, LongExactNumeric, LongType, MapType, NullType, NumericType, ShortExactNumeric, ShortType, StringType, StructField, StructType, TimestampNTZType, TimestampType, YearMonthIntervalType} +import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteExactNumeric, ByteType, CalendarIntervalType, CharType, DataType, DateType, DayTimeIntervalType, Decimal, DecimalExactNumeric, DecimalType, DoubleExactNumeric, DoubleType, FloatExactNumeric, FloatType, FractionalType, IntegerExactNumeric, IntegerType, IntegralType, LongExactNumeric, LongType, MapType, NullType, NumericType, ShortExactNumeric, ShortType, StringType, StructField, StructType, TimestampNTZType, Timest [...] import org.apache.spark.unsafe.types.{ByteArray, UTF8String} sealed abstract class PhysicalDataType { @@ -39,6 +39,8 @@ object PhysicalDataType { case ShortType => PhysicalShortType case IntegerType => PhysicalIntegerType case LongType => PhysicalLongType + case VarcharType(_) => PhysicalStringType + case CharType(_) => PhysicalStringType case StringType => PhysicalStringType case FloatType => PhysicalFloatType case DoubleType => PhysicalDoubleType @@ -47,6 +49,7 @@ object PhysicalDataType { case BinaryType => PhysicalBinaryType case TimestampType => PhysicalLongType case TimestampNTZType => PhysicalLongType + case CalendarIntervalType => PhysicalCalendarIntervalType case DayTimeIntervalType(_, _) => PhysicalLongType case YearMonthIntervalType(_, _) => PhysicalIntegerType case DateType => PhysicalIntegerType diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala index 553ac4e15a5..d6f5c3bdf43 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala @@ -20,7 +20,6 @@ package org.apache.spark.sql.types import org.json4s.JsonDSL._ import org.apache.spark.annotation.Stable -import org.apache.spark.sql.catalyst.types.{PhysicalArrayType, PhysicalDataType} import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat /** @@ -88,9 +87,6 @@ case class ArrayType(elementType: DataType, containsNull: Boolean) extends DataT */ override def defaultSize: Int = 1 * elementType.defaultSize - private[sql] override def physicalDataType: PhysicalDataType = - PhysicalArrayType(elementType, containsNull) - override def simpleString: String = s"array<${elementType.simpleString}>" override def catalogString: String = s"array<${elementType.catalogString}>" diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BinaryType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BinaryType.scala index 25f29e12dce..c280f66f943 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BinaryType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BinaryType.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.types import org.apache.spark.annotation.Stable -import org.apache.spark.sql.catalyst.types.{PhysicalBinaryType, PhysicalDataType} /** * The data type representing `Array[Byte]` values. @@ -31,8 +30,6 @@ class BinaryType private() extends AtomicType { */ override def defaultSize: Int = 100 - private[sql] override def physicalDataType: PhysicalDataType = PhysicalBinaryType - private[spark] override def asNullable: BinaryType = this } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BooleanType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BooleanType.scala index ddcc2fa75bd..836c41a996a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BooleanType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BooleanType.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.types import org.apache.spark.annotation.Stable -import org.apache.spark.sql.catalyst.types.{PhysicalBooleanType, PhysicalDataType} /** * The data type representing `Boolean` values. Please use the singleton `DataTypes.BooleanType`. @@ -32,8 +31,6 @@ class BooleanType private() extends AtomicType { */ override def defaultSize: Int = 1 - private[sql] override def physicalDataType: PhysicalDataType = PhysicalBooleanType - private[spark] override def asNullable: BooleanType = this } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ByteType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ByteType.scala index d525a9072ed..546ac02f263 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ByteType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ByteType.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.types import org.apache.spark.annotation.Stable -import org.apache.spark.sql.catalyst.types.{PhysicalByteType, PhysicalDataType} /** * The data type representing `Byte` values. Please use the singleton `DataTypes.ByteType`. @@ -32,8 +31,6 @@ class ByteType private() extends IntegralType { */ override def defaultSize: Int = 1 - private[sql] override def physicalDataType: PhysicalDataType = PhysicalByteType - override def simpleString: String = "tinyint" private[spark] override def asNullable: ByteType = this diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CalendarIntervalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CalendarIntervalType.scala index 7f1c51ef23d..d506a1521e1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CalendarIntervalType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CalendarIntervalType.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.types import org.apache.spark.annotation.Stable -import org.apache.spark.sql.catalyst.types.{PhysicalCalendarIntervalType, PhysicalDataType} /** * The data type representing calendar intervals. The calendar interval is stored internally in @@ -38,8 +37,6 @@ class CalendarIntervalType private() extends DataType { override def defaultSize: Int = 16 - private[sql] override def physicalDataType: PhysicalDataType = PhysicalCalendarIntervalType - override def typeName: String = "interval" private[spark] override def asNullable: CalendarIntervalType = this diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CharType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CharType.scala index 1a721588344..5e30ff6e52a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CharType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/CharType.scala @@ -18,14 +18,12 @@ package org.apache.spark.sql.types import org.apache.spark.annotation.Experimental -import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalStringType} @Experimental case class CharType(length: Int) extends AtomicType { require(length >= 0, "The length of char type cannot be negative.") override def defaultSize: Int = length - private[sql] override def physicalDataType: PhysicalDataType = PhysicalStringType override def typeName: String = s"char($length)" override def toString: String = s"CharType($length)" private[spark] override def asNullable: CharType = this diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala index 5e2974c2645..fb9241a637d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala @@ -32,7 +32,7 @@ import org.apache.spark.annotation.Stable import org.apache.spark.sql.catalyst.analysis.Resolver import org.apache.spark.sql.catalyst.expressions.{Cast, Expression} import org.apache.spark.sql.catalyst.parser.CatalystSqlParser -import org.apache.spark.sql.catalyst.types._ +import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.catalyst.util.DataTypeJsonUtils.{DataTypeJsonDeserializer, DataTypeJsonSerializer} import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat import org.apache.spark.sql.errors.QueryCompilationErrors @@ -107,8 +107,6 @@ abstract class DataType extends AbstractDataType { override private[sql] def acceptsType(other: DataType): Boolean = DataTypeUtils.sameType(this, other) - - private[sql] def physicalDataType: PhysicalDataType = UninitializedPhysicalType } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DateType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DateType.scala index 91d40163618..d37ebbcdad7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DateType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DateType.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.types import org.apache.spark.annotation.Stable -import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalIntegerType} /** * The date type represents a valid date in the proleptic Gregorian calendar. @@ -34,8 +33,6 @@ class DateType private() extends DatetimeType { */ override def defaultSize: Int = 4 - private[sql] override def physicalDataType: PhysicalDataType = PhysicalIntegerType - private[spark] override def asNullable: DateType = this } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala index 78c68ae28b1..aaea89e9a6e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.types import org.apache.spark.annotation.Unstable -import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalLongType} import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.sql.types.DayTimeIntervalType.fieldToString @@ -47,8 +46,6 @@ case class DayTimeIntervalType(startField: Byte, endField: Byte) extends AnsiInt */ override def defaultSize: Int = 8 - private[sql] override def physicalDataType: PhysicalDataType = PhysicalLongType - private[spark] override def asNullable: DayTimeIntervalType = this override val typeName: String = { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala index e4e067b9684..b48bbcad578 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala @@ -23,7 +23,6 @@ import scala.annotation.tailrec import org.apache.spark.annotation.Stable import org.apache.spark.sql.catalyst.expressions.{Expression, Literal} -import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalDecimalType} import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors} import org.apache.spark.sql.internal.SQLConf @@ -101,9 +100,6 @@ case class DecimalType(precision: Int, scale: Int) extends FractionalType { */ override def defaultSize: Int = if (precision <= Decimal.MAX_LONG_DIGITS) 8 else 16 - private[sql] override def physicalDataType: PhysicalDataType = - PhysicalDecimalType(precision, scale) - override def simpleString: String = s"decimal($precision,$scale)" private[spark] override def asNullable: DecimalType = this diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala index b1f9f58f7bd..d18c7b98af2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala @@ -20,7 +20,6 @@ package org.apache.spark.sql.types import scala.util.Try import org.apache.spark.annotation.Stable -import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalDoubleType} /** * The data type representing `Double` values. Please use the singleton `DataTypes.DoubleType`. @@ -34,8 +33,6 @@ class DoubleType private() extends FractionalType { */ override def defaultSize: Int = 8 - private[sql] override def physicalDataType: PhysicalDataType = PhysicalDoubleType - private[spark] override def asNullable: DoubleType = this } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala index 59022b99939..978384eebfe 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala @@ -20,7 +20,6 @@ package org.apache.spark.sql.types import scala.util.Try import org.apache.spark.annotation.Stable -import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalFloatType} /** * The data type representing `Float` values. Please use the singleton `DataTypes.FloatType`. @@ -34,8 +33,6 @@ class FloatType private() extends FractionalType { */ override def defaultSize: Int = 4 - private[sql] override def physicalDataType: PhysicalDataType = PhysicalFloatType - private[spark] override def asNullable: FloatType = this } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/IntegerType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/IntegerType.scala index de2c6f3fa20..b26a555c9b5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/IntegerType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/IntegerType.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.types import org.apache.spark.annotation.Stable -import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalIntegerType} /** * The data type representing `Int` values. Please use the singleton `DataTypes.IntegerType`. @@ -32,8 +31,6 @@ class IntegerType private() extends IntegralType { */ override def defaultSize: Int = 4 - private[sql] override def physicalDataType: PhysicalDataType = PhysicalIntegerType - override def simpleString: String = "int" private[spark] override def asNullable: IntegerType = this diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/LongType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/LongType.scala index 320c7005caf..87ebacfe9ce 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/LongType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/LongType.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.types import org.apache.spark.annotation.Stable -import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalLongType} /** * The data type representing `Long` values. Please use the singleton `DataTypes.LongType`. @@ -32,8 +31,6 @@ class LongType private() extends IntegralType { */ override def defaultSize: Int = 8 - private[sql] override def physicalDataType: PhysicalDataType = PhysicalLongType - override def simpleString: String = "bigint" private[spark] override def asNullable: LongType = this diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/MapType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/MapType.scala index 48a8e77250a..2e5c7f731dc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/MapType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/MapType.scala @@ -21,7 +21,6 @@ import org.json4s.JsonAST.JValue import org.json4s.JsonDSL._ import org.apache.spark.annotation.Stable -import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalMapType} import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat /** @@ -68,9 +67,6 @@ case class MapType( */ override def defaultSize: Int = 1 * (keyType.defaultSize + valueType.defaultSize) - private[sql] override def physicalDataType: PhysicalDataType = - PhysicalMapType(keyType, valueType, valueContainsNull) - override def simpleString: String = s"map<${keyType.simpleString},${valueType.simpleString}>" override def catalogString: String = s"map<${keyType.catalogString},${valueType.catalogString}>" diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/NullType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/NullType.scala index e5e8c4b6a7c..d211fac70c6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/NullType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/NullType.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.types import org.apache.spark.annotation.Stable -import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalNullType} /** * The data type representing `NULL` values. Please use the singleton `DataTypes.NullType`. @@ -32,8 +31,6 @@ class NullType private() extends DataType { // Defined with a private constructor so the companion object is the only possible instantiation. override def defaultSize: Int = 1 - private[sql] override def physicalDataType: PhysicalDataType = PhysicalNullType - private[spark] override def asNullable: NullType = this override def typeName: String = "void" diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ShortType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ShortType.scala index 0ec50b3df0b..66696793e62 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ShortType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ShortType.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.types import org.apache.spark.annotation.Stable -import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalShortType} /** * The data type representing `Short` values. Please use the singleton `DataTypes.ShortType`. @@ -32,8 +31,6 @@ class ShortType private() extends IntegralType { */ override def defaultSize: Int = 2 - private[sql] override def physicalDataType: PhysicalDataType = PhysicalShortType - override def simpleString: String = "smallint" private[spark] override def asNullable: ShortType = this diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StringType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StringType.scala index a4954836f9e..5985238a863 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StringType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StringType.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.types import org.apache.spark.annotation.Stable -import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalStringType} /** * The data type representing `String` values. Please use the singleton `DataTypes.StringType`. @@ -32,8 +31,6 @@ class StringType private() extends AtomicType { */ override def defaultSize: Int = 20 - private[sql] override def physicalDataType: PhysicalDataType = PhysicalStringType - private[spark] override def asNullable: StringType = this } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala index db6859fd0c7..895e4bbfee9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala @@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.analysis.Resolver import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, LegacyTypeStringParser} import org.apache.spark.sql.catalyst.trees.Origin -import org.apache.spark.sql.catalyst.types.{DataTypeUtils, PhysicalDataType, PhysicalStructType} +import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.catalyst.util.{truncatedString, StringUtils} import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns._ import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat @@ -431,8 +431,6 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru */ override def defaultSize: Int = fields.map(_.dataType.defaultSize).sum - private[sql] override def physicalDataType: PhysicalDataType = PhysicalStructType(fields) - override def simpleString: String = { val fieldTypes = fields.view.map(field => s"${field.name}:${field.dataType.simpleString}").toSeq truncatedString( diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampNTZType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampNTZType.scala index 731549c3015..9968d75dd25 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampNTZType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampNTZType.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.types import org.apache.spark.annotation.Unstable -import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalLongType} /** * The timestamp without time zone type represents a local time in microsecond precision, @@ -36,8 +35,6 @@ class TimestampNTZType private() extends DatetimeType { */ override def defaultSize: Int = 8 - private[sql] override def physicalDataType: PhysicalDataType = PhysicalLongType - override def typeName: String = "timestamp_ntz" private[spark] override def asNullable: TimestampNTZType = this diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampType.scala index f077f112e55..1185e4a9e32 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampType.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.types import org.apache.spark.annotation.Stable -import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalLongType} /** * The timestamp type represents a time instant in microsecond precision. @@ -36,8 +35,6 @@ class TimestampType private() extends DatetimeType { */ override def defaultSize: Int = 8 - private[sql] override def physicalDataType: PhysicalDataType = PhysicalLongType - private[spark] override def asNullable: TimestampType = this } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/VarcharType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/VarcharType.scala index 9c282a9020c..3d21e2e6580 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/VarcharType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/VarcharType.scala @@ -17,14 +17,11 @@ package org.apache.spark.sql.types import org.apache.spark.annotation.Experimental -import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalStringType} @Experimental case class VarcharType(length: Int) extends AtomicType { require(length >= 0, "The length of varchar type cannot be negative.") - private[sql] override def physicalDataType: PhysicalDataType = PhysicalStringType - override def defaultSize: Int = length override def typeName: String = s"varchar($length)" override def toString: String = s"VarcharType($length)" diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/YearMonthIntervalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/YearMonthIntervalType.scala index 691df915670..4cb3fda3e99 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/YearMonthIntervalType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/YearMonthIntervalType.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.types import org.apache.spark.annotation.Unstable -import org.apache.spark.sql.catalyst.types.{PhysicalDataType, PhysicalIntegerType} import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.sql.types.YearMonthIntervalType.fieldToString @@ -45,8 +44,6 @@ case class YearMonthIntervalType(startField: Byte, endField: Byte) extends AnsiI */ override def defaultSize: Int = 4 - private[sql] override def physicalDataType: PhysicalDataType = PhysicalIntegerType - private[spark] override def asNullable: YearMonthIntervalType = this override val typeName: String = { diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java index f89c10155a7..6ab81cf4048 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java +++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java @@ -51,7 +51,7 @@ public class ColumnVectorUtils { */ public static void populate(ConstantColumnVector col, InternalRow row, int fieldIdx) { DataType t = col.dataType(); - PhysicalDataType pdt = t.physicalDataType(); + PhysicalDataType pdt = PhysicalDataType.apply(t); if (row.isNullAt(fieldIdx)) { col.setNull(); --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org