This is an automated email from the ASF dual-hosted git repository. srowen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 733f2c0 [MINOR][SQL] Deduplicate huge if statements in get between specialized getters 733f2c0 is described below commit 733f2c0b98208815f8408e36ab669d7c07e3767f Author: Jungtaek Lim (HeartSaVioR) <kabh...@gmail.com> AuthorDate: Wed Mar 13 15:52:21 2019 -0500 [MINOR][SQL] Deduplicate huge if statements in get between specialized getters ## What changes were proposed in this pull request? This patch deduplicates the huge if statements regarding getting value between specialized getters. ## How was this patch tested? Existing UT. Closes #24016 from HeartSaVioR/MINOR-deduplicate-get-from-specialized-getters. Authored-by: Jungtaek Lim (HeartSaVioR) <kabh...@gmail.com> Signed-off-by: Sean Owen <sean.o...@databricks.com> --- .../expressions/SpecializedGettersReader.java | 89 ++++++++++++++++++++++ .../sql/catalyst/expressions/UnsafeArrayData.java | 42 +--------- .../spark/sql/catalyst/expressions/UnsafeRow.java | 41 +--------- .../apache/spark/sql/vectorized/ColumnarArray.java | 38 +-------- 4 files changed, 93 insertions(+), 117 deletions(-) diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java new file mode 100644 index 0000000..ea0648a --- /dev/null +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.expressions; + +import org.apache.spark.sql.types.*; + +public final class SpecializedGettersReader { + private SpecializedGettersReader() {} + + public static Object read( + SpecializedGetters obj, + int ordinal, + DataType dataType, + boolean handleNull, + boolean handleUserDefinedType) { + if (handleNull && (obj.isNullAt(ordinal) || dataType instanceof NullType)) { + return null; + } + if (dataType instanceof BooleanType) { + return obj.getBoolean(ordinal); + } + if (dataType instanceof ByteType) { + return obj.getByte(ordinal); + } + if (dataType instanceof ShortType) { + return obj.getShort(ordinal); + } + if (dataType instanceof IntegerType) { + return obj.getInt(ordinal); + } + if (dataType instanceof LongType) { + return obj.getLong(ordinal); + } + if (dataType instanceof FloatType) { + return obj.getFloat(ordinal); + } + if (dataType instanceof DoubleType) { + return obj.getDouble(ordinal); + } + if (dataType instanceof StringType) { + return obj.getUTF8String(ordinal); + } + if (dataType instanceof DecimalType) { + DecimalType dt = (DecimalType) dataType; + return obj.getDecimal(ordinal, dt.precision(), dt.scale()); + } + if (dataType instanceof DateType) { + return obj.getInt(ordinal); + } + if (dataType instanceof TimestampType) { + return obj.getLong(ordinal); + } + if (dataType instanceof CalendarIntervalType) { + return obj.getInterval(ordinal); + } + if (dataType instanceof BinaryType) { + return obj.getBinary(ordinal); + } + if (dataType instanceof StructType) { + return obj.getStruct(ordinal, ((StructType) dataType).size()); + } + if (dataType instanceof ArrayType) { + return obj.getArray(ordinal); + } + if (dataType instanceof MapType) { + return obj.getMap(ordinal); + } + if (handleUserDefinedType && dataType instanceof UserDefinedType) { + return obj.get(ordinal, ((UserDefinedType)dataType).sqlType()); + } + + throw new UnsupportedOperationException("Unsupported data type " + dataType.simpleString()); + } +} diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java index a07d6de..04fc986 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java @@ -53,7 +53,6 @@ import org.apache.spark.unsafe.types.UTF8String; */ public final class UnsafeArrayData extends ArrayData { - public static int calculateHeaderPortionInBytes(int numFields) { return (int)calculateHeaderPortionInBytes((long)numFields); } @@ -137,46 +136,7 @@ public final class UnsafeArrayData extends ArrayData { @Override public Object get(int ordinal, DataType dataType) { - if (isNullAt(ordinal) || dataType instanceof NullType) { - return null; - } else if (dataType instanceof BooleanType) { - return getBoolean(ordinal); - } else if (dataType instanceof ByteType) { - return getByte(ordinal); - } else if (dataType instanceof ShortType) { - return getShort(ordinal); - } else if (dataType instanceof IntegerType) { - return getInt(ordinal); - } else if (dataType instanceof LongType) { - return getLong(ordinal); - } else if (dataType instanceof FloatType) { - return getFloat(ordinal); - } else if (dataType instanceof DoubleType) { - return getDouble(ordinal); - } else if (dataType instanceof DecimalType) { - DecimalType dt = (DecimalType) dataType; - return getDecimal(ordinal, dt.precision(), dt.scale()); - } else if (dataType instanceof DateType) { - return getInt(ordinal); - } else if (dataType instanceof TimestampType) { - return getLong(ordinal); - } else if (dataType instanceof BinaryType) { - return getBinary(ordinal); - } else if (dataType instanceof StringType) { - return getUTF8String(ordinal); - } else if (dataType instanceof CalendarIntervalType) { - return getInterval(ordinal); - } else if (dataType instanceof StructType) { - return getStruct(ordinal, ((StructType) dataType).size()); - } else if (dataType instanceof ArrayType) { - return getArray(ordinal); - } else if (dataType instanceof MapType) { - return getMap(ordinal); - } else if (dataType instanceof UserDefinedType) { - return get(ordinal, ((UserDefinedType)dataType).sqlType()); - } else { - throw new UnsupportedOperationException("Unsupported data type " + dataType.simpleString()); - } + return SpecializedGettersReader.read(this, ordinal, dataType, true, true); } @Override diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java index 9bf9452..11561fa 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java @@ -299,46 +299,7 @@ public final class UnsafeRow extends InternalRow implements Externalizable, Kryo @Override public Object get(int ordinal, DataType dataType) { - if (isNullAt(ordinal) || dataType instanceof NullType) { - return null; - } else if (dataType instanceof BooleanType) { - return getBoolean(ordinal); - } else if (dataType instanceof ByteType) { - return getByte(ordinal); - } else if (dataType instanceof ShortType) { - return getShort(ordinal); - } else if (dataType instanceof IntegerType) { - return getInt(ordinal); - } else if (dataType instanceof LongType) { - return getLong(ordinal); - } else if (dataType instanceof FloatType) { - return getFloat(ordinal); - } else if (dataType instanceof DoubleType) { - return getDouble(ordinal); - } else if (dataType instanceof DecimalType) { - DecimalType dt = (DecimalType) dataType; - return getDecimal(ordinal, dt.precision(), dt.scale()); - } else if (dataType instanceof DateType) { - return getInt(ordinal); - } else if (dataType instanceof TimestampType) { - return getLong(ordinal); - } else if (dataType instanceof BinaryType) { - return getBinary(ordinal); - } else if (dataType instanceof StringType) { - return getUTF8String(ordinal); - } else if (dataType instanceof CalendarIntervalType) { - return getInterval(ordinal); - } else if (dataType instanceof StructType) { - return getStruct(ordinal, ((StructType) dataType).size()); - } else if (dataType instanceof ArrayType) { - return getArray(ordinal); - } else if (dataType instanceof MapType) { - return getMap(ordinal); - } else if (dataType instanceof UserDefinedType) { - return get(ordinal, ((UserDefinedType)dataType).sqlType()); - } else { - throw new UnsupportedOperationException("Unsupported data type " + dataType.simpleString()); - } + return SpecializedGettersReader.read(this, ordinal, dataType, true, true); } @Override diff --git a/sql/core/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java b/sql/core/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java index 8dc7b11..147dd24 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java +++ b/sql/core/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java @@ -17,6 +17,7 @@ package org.apache.spark.sql.vectorized; import org.apache.spark.annotation.Evolving; +import org.apache.spark.sql.catalyst.expressions.SpecializedGettersReader; import org.apache.spark.sql.catalyst.expressions.UnsafeArrayData; import org.apache.spark.sql.catalyst.util.ArrayData; import org.apache.spark.sql.catalyst.util.GenericArrayData; @@ -174,42 +175,7 @@ public final class ColumnarArray extends ArrayData { @Override public Object get(int ordinal, DataType dataType) { - if (dataType instanceof BooleanType) { - return getBoolean(ordinal); - } else if (dataType instanceof ByteType) { - return getByte(ordinal); - } else if (dataType instanceof ShortType) { - return getShort(ordinal); - } else if (dataType instanceof IntegerType) { - return getInt(ordinal); - } else if (dataType instanceof LongType) { - return getLong(ordinal); - } else if (dataType instanceof FloatType) { - return getFloat(ordinal); - } else if (dataType instanceof DoubleType) { - return getDouble(ordinal); - } else if (dataType instanceof StringType) { - return getUTF8String(ordinal); - } else if (dataType instanceof BinaryType) { - return getBinary(ordinal); - } else if (dataType instanceof DecimalType) { - DecimalType t = (DecimalType) dataType; - return getDecimal(ordinal, t.precision(), t.scale()); - } else if (dataType instanceof DateType) { - return getInt(ordinal); - } else if (dataType instanceof TimestampType) { - return getLong(ordinal); - } else if (dataType instanceof ArrayType) { - return getArray(ordinal); - } else if (dataType instanceof StructType) { - return getStruct(ordinal, ((StructType)dataType).fields().length); - } else if (dataType instanceof MapType) { - return getMap(ordinal); - } else if (dataType instanceof CalendarIntervalType) { - return getInterval(ordinal); - } else { - throw new UnsupportedOperationException("Datatype not supported " + dataType); - } + return SpecializedGettersReader.read(this, ordinal, dataType, false, false); } @Override --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org