[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1096439296
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,275 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+
+/**
+ * Truncates a number to the specified number of digits.
+ * @param child
+ * expression to get the number to be truncated.
+ * @param scale
+ * expression to get the number of decimal places to truncate to.
+ */
+case class TruncNumber(child: Expression, scale: Expression)
+extends BaseBinaryExpression
+with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = child.dataType
+
+ /**
+ * This overridden implementation delegates the overloaded TruncNumber.trunc
methods based on
+ * data type of input values
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+(dataType, input1) match {
+ // Trunc function accepts a second parameter to truncate the input
number.
+ // If 0, it removes all the decimal values and returns only the integer.
+ // If negative, the number is truncated to the left side of the decimal
point.
+ // Value of decimal places to truncate can range from -ve to +ve
+ // 1) In the case of integral numbers, as there is no decimal part if
the value of decimal
+ // places to truncate is +ve, then we can return that input value
without any
+ // modification as there is no +ve decimal place to be truncated from an
integral number
+ // Truncate the input only if the value of decimal places to truncate is
< 0
+ case (ByteType, input: Byte) if (scaleValue < 0) =>
+TruncNumber.trunc(input.toLong, scaleValue).toByte
+ case (ShortType, input: Short) if (scaleValue < 0) =>
+TruncNumber.trunc(input.toLong, scaleValue).shortValue
+ case (IntegerType, input: Int) if (scaleValue < 0) =>
+TruncNumber.trunc(input.toLong, scaleValue).intValue
+ case (LongType, input: Long) if (scaleValue < 0) =>
+TruncNumber.trunc(input, scaleValue).longValue
+ // 2) In the case of Float, Double, and Decimal , TruncNumber.trunc
+ // will accept both -ve and +ve values
+ case (FloatType, input: Float) =>
+TruncNumber.trunc(input, scaleValue).floatValue
+ case (DoubleType, input: Double) =>
+TruncNumber.trunc(input, scaleValue).doubleValue
+ case (DecimalType.Fixed(p, s), input: Decimal) =>
+Decimal(TruncNumber.trunc(input.toJavaBigDecimal, scaleValue), p, s)
+ case _ => input1
+}
+ }
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this expression.
+ * This overridden implementation delegates the overloaded TruncNumber.trunc
methods based on
+ * data type of input values
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+val methodName =
"org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc"
+// Trunc function accepts a second parameter to truncate the input
number.
+// If 0, it removes all the decimal values and returns only the
integer.
+// If negative, the number is truncated to the left side of the
decimal point.
+// Value of decimal places to truncate can range from -ve to +ve
+// 1) In the case of integral numbers, as there is no decimal part if
the value of decimal
+// places to truncate is +ve, then we can return that input value
without any
+// modification as there is no +ve decimal place to be truncated from
an integral number
+// Truncate the input only if the value of decimal places to truncate
is < 0
+dataType match {
+ case ByteType if (scaleValue < 0) =>
+s"""(byte)($methodName(
+ |(long)$input, $scaleValue))""".stripMargin
+ case ShortType if (scaleValue < 0) =>
+s"""(short)($methodName(
+ |(long)$input, $scaleValue))""".stripMargin
+ case IntegerType if (scaleValue < 0) =>
+s"""(int)($methodName(
+ |(long)$input, $scaleValue))""".stripMargin
+ case LongType if (scaleValue < 0) =>
+s"""($methodName(
+ |$input,
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1096439062
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,275 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+
+/**
+ * Truncates a number to the specified number of digits.
+ * @param child
+ * expression to get the number to be truncated.
+ * @param scale
+ * expression to get the number of decimal places to truncate to.
+ */
+case class TruncNumber(child: Expression, scale: Expression)
+extends BaseBinaryExpression
+with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = child.dataType
+
+ /**
+ * This overridden implementation delegates the overloaded TruncNumber.trunc
methods based on
+ * data type of input values
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+(dataType, input1) match {
+ // Trunc function accepts a second parameter to truncate the input
number.
+ // If 0, it removes all the decimal values and returns only the integer.
+ // If negative, the number is truncated to the left side of the decimal
point.
+ // Value of decimal places to truncate can range from -ve to +ve
+ // 1) In the case of integral numbers, as there is no decimal part if
the value of decimal
+ // places to truncate is +ve, then we can return that input value
without any
+ // modification as there is no +ve decimal place to be truncated from an
integral number
+ // Truncate the input only if the value of decimal places to truncate is
< 0
+ case (ByteType, input: Byte) if (scaleValue < 0) =>
+TruncNumber.trunc(input.toLong, scaleValue).toByte
+ case (ShortType, input: Short) if (scaleValue < 0) =>
+TruncNumber.trunc(input.toLong, scaleValue).shortValue
+ case (IntegerType, input: Int) if (scaleValue < 0) =>
+TruncNumber.trunc(input.toLong, scaleValue).intValue
+ case (LongType, input: Long) if (scaleValue < 0) =>
+TruncNumber.trunc(input, scaleValue).longValue
+ // 2) In the case of Float, Double, and Decimal , TruncNumber.trunc
+ // will accept both -ve and +ve values
+ case (FloatType, input: Float) =>
+TruncNumber.trunc(input, scaleValue).floatValue
+ case (DoubleType, input: Double) =>
+TruncNumber.trunc(input, scaleValue).doubleValue
+ case (DecimalType.Fixed(p, s), input: Decimal) =>
+Decimal(TruncNumber.trunc(input.toJavaBigDecimal, scaleValue), p, s)
+ case _ => input1
+}
+ }
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this expression.
+ * This overridden implementation delegates the overloaded TruncNumber.trunc
methods based on
+ * data type of input values
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+val methodName =
"org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc"
+// Trunc function accepts a second parameter to truncate the input
number.
+// If 0, it removes all the decimal values and returns only the
integer.
+// If negative, the number is truncated to the left side of the
decimal point.
+// Value of decimal places to truncate can range from -ve to +ve
+// 1) In the case of integral numbers, as there is no decimal part if
the value of decimal
+// places to truncate is +ve, then we can return that input value
without any
+// modification as there is no +ve decimal place to be truncated from
an integral number
+// Truncate the input only if the value of decimal places to truncate
is < 0
+dataType match {
+ case ByteType if (scaleValue < 0) =>
+s"""(byte)($methodName(
+ |(long)$input, $scaleValue))""".stripMargin
+ case ShortType if (scaleValue < 0) =>
+s"""(short)($methodName(
+ |(long)$input, $scaleValue))""".stripMargin
+ case IntegerType if (scaleValue < 0) =>
+s"""(int)($methodName(
+ |(long)$input, $scaleValue))""".stripMargin
+ case LongType if (scaleValue < 0) =>
+s"""($methodName(
+ |$input,
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1096438984
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,275 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+
+/**
+ * Truncates a number to the specified number of digits.
+ * @param child
+ * expression to get the number to be truncated.
+ * @param scale
+ * expression to get the number of decimal places to truncate to.
+ */
+case class TruncNumber(child: Expression, scale: Expression)
+extends BaseBinaryExpression
+with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = child.dataType
+
+ /**
+ * This overridden implementation delegates the overloaded TruncNumber.trunc
methods based on
+ * data type of input values
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+(dataType, input1) match {
+ // Trunc function accepts a second parameter to truncate the input
number.
+ // If 0, it removes all the decimal values and returns only the integer.
+ // If negative, the number is truncated to the left side of the decimal
point.
+ // Value of decimal places to truncate can range from -ve to +ve
+ // 1) In the case of integral numbers, as there is no decimal part if
the value of decimal
+ // places to truncate is +ve, then we can return that input value
without any
+ // modification as there is no +ve decimal place to be truncated from an
integral number
+ // Truncate the input only if the value of decimal places to truncate is
< 0
+ case (ByteType, input: Byte) if (scaleValue < 0) =>
+TruncNumber.trunc(input.toLong, scaleValue).toByte
+ case (ShortType, input: Short) if (scaleValue < 0) =>
+TruncNumber.trunc(input.toLong, scaleValue).shortValue
+ case (IntegerType, input: Int) if (scaleValue < 0) =>
+TruncNumber.trunc(input.toLong, scaleValue).intValue
+ case (LongType, input: Long) if (scaleValue < 0) =>
+TruncNumber.trunc(input, scaleValue).longValue
+ // 2) In the case of Float, Double, and Decimal , TruncNumber.trunc
+ // will accept both -ve and +ve values
+ case (FloatType, input: Float) =>
+TruncNumber.trunc(input, scaleValue).floatValue
+ case (DoubleType, input: Double) =>
+TruncNumber.trunc(input, scaleValue).doubleValue
+ case (DecimalType.Fixed(p, s), input: Decimal) =>
+Decimal(TruncNumber.trunc(input.toJavaBigDecimal, scaleValue), p, s)
+ case _ => input1
+}
+ }
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this expression.
+ * This overridden implementation delegates the overloaded TruncNumber.trunc
methods based on
+ * data type of input values
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+val methodName =
"org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc"
+// Trunc function accepts a second parameter to truncate the input
number.
+// If 0, it removes all the decimal values and returns only the
integer.
+// If negative, the number is truncated to the left side of the
decimal point.
+// Value of decimal places to truncate can range from -ve to +ve
+// 1) In the case of integral numbers, as there is no decimal part if
the value of decimal
+// places to truncate is +ve, then we can return that input value
without any
+// modification as there is no +ve decimal place to be truncated from
an integral number
+// Truncate the input only if the value of decimal places to truncate
is < 0
+dataType match {
+ case ByteType if (scaleValue < 0) =>
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1096438825
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,275 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+
+/**
+ * Truncates a number to the specified number of digits.
+ * @param child
+ * expression to get the number to be truncated.
+ * @param scale
+ * expression to get the number of decimal places to truncate to.
+ */
+case class TruncNumber(child: Expression, scale: Expression)
+extends BaseBinaryExpression
+with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
Review Comment:
Done
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,275 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+
+/**
+ * Truncates a number to the specified number of digits.
+ * @param child
+ * expression to get the number to be truncated.
+ * @param scale
+ * expression to get the number of decimal places to truncate to.
+ */
+case class TruncNumber(child: Expression, scale: Expression)
+extends BaseBinaryExpression
+with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = child.dataType
+
+ /**
+ * This overridden implementation delegates the overloaded TruncNumber.trunc
methods based on
+ * data type of input values
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+(dataType, input1) match {
+ // Trunc function accepts a second parameter to truncate the input
number.
+ // If 0, it removes all the decimal values and returns only the integer.
+ // If negative, the number is truncated to the left side of the decimal
point.
+ // Value of decimal places to truncate can range from -ve to +ve
+ // 1) In the case of integral numbers, as there is no decimal part if
the value of decimal
+ // places to truncate is +ve, then we can return that input value
without any
+ // modification as there is no +ve decimal place to be truncated from an
integral number
+ // Truncate the input only if the value of decimal places to truncate is
< 0
+ case (ByteType, input: Byte) if (scaleValue < 0) =>
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1089830559
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,247 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
Review Comment:
Added more comments and changed condition to `if (scaleValue < 0)`
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1089830559
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,247 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1089829294
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,247 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (scaleValue <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, $scaleValue))""".stripMargin
+ case ShortType if (scaleValue <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, $scaleValue))""".stripMargin
+ case IntegerType if (scaleValue <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, $scaleValue))""".stripMargin
+ case LongType if (scaleValue <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, $scaleValue))""".stripMargin
+ case FloatType if (scaleValue <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, $scaleValue).floatValue()""".stripMargin
+ case DoubleType if (scaleValue <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, $scaleValue).doubleValue()""".stripMargin
+ case DecimalType.Fixed(p, s) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), $scaleValue), $p, $s)""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = child.dataType
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+(dataType, input1) match {
+ case (ByteType, input: Byte) if (scaleValue <= 0) =>
+TruncNumber.trunc(input.toLong, scaleValue).toByte
+ case (ShortType, input: Short) if (scaleValue <= 0) =>
+TruncNumber.trunc(input.toLong, scaleValue).shortValue
+ case (IntegerType, input: Int) if (scaleValue <= 0) =>
+TruncNumber.trunc(input.toLong, scaleValue).intValue
+ case (LongType, input: Long) if (scaleValue <= 0) =>
+TruncNumber.trunc(input, scaleValue).longValue
+ case (FloatType, input: Float) =>
+TruncNumber.trunc(input, scaleValue).floatValue
+ case (DoubleType, input: Double) =>
+TruncNumber.trunc(input, scaleValue).doubleValue
+ case (DecimalType.Fixed(p, s), input: Decimal) =>
+Decimal(TruncNumber.trunc(input.toJavaBigDecimal, scaleValue), p, s)
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers; byte, short, int, and long types.
+ */
+ def trunc(input: Long, position: Int): Long = {
+if (position >= 0) {
+ input
+} else {
+ // Here we truncate the number by the absolute value of the position.
+ // For example, if the input is 123 and the scale is -2, then the result
is 100.
+ val pow = Math.pow(10, Math.abs(position)).toLong
+ (input / pow) * pow
+}
+ }
+
+ /**
+ * To truncate double and float type
+ */
+ def trunc(input: Double, position: Int): BigDecimal = {
+trunc(jm.BigDecimal.valueOf(input), position)
+ }
+
+ /**
+ * To truncate decimal type
+ */
+ def trunc(input: jm.BigDecimal, position: Int): jm.BigDecimal = {
+if
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1089829236
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,247 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (scaleValue <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, $scaleValue))""".stripMargin
+ case ShortType if (scaleValue <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, $scaleValue))""".stripMargin
+ case IntegerType if (scaleValue <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, $scaleValue))""".stripMargin
+ case LongType if (scaleValue <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, $scaleValue))""".stripMargin
+ case FloatType if (scaleValue <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, $scaleValue).floatValue()""".stripMargin
+ case DoubleType if (scaleValue <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, $scaleValue).doubleValue()""".stripMargin
+ case DecimalType.Fixed(p, s) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), $scaleValue), $p, $s)""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = child.dataType
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1089829197
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,247 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (scaleValue <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, $scaleValue))""".stripMargin
+ case ShortType if (scaleValue <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, $scaleValue))""".stripMargin
+ case IntegerType if (scaleValue <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, $scaleValue))""".stripMargin
+ case LongType if (scaleValue <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, $scaleValue))""".stripMargin
+ case FloatType if (scaleValue <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, $scaleValue).floatValue()""".stripMargin
+ case DoubleType if (scaleValue <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, $scaleValue).doubleValue()""".stripMargin
+ case DecimalType.Fixed(p, s) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419: URL: https://github.com/apache/spark/pull/38419#discussion_r1089829142 ## sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala: ## @@ -331,6 +332,247 @@ case class RoundCeil(child: Expression, scale: Expression) copy(child = newLeft, scale = newRight) } +case class TruncNumber(child: Expression, scale: Expression) Review Comment: Done -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419: URL: https://github.com/apache/spark/pull/38419#discussion_r1089829089 ## sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala: ## @@ -331,6 +332,247 @@ case class RoundCeil(child: Expression, scale: Expression) copy(child = newLeft, scale = newRight) } +case class TruncNumber(child: Expression, scale: Expression) Review Comment: Done -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419: URL: https://github.com/apache/spark/pull/38419#discussion_r1088405813 ## sql/core/src/test/resources/sql-tests/inputs/trunc.sql: ## @@ -0,0 +1,136 @@ +-- trunc decimal Review Comment: @srielau , I've updated the code to match and added test cases in sql/core/src/test/resources/sql-tests/inputs/trunc.sql Ref : sql/core/src/test/resources/sql-tests/results/trunc.sql.out struct -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1088393009
##
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathExpressionsSuite.scala:
##
@@ -937,4 +937,135 @@ class MathExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
checkEvaluation(WidthBucket(5.35, 0.024, Double.NegativeInfinity, 5L),
null)
checkEvaluation(WidthBucket(5.35, 0.024, Double.PositiveInfinity, 5L),
null)
}
+
+test("truncate number") {
+def evaluateForEachItem(from: Int, to: Int, increment: Int)(f: (Int, Int)
=> Unit): Unit = {
+ Range(from, to, increment).zipWithIndex.foreach { case (scale, i) =>
+f(scale, i)
+ }
+}
+
+// Decimal
+val decimalInput = BigDecimal("123456789123456789.123456789")
+val truncDecimalResults = Seq(
+ "123456789123456789.123456789",
+ "123456789123456789.123456789",
+ "123456789123456789.12345678",
+ "123456789123456789.1234567",
+ "123456789123456789.123456",
+ "123456789123456789.12345",
+ "123456789123456789.1234",
+ "123456789123456789.123",
+ "123456789123456789.12",
+ "123456789123456789.1",
+ "123456789123456789",
+ "123456789123456780",
+ "123456789123456700",
+ "123456789123456000",
+ "12345678912345",
+ "12345678912340",
+ "12345678912300",
+ "12345678912000",
+ "1234567891",
+ "1234567890",
+ "1234567800",
+ "1234567000",
+ "123456",
+ "123450",
+ "123400",
+ "123000",
+ "12",
+ "10",
+ "0",
+ "0")
+
+evaluateForEachItem(10, -20, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(decimalInput, scale),
BigDecimal(truncDecimalResults(index)))
+ checkEvaluation(TruncNumber(-decimalInput, scale),
-BigDecimal(truncDecimalResults(index)))
+})
+// Double
+val negativeDouble = -12345678.1234
+val neg_DoubleResults: Seq[Double] =
+ Seq(-12345678.0, -12345670.0, -12345600.0, -12345000.0, -1234.0,
-1230.0,
+-1200.0, -1000.0, 0.0, 0.0)
+val positiveDouble = 12345678.1234
+val pos_DoubleResults: Seq[Double] =
+ Seq(12345678.0, 12345670.0, 12345600.0, 12345000.0, 1234.0,
1230.0, 1200.0,
+1000.0, 0.0, 0.0)
+
+evaluateForEachItem(0, -10, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(positiveDouble, scale),
pos_DoubleResults(index))
+ checkEvaluation(TruncNumber(negativeDouble, scale),
neg_DoubleResults(index))
+})
+
+// Float
+val negativeFloat = -12345678.123f
+val neg_floatResults: Seq[Float] =
+ Seq(-12345678f, -12345670f, -12345600f, -12345000f, -1234f,
-1230f, -1200f,
+-1000f, 0.0f, 0.0f)
+val positiveFloat = 12345678.123f
+val pos_FloatResults: Seq[Float] =
+ Seq(12345678f, 12345670f, 12345600f, 12345000f, 1234f, 1230f,
1200f, 1000f,
+0.0f, 0.0f)
+
+evaluateForEachItem(0, -10, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(positiveFloat, scale),
pos_FloatResults(index))
+ checkEvaluation(TruncNumber(negativeFloat, scale),
neg_floatResults(index))
+})
+
+// Long
+val longInput = 123456789L
+val longResults: Seq[Long] =
+ Seq(123456789L, 123456789L, 123456789L, 123456789L, 123456780L,
123456700L, 123456000L,
+12345L, 12340L, 12300L, 12000L, 1L, 0, 0)
+evaluateForEachItem(3, -11, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(longInput, scale), longResults(index))
+ checkEvaluation(TruncNumber(-longInput, scale), -longResults(index))
+})
+
+// Int
+val intInput = 123456789
+val intResults: Seq[Int] =
+ Seq(123456789, 123456789, 123456789, 123456789, 123456780, 123456700,
123456000, 12345,
+12340, 12300, 12000, 1, 0, 0)
+evaluateForEachItem(3, -11, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(intInput, scale), intResults(index))
+ checkEvaluation(TruncNumber(-intInput, scale), -intResults(index))
+})
+
+// Short
+val shortInput: Short = 32767
+val shortResults: Seq[Short] =
+ Seq(32767, 32767, 32767, 32767, 32760, 32700, 32000, 3, 0, 0)
+evaluateForEachItem(3, -6, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(shortInput, scale), shortResults(index))
+ checkEvaluation(TruncNumber(-shortInput, scale), -shortResults(index))
+})
+
+// Byte
+val byteInput: Byte = 127
+val byteResults: Seq[Byte] =
+ Seq(127, 127, 127, 127, 120, 100, 0, 0)
+evaluateForEachItem(3, -4, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(byteInput, scale), byteResults(index))
+ checkEvaluation(TruncNumber(-byteInput, scale), -byteResults(index))
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1083371035
##
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathExpressionsSuite.scala:
##
@@ -937,4 +937,135 @@ class MathExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
checkEvaluation(WidthBucket(5.35, 0.024, Double.NegativeInfinity, 5L),
null)
checkEvaluation(WidthBucket(5.35, 0.024, Double.PositiveInfinity, 5L),
null)
}
+
+test("truncate number") {
+def evaluateForEachItem(from: Int, to: Int, increment: Int)(f: (Int, Int)
=> Unit): Unit = {
+ Range(from, to, increment).zipWithIndex.foreach { case (scale, i) =>
+f(scale, i)
+ }
+}
+
+// Decimal
+val decimalInput = BigDecimal("123456789123456789.123456789")
+val truncDecimalResults = Seq(
+ "123456789123456789.123456789",
+ "123456789123456789.123456789",
+ "123456789123456789.12345678",
+ "123456789123456789.1234567",
+ "123456789123456789.123456",
+ "123456789123456789.12345",
+ "123456789123456789.1234",
+ "123456789123456789.123",
+ "123456789123456789.12",
+ "123456789123456789.1",
+ "123456789123456789",
+ "123456789123456780",
+ "123456789123456700",
+ "123456789123456000",
+ "12345678912345",
+ "12345678912340",
+ "12345678912300",
+ "12345678912000",
+ "1234567891",
+ "1234567890",
+ "1234567800",
+ "1234567000",
+ "123456",
+ "123450",
+ "123400",
+ "123000",
+ "12",
+ "10",
+ "0",
+ "0")
+
+evaluateForEachItem(10, -20, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(decimalInput, scale),
BigDecimal(truncDecimalResults(index)))
+ checkEvaluation(TruncNumber(-decimalInput, scale),
-BigDecimal(truncDecimalResults(index)))
+})
+// Double
+val negativeDouble = -12345678.1234
+val neg_DoubleResults: Seq[Double] =
+ Seq(-12345678.0, -12345670.0, -12345600.0, -12345000.0, -1234.0,
-1230.0,
+-1200.0, -1000.0, 0.0, 0.0)
+val positiveDouble = 12345678.1234
+val pos_DoubleResults: Seq[Double] =
+ Seq(12345678.0, 12345670.0, 12345600.0, 12345000.0, 1234.0,
1230.0, 1200.0,
+1000.0, 0.0, 0.0)
+
+evaluateForEachItem(0, -10, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(positiveDouble, scale),
pos_DoubleResults(index))
+ checkEvaluation(TruncNumber(negativeDouble, scale),
neg_DoubleResults(index))
+})
+
+// Float
+val negativeFloat = -12345678.123f
+val neg_floatResults: Seq[Float] =
+ Seq(-12345678f, -12345670f, -12345600f, -12345000f, -1234f,
-1230f, -1200f,
+-1000f, 0.0f, 0.0f)
+val positiveFloat = 12345678.123f
+val pos_FloatResults: Seq[Float] =
+ Seq(12345678f, 12345670f, 12345600f, 12345000f, 1234f, 1230f,
1200f, 1000f,
+0.0f, 0.0f)
+
+evaluateForEachItem(0, -10, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(positiveFloat, scale),
pos_FloatResults(index))
+ checkEvaluation(TruncNumber(negativeFloat, scale),
neg_floatResults(index))
+})
+
+// Long
+val longInput = 123456789L
+val longResults: Seq[Long] =
+ Seq(123456789L, 123456789L, 123456789L, 123456789L, 123456780L,
123456700L, 123456000L,
+12345L, 12340L, 12300L, 12000L, 1L, 0, 0)
+evaluateForEachItem(3, -11, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(longInput, scale), longResults(index))
+ checkEvaluation(TruncNumber(-longInput, scale), -longResults(index))
+})
+
+// Int
+val intInput = 123456789
+val intResults: Seq[Int] =
+ Seq(123456789, 123456789, 123456789, 123456789, 123456780, 123456700,
123456000, 12345,
+12340, 12300, 12000, 1, 0, 0)
+evaluateForEachItem(3, -11, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(intInput, scale), intResults(index))
+ checkEvaluation(TruncNumber(-intInput, scale), -intResults(index))
+})
+
+// Short
+val shortInput: Short = 32767
+val shortResults: Seq[Short] =
+ Seq(32767, 32767, 32767, 32767, 32760, 32700, 32000, 3, 0, 0)
+evaluateForEachItem(3, -6, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(shortInput, scale), shortResults(index))
+ checkEvaluation(TruncNumber(-shortInput, scale), -shortResults(index))
+})
+
+// Byte
+val byteInput: Byte = 127
+val byteResults: Seq[Byte] =
+ Seq(127, 127, 127, 127, 120, 100, 0, 0)
+evaluateForEachItem(3, -4, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(byteInput, scale), byteResults(index))
+ checkEvaluation(TruncNumber(-byteInput, scale), -byteResults(index))
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1083371035
##
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathExpressionsSuite.scala:
##
@@ -937,4 +937,135 @@ class MathExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
checkEvaluation(WidthBucket(5.35, 0.024, Double.NegativeInfinity, 5L),
null)
checkEvaluation(WidthBucket(5.35, 0.024, Double.PositiveInfinity, 5L),
null)
}
+
+test("truncate number") {
+def evaluateForEachItem(from: Int, to: Int, increment: Int)(f: (Int, Int)
=> Unit): Unit = {
+ Range(from, to, increment).zipWithIndex.foreach { case (scale, i) =>
+f(scale, i)
+ }
+}
+
+// Decimal
+val decimalInput = BigDecimal("123456789123456789.123456789")
+val truncDecimalResults = Seq(
+ "123456789123456789.123456789",
+ "123456789123456789.123456789",
+ "123456789123456789.12345678",
+ "123456789123456789.1234567",
+ "123456789123456789.123456",
+ "123456789123456789.12345",
+ "123456789123456789.1234",
+ "123456789123456789.123",
+ "123456789123456789.12",
+ "123456789123456789.1",
+ "123456789123456789",
+ "123456789123456780",
+ "123456789123456700",
+ "123456789123456000",
+ "12345678912345",
+ "12345678912340",
+ "12345678912300",
+ "12345678912000",
+ "1234567891",
+ "1234567890",
+ "1234567800",
+ "1234567000",
+ "123456",
+ "123450",
+ "123400",
+ "123000",
+ "12",
+ "10",
+ "0",
+ "0")
+
+evaluateForEachItem(10, -20, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(decimalInput, scale),
BigDecimal(truncDecimalResults(index)))
+ checkEvaluation(TruncNumber(-decimalInput, scale),
-BigDecimal(truncDecimalResults(index)))
+})
+// Double
+val negativeDouble = -12345678.1234
+val neg_DoubleResults: Seq[Double] =
+ Seq(-12345678.0, -12345670.0, -12345600.0, -12345000.0, -1234.0,
-1230.0,
+-1200.0, -1000.0, 0.0, 0.0)
+val positiveDouble = 12345678.1234
+val pos_DoubleResults: Seq[Double] =
+ Seq(12345678.0, 12345670.0, 12345600.0, 12345000.0, 1234.0,
1230.0, 1200.0,
+1000.0, 0.0, 0.0)
+
+evaluateForEachItem(0, -10, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(positiveDouble, scale),
pos_DoubleResults(index))
+ checkEvaluation(TruncNumber(negativeDouble, scale),
neg_DoubleResults(index))
+})
+
+// Float
+val negativeFloat = -12345678.123f
+val neg_floatResults: Seq[Float] =
+ Seq(-12345678f, -12345670f, -12345600f, -12345000f, -1234f,
-1230f, -1200f,
+-1000f, 0.0f, 0.0f)
+val positiveFloat = 12345678.123f
+val pos_FloatResults: Seq[Float] =
+ Seq(12345678f, 12345670f, 12345600f, 12345000f, 1234f, 1230f,
1200f, 1000f,
+0.0f, 0.0f)
+
+evaluateForEachItem(0, -10, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(positiveFloat, scale),
pos_FloatResults(index))
+ checkEvaluation(TruncNumber(negativeFloat, scale),
neg_floatResults(index))
+})
+
+// Long
+val longInput = 123456789L
+val longResults: Seq[Long] =
+ Seq(123456789L, 123456789L, 123456789L, 123456789L, 123456780L,
123456700L, 123456000L,
+12345L, 12340L, 12300L, 12000L, 1L, 0, 0)
+evaluateForEachItem(3, -11, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(longInput, scale), longResults(index))
+ checkEvaluation(TruncNumber(-longInput, scale), -longResults(index))
+})
+
+// Int
+val intInput = 123456789
+val intResults: Seq[Int] =
+ Seq(123456789, 123456789, 123456789, 123456789, 123456780, 123456700,
123456000, 12345,
+12340, 12300, 12000, 1, 0, 0)
+evaluateForEachItem(3, -11, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(intInput, scale), intResults(index))
+ checkEvaluation(TruncNumber(-intInput, scale), -intResults(index))
+})
+
+// Short
+val shortInput: Short = 32767
+val shortResults: Seq[Short] =
+ Seq(32767, 32767, 32767, 32767, 32760, 32700, 32000, 3, 0, 0)
+evaluateForEachItem(3, -6, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(shortInput, scale), shortResults(index))
+ checkEvaluation(TruncNumber(-shortInput, scale), -shortResults(index))
+})
+
+// Byte
+val byteInput: Byte = 127
+val byteResults: Seq[Byte] =
+ Seq(127, 127, 127, 127, 120, 100, 0, 0)
+evaluateForEachItem(3, -4, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(byteInput, scale), byteResults(index))
+ checkEvaluation(TruncNumber(-byteInput, scale), -byteResults(index))
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1083045053
##
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathExpressionsSuite.scala:
##
@@ -937,4 +937,135 @@ class MathExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
checkEvaluation(WidthBucket(5.35, 0.024, Double.NegativeInfinity, 5L),
null)
checkEvaluation(WidthBucket(5.35, 0.024, Double.PositiveInfinity, 5L),
null)
}
+
+test("truncate number") {
+def evaluateForEachItem(from: Int, to: Int, increment: Int)(f: (Int, Int)
=> Unit): Unit = {
+ Range(from, to, increment).zipWithIndex.foreach { case (scale, i) =>
+f(scale, i)
+ }
+}
+
+// Decimal
+val decimalInput = BigDecimal("123456789123456789.123456789")
+val truncDecimalResults = Seq(
+ "123456789123456789.123456789",
+ "123456789123456789.123456789",
+ "123456789123456789.12345678",
+ "123456789123456789.1234567",
+ "123456789123456789.123456",
+ "123456789123456789.12345",
+ "123456789123456789.1234",
+ "123456789123456789.123",
+ "123456789123456789.12",
+ "123456789123456789.1",
+ "123456789123456789",
+ "123456789123456780",
+ "123456789123456700",
+ "123456789123456000",
+ "12345678912345",
+ "12345678912340",
+ "12345678912300",
+ "12345678912000",
+ "1234567891",
+ "1234567890",
+ "1234567800",
+ "1234567000",
+ "123456",
+ "123450",
+ "123400",
+ "123000",
+ "12",
+ "10",
+ "0",
+ "0")
+
+evaluateForEachItem(10, -20, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(decimalInput, scale),
BigDecimal(truncDecimalResults(index)))
+ checkEvaluation(TruncNumber(-decimalInput, scale),
-BigDecimal(truncDecimalResults(index)))
+})
+// Double
+val negativeDouble = -12345678.1234
+val neg_DoubleResults: Seq[Double] =
+ Seq(-12345678.0, -12345670.0, -12345600.0, -12345000.0, -1234.0,
-1230.0,
+-1200.0, -1000.0, 0.0, 0.0)
+val positiveDouble = 12345678.1234
+val pos_DoubleResults: Seq[Double] =
+ Seq(12345678.0, 12345670.0, 12345600.0, 12345000.0, 1234.0,
1230.0, 1200.0,
+1000.0, 0.0, 0.0)
+
+evaluateForEachItem(0, -10, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(positiveDouble, scale),
pos_DoubleResults(index))
+ checkEvaluation(TruncNumber(negativeDouble, scale),
neg_DoubleResults(index))
+})
+
+// Float
+val negativeFloat = -12345678.123f
+val neg_floatResults: Seq[Float] =
+ Seq(-12345678f, -12345670f, -12345600f, -12345000f, -1234f,
-1230f, -1200f,
+-1000f, 0.0f, 0.0f)
+val positiveFloat = 12345678.123f
+val pos_FloatResults: Seq[Float] =
+ Seq(12345678f, 12345670f, 12345600f, 12345000f, 1234f, 1230f,
1200f, 1000f,
+0.0f, 0.0f)
+
+evaluateForEachItem(0, -10, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(positiveFloat, scale),
pos_FloatResults(index))
+ checkEvaluation(TruncNumber(negativeFloat, scale),
neg_floatResults(index))
+})
+
+// Long
+val longInput = 123456789L
+val longResults: Seq[Long] =
+ Seq(123456789L, 123456789L, 123456789L, 123456789L, 123456780L,
123456700L, 123456000L,
+12345L, 12340L, 12300L, 12000L, 1L, 0, 0)
+evaluateForEachItem(3, -11, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(longInput, scale), longResults(index))
+ checkEvaluation(TruncNumber(-longInput, scale), -longResults(index))
+})
+
+// Int
+val intInput = 123456789
+val intResults: Seq[Int] =
+ Seq(123456789, 123456789, 123456789, 123456789, 123456780, 123456700,
123456000, 12345,
+12340, 12300, 12000, 1, 0, 0)
+evaluateForEachItem(3, -11, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(intInput, scale), intResults(index))
+ checkEvaluation(TruncNumber(-intInput, scale), -intResults(index))
+})
+
+// Short
+val shortInput: Short = 32767
+val shortResults: Seq[Short] =
+ Seq(32767, 32767, 32767, 32767, 32760, 32700, 32000, 3, 0, 0)
+evaluateForEachItem(3, -6, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(shortInput, scale), shortResults(index))
+ checkEvaluation(TruncNumber(-shortInput, scale), -shortResults(index))
+})
+
+// Byte
+val byteInput: Byte = 127
+val byteResults: Seq[Byte] =
+ Seq(127, 127, 127, 127, 120, 100, 0, 0)
+evaluateForEachItem(3, -4, -1)((scale, index) => {
+ checkEvaluation(TruncNumber(byteInput, scale), byteResults(index))
+ checkEvaluation(TruncNumber(-byteInput, scale), -byteResults(index))
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419: URL: https://github.com/apache/spark/pull/38419#discussion_r1082028004 ## sql/core/src/test/resources/sql-tests/inputs/trunc.sql: ## @@ -0,0 +1,136 @@ +-- trunc decimal Review Comment: Result type vs Input type matchings are validated in these [tests](https://github.com/apache/spark/pull/38419/files#diff-d9ba3f61e2d32dabcdff1b0aeb78c40628efcea440eadcf13336a36d1a626282R1065) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1082023362
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -1432,6 +1681,53 @@ case class Logarithm(left: Expression, right: Expression)
newLeft: Expression, newRight: Expression): Logarithm = copy(left =
newLeft, right = newRight)
}
+trait BaseBinaryExpression extends BinaryExpression
+ with ExpectsInputTypes
+ with Serializable
+ with ImplicitCastInputTypes {
+ val child: Expression
+ val scale: Expression
+ override def left: Expression = child
+ override def right: Expression = scale
+ override def nullable: Boolean = true
+ override def foldable: Boolean = child.foldable
+
+ /**
+ * Expected input types from child expressions. The i-th position in the
returned seq indicates
+ * the type requirement for the i-th child.
+ *
+ * The possible values at each position are:
+ * 1. a specific data type, such as LongType or StringType.
+ * 2. a non-leaf abstract data type,
+ * such as NumericType, IntegralType, FractionalType.
+ */
+ override def inputTypes: Seq[AbstractDataType] = Seq(NumericType,
IntegerType)
+
+ // Avoid repeated evaluation since `scale` is a constant int,
+ // avoid unnecessary `child` evaluation in both codegen and non-codegen eval
+ // by checking if scaleV == null as well.
+ protected lazy val scaleV: Any = scale.eval(EmptyRow)
+
+ protected lazy val scaleValue: Int = scaleV.asInstanceOf[Int]
+
+ override def checkInputDataTypes(): TypeCheckResult = {
Review Comment:
other related udfs `ceil` , `floor` also do not work with dynamic values.
To make it consistent with those udfs, here `scale` is made constant . This
part of the code was moved from `class RoundBase ` to new trait
`BaseBinaryExpression`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1082023362
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -1432,6 +1681,53 @@ case class Logarithm(left: Expression, right: Expression)
newLeft: Expression, newRight: Expression): Logarithm = copy(left =
newLeft, right = newRight)
}
+trait BaseBinaryExpression extends BinaryExpression
+ with ExpectsInputTypes
+ with Serializable
+ with ImplicitCastInputTypes {
+ val child: Expression
+ val scale: Expression
+ override def left: Expression = child
+ override def right: Expression = scale
+ override def nullable: Boolean = true
+ override def foldable: Boolean = child.foldable
+
+ /**
+ * Expected input types from child expressions. The i-th position in the
returned seq indicates
+ * the type requirement for the i-th child.
+ *
+ * The possible values at each position are:
+ * 1. a specific data type, such as LongType or StringType.
+ * 2. a non-leaf abstract data type,
+ * such as NumericType, IntegralType, FractionalType.
+ */
+ override def inputTypes: Seq[AbstractDataType] = Seq(NumericType,
IntegerType)
+
+ // Avoid repeated evaluation since `scale` is a constant int,
+ // avoid unnecessary `child` evaluation in both codegen and non-codegen eval
+ // by checking if scaleV == null as well.
+ protected lazy val scaleV: Any = scale.eval(EmptyRow)
+
+ protected lazy val scaleValue: Int = scaleV.asInstanceOf[Int]
+
+ override def checkInputDataTypes(): TypeCheckResult = {
Review Comment:
other related udfs `ceil` , `floor` also does not work with dynamic values.
To make it consistent with those udfs, here `scale`is made constant . This
part of the code was moved from `class RoundBase ` to new trait
`BaseBinaryExpression`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080717413
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
+if (_scale >= s) {
+ s
+} else {
+ _scale
+}
+ } else {
+0
+ }
+DecimalType(p - s + newPosition, newPosition)
+ case t => t
+}
+ }
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Byte].toLong, _scale).toByte
+ case ShortType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Short].toLong, _scale).shortValue
+ case IntegerType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Int].toLong, _scale).intValue
+ case LongType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Long], _scale).longValue
+ case FloatType =>
+TruncNumber.trunc(input1.asInstanceOf[Float], _scale).floatValue
+ case DoubleType =>
+TruncNumber.trunc(input1.asInstanceOf[Double], _scale).doubleValue
+ case DecimalType.Fixed(p, s) =>
+
Decimal(TruncNumber.trunc(input1.asInstanceOf[Decimal].toJavaBigDecimal,
_scale))
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers ; byte, short, int, long types
+ */
+ def trunc(input: Long, position: Int): Long = {
+if (position >= 0) {
+ input
+} else {
+ // position is -ve, truncate the number by absolute value of position
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe,
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080715806
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
+if (_scale >= s) {
+ s
+} else {
+ _scale
+}
+ } else {
+0
+ }
+DecimalType(p - s + newPosition, newPosition)
+ case t => t
+}
+ }
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Byte].toLong, _scale).toByte
+ case ShortType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Short].toLong, _scale).shortValue
+ case IntegerType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Int].toLong, _scale).intValue
+ case LongType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Long], _scale).longValue
+ case FloatType =>
+TruncNumber.trunc(input1.asInstanceOf[Float], _scale).floatValue
+ case DoubleType =>
+TruncNumber.trunc(input1.asInstanceOf[Double], _scale).doubleValue
+ case DecimalType.Fixed(p, s) =>
+
Decimal(TruncNumber.trunc(input1.asInstanceOf[Decimal].toJavaBigDecimal,
_scale))
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers ; byte, short, int, long types
+ */
+ def trunc(input: Long, position: Int): Long = {
+if (position >= 0) {
+ input
+} else {
+ // position is -ve, truncate the number by absolute value of position
+ // eg: input 123 , scale -2 , result 100
+ val pow = Math.pow(10, Math.abs(position)).toLong
+ (input / pow) * pow
+}
+ }
+
+ /**
+ * To truncate double and float type
+ */
+ def
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080715317
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
+if (_scale >= s) {
+ s
+} else {
+ _scale
+}
+ } else {
+0
+ }
+DecimalType(p - s + newPosition, newPosition)
+ case t => t
+}
+ }
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Byte].toLong, _scale).toByte
+ case ShortType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Short].toLong, _scale).shortValue
+ case IntegerType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Int].toLong, _scale).intValue
+ case LongType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Long], _scale).longValue
+ case FloatType =>
+TruncNumber.trunc(input1.asInstanceOf[Float], _scale).floatValue
+ case DoubleType =>
+TruncNumber.trunc(input1.asInstanceOf[Double], _scale).doubleValue
+ case DecimalType.Fixed(p, s) =>
+
Decimal(TruncNumber.trunc(input1.asInstanceOf[Decimal].toJavaBigDecimal,
_scale))
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers ; byte, short, int, long types
+ */
+ def trunc(input: Long, position: Int): Long = {
+if (position >= 0) {
+ input
+} else {
+ // position is -ve, truncate the number by absolute value of position
+ // eg: input 123 , scale -2 , result 100
+ val pow = Math.pow(10, Math.abs(position)).toLong
+ (input / pow) * pow
+}
+ }
+
+ /**
+ * To truncate double and float type
+ */
+ def
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080715624
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
+if (_scale >= s) {
+ s
+} else {
+ _scale
+}
+ } else {
+0
+ }
+DecimalType(p - s + newPosition, newPosition)
+ case t => t
+}
+ }
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Byte].toLong, _scale).toByte
+ case ShortType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Short].toLong, _scale).shortValue
+ case IntegerType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Int].toLong, _scale).intValue
+ case LongType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Long], _scale).longValue
+ case FloatType =>
+TruncNumber.trunc(input1.asInstanceOf[Float], _scale).floatValue
+ case DoubleType =>
+TruncNumber.trunc(input1.asInstanceOf[Double], _scale).doubleValue
+ case DecimalType.Fixed(p, s) =>
+
Decimal(TruncNumber.trunc(input1.asInstanceOf[Decimal].toJavaBigDecimal,
_scale))
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers ; byte, short, int, long types
+ */
+ def trunc(input: Long, position: Int): Long = {
+if (position >= 0) {
+ input
+} else {
+ // position is -ve, truncate the number by absolute value of position
+ // eg: input 123 , scale -2 , result 100
+ val pow = Math.pow(10, Math.abs(position)).toLong
+ (input / pow) * pow
+}
+ }
+
+ /**
+ * To truncate double and float type
+ */
+ def
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080715469
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
+if (_scale >= s) {
+ s
+} else {
+ _scale
+}
+ } else {
+0
+ }
+DecimalType(p - s + newPosition, newPosition)
+ case t => t
+}
+ }
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Byte].toLong, _scale).toByte
+ case ShortType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Short].toLong, _scale).shortValue
+ case IntegerType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Int].toLong, _scale).intValue
+ case LongType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Long], _scale).longValue
+ case FloatType =>
+TruncNumber.trunc(input1.asInstanceOf[Float], _scale).floatValue
+ case DoubleType =>
+TruncNumber.trunc(input1.asInstanceOf[Double], _scale).doubleValue
+ case DecimalType.Fixed(p, s) =>
+
Decimal(TruncNumber.trunc(input1.asInstanceOf[Decimal].toJavaBigDecimal,
_scale))
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers ; byte, short, int, long types
+ */
+ def trunc(input: Long, position: Int): Long = {
+if (position >= 0) {
+ input
+} else {
+ // position is -ve, truncate the number by absolute value of position
+ // eg: input 123 , scale -2 , result 100
+ val pow = Math.pow(10, Math.abs(position)).toLong
+ (input / pow) * pow
+}
+ }
+
+ /**
+ * To truncate double and float type
+ */
+ def
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080715160
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
+if (_scale >= s) {
+ s
+} else {
+ _scale
+}
+ } else {
+0
+ }
+DecimalType(p - s + newPosition, newPosition)
+ case t => t
+}
+ }
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Byte].toLong, _scale).toByte
+ case ShortType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Short].toLong, _scale).shortValue
+ case IntegerType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Int].toLong, _scale).intValue
+ case LongType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Long], _scale).longValue
+ case FloatType =>
+TruncNumber.trunc(input1.asInstanceOf[Float], _scale).floatValue
+ case DoubleType =>
+TruncNumber.trunc(input1.asInstanceOf[Double], _scale).doubleValue
+ case DecimalType.Fixed(p, s) =>
+
Decimal(TruncNumber.trunc(input1.asInstanceOf[Decimal].toJavaBigDecimal,
_scale))
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers ; byte, short, int, long types
+ */
+ def trunc(input: Long, position: Int): Long = {
+if (position >= 0) {
+ input
+} else {
+ // position is -ve, truncate the number by absolute value of position
+ // eg: input 123 , scale -2 , result 100
+ val pow = Math.pow(10, Math.abs(position)).toLong
+ (input / pow) * pow
+}
+ }
+
+ /**
+ * To truncate double and float type
+ */
+ def
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080714814
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
+if (_scale >= s) {
+ s
+} else {
+ _scale
+}
+ } else {
+0
+ }
+DecimalType(p - s + newPosition, newPosition)
+ case t => t
+}
+ }
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Byte].toLong, _scale).toByte
+ case ShortType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Short].toLong, _scale).shortValue
+ case IntegerType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Int].toLong, _scale).intValue
+ case LongType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Long], _scale).longValue
+ case FloatType =>
+TruncNumber.trunc(input1.asInstanceOf[Float], _scale).floatValue
+ case DoubleType =>
+TruncNumber.trunc(input1.asInstanceOf[Double], _scale).doubleValue
+ case DecimalType.Fixed(p, s) =>
+
Decimal(TruncNumber.trunc(input1.asInstanceOf[Decimal].toJavaBigDecimal,
_scale))
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers ; byte, short, int, long types
+ */
+ def trunc(input: Long, position: Int): Long = {
+if (position >= 0) {
+ input
+} else {
+ // position is -ve, truncate the number by absolute value of position
+ // eg: input 123 , scale -2 , result 100
+ val pow = Math.pow(10, Math.abs(position)).toLong
+ (input / pow) * pow
+}
+ }
+
+ /**
+ * To truncate double and float type
+ */
+ def
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080714916
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
+if (_scale >= s) {
+ s
+} else {
+ _scale
+}
+ } else {
+0
+ }
+DecimalType(p - s + newPosition, newPosition)
+ case t => t
+}
+ }
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Byte].toLong, _scale).toByte
+ case ShortType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Short].toLong, _scale).shortValue
+ case IntegerType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Int].toLong, _scale).intValue
+ case LongType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Long], _scale).longValue
+ case FloatType =>
+TruncNumber.trunc(input1.asInstanceOf[Float], _scale).floatValue
+ case DoubleType =>
+TruncNumber.trunc(input1.asInstanceOf[Double], _scale).doubleValue
+ case DecimalType.Fixed(p, s) =>
+
Decimal(TruncNumber.trunc(input1.asInstanceOf[Decimal].toJavaBigDecimal,
_scale))
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers ; byte, short, int, long types
+ */
+ def trunc(input: Long, position: Int): Long = {
+if (position >= 0) {
+ input
+} else {
+ // position is -ve, truncate the number by absolute value of position
+ // eg: input 123 , scale -2 , result 100
+ val pow = Math.pow(10, Math.abs(position)).toLong
+ (input / pow) * pow
+}
+ }
+
+ /**
+ * To truncate double and float type
+ */
+ def
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080714699
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
+if (_scale >= s) {
+ s
+} else {
+ _scale
+}
+ } else {
+0
+ }
+DecimalType(p - s + newPosition, newPosition)
+ case t => t
+}
+ }
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Byte].toLong, _scale).toByte
+ case ShortType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Short].toLong, _scale).shortValue
+ case IntegerType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Int].toLong, _scale).intValue
+ case LongType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Long], _scale).longValue
+ case FloatType =>
+TruncNumber.trunc(input1.asInstanceOf[Float], _scale).floatValue
+ case DoubleType =>
+TruncNumber.trunc(input1.asInstanceOf[Double], _scale).doubleValue
+ case DecimalType.Fixed(p, s) =>
+
Decimal(TruncNumber.trunc(input1.asInstanceOf[Decimal].toJavaBigDecimal,
_scale))
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers ; byte, short, int, long types
+ */
+ def trunc(input: Long, position: Int): Long = {
+if (position >= 0) {
+ input
+} else {
+ // position is -ve, truncate the number by absolute value of position
+ // eg: input 123 , scale -2 , result 100
+ val pow = Math.pow(10, Math.abs(position)).toLong
+ (input / pow) * pow
+}
+ }
+
+ /**
+ * To truncate double and float type
+ */
+ def
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080713440
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
+if (_scale >= s) {
+ s
+} else {
+ _scale
+}
+ } else {
+0
+ }
+DecimalType(p - s + newPosition, newPosition)
+ case t => t
+}
+ }
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Byte].toLong, _scale).toByte
+ case ShortType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Short].toLong, _scale).shortValue
+ case IntegerType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Int].toLong, _scale).intValue
+ case LongType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Long], _scale).longValue
+ case FloatType =>
+TruncNumber.trunc(input1.asInstanceOf[Float], _scale).floatValue
+ case DoubleType =>
+TruncNumber.trunc(input1.asInstanceOf[Double], _scale).doubleValue
+ case DecimalType.Fixed(p, s) =>
+
Decimal(TruncNumber.trunc(input1.asInstanceOf[Decimal].toJavaBigDecimal,
_scale))
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers ; byte, short, int, long types
+ */
+ def trunc(input: Long, position: Int): Long = {
+if (position >= 0) {
+ input
+} else {
+ // position is -ve, truncate the number by absolute value of position
+ // eg: input 123 , scale -2 , result 100
+ val pow = Math.pow(10, Math.abs(position)).toLong
+ (input / pow) * pow
+}
+ }
+
+ /**
+ * To truncate double and float type
+ */
+ def
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080713312
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
+if (_scale >= s) {
+ s
+} else {
+ _scale
+}
+ } else {
+0
+ }
+DecimalType(p - s + newPosition, newPosition)
+ case t => t
+}
+ }
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Byte].toLong, _scale).toByte
+ case ShortType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Short].toLong, _scale).shortValue
+ case IntegerType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Int].toLong, _scale).intValue
+ case LongType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Long], _scale).longValue
+ case FloatType =>
+TruncNumber.trunc(input1.asInstanceOf[Float], _scale).floatValue
+ case DoubleType =>
+TruncNumber.trunc(input1.asInstanceOf[Double], _scale).doubleValue
+ case DecimalType.Fixed(p, s) =>
+
Decimal(TruncNumber.trunc(input1.asInstanceOf[Decimal].toJavaBigDecimal,
_scale))
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers ; byte, short, int, long types
+ */
+ def trunc(input: Long, position: Int): Long = {
+if (position >= 0) {
+ input
+} else {
+ // position is -ve, truncate the number by absolute value of position
+ // eg: input 123 , scale -2 , result 100
+ val pow = Math.pow(10, Math.abs(position)).toLong
+ (input / pow) * pow
+}
+ }
+
+ /**
+ * To truncate double and float type
+ */
+ def
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080713141
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
+if (_scale >= s) {
+ s
+} else {
+ _scale
+}
+ } else {
+0
+ }
+DecimalType(p - s + newPosition, newPosition)
+ case t => t
+}
+ }
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Byte].toLong, _scale).toByte
+ case ShortType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Short].toLong, _scale).shortValue
+ case IntegerType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Int].toLong, _scale).intValue
+ case LongType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Long], _scale).longValue
+ case FloatType =>
+TruncNumber.trunc(input1.asInstanceOf[Float], _scale).floatValue
+ case DoubleType =>
+TruncNumber.trunc(input1.asInstanceOf[Double], _scale).doubleValue
+ case DecimalType.Fixed(p, s) =>
+
Decimal(TruncNumber.trunc(input1.asInstanceOf[Decimal].toJavaBigDecimal,
_scale))
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers ; byte, short, int, long types
+ */
+ def trunc(input: Long, position: Int): Long = {
+if (position >= 0) {
+ input
+} else {
+ // position is -ve, truncate the number by absolute value of position
+ // eg: input 123 , scale -2 , result 100
+ val pow = Math.pow(10, Math.abs(position)).toLong
+ (input / pow) * pow
+}
+ }
+
+ /**
+ * To truncate double and float type
+ */
+ def
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080711856
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
+if (_scale >= s) {
+ s
+} else {
+ _scale
+}
+ } else {
+0
+ }
+DecimalType(p - s + newPosition, newPosition)
+ case t => t
+}
+ }
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Byte].toLong, _scale).toByte
+ case ShortType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Short].toLong, _scale).shortValue
+ case IntegerType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Int].toLong, _scale).intValue
+ case LongType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Long], _scale).longValue
+ case FloatType =>
+TruncNumber.trunc(input1.asInstanceOf[Float], _scale).floatValue
+ case DoubleType =>
+TruncNumber.trunc(input1.asInstanceOf[Double], _scale).doubleValue
+ case DecimalType.Fixed(p, s) =>
+
Decimal(TruncNumber.trunc(input1.asInstanceOf[Decimal].toJavaBigDecimal,
_scale))
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers ; byte, short, int, long types
+ */
+ def trunc(input: Long, position: Int): Long = {
+if (position >= 0) {
+ input
+} else {
+ // position is -ve, truncate the number by absolute value of position
+ // eg: input 123 , scale -2 , result 100
+ val pow = Math.pow(10, Math.abs(position)).toLong
+ (input / pow) * pow
+}
+ }
+
+ /**
+ * To truncate double and float type
Review Comment:
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080701463
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
+if (_scale >= s) {
+ s
+} else {
+ _scale
+}
+ } else {
+0
+ }
+DecimalType(p - s + newPosition, newPosition)
+ case t => t
+}
+ }
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Byte].toLong, _scale).toByte
+ case ShortType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Short].toLong, _scale).shortValue
+ case IntegerType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Int].toLong, _scale).intValue
+ case LongType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Long], _scale).longValue
+ case FloatType =>
+TruncNumber.trunc(input1.asInstanceOf[Float], _scale).floatValue
+ case DoubleType =>
+TruncNumber.trunc(input1.asInstanceOf[Double], _scale).doubleValue
+ case DecimalType.Fixed(p, s) =>
+
Decimal(TruncNumber.trunc(input1.asInstanceOf[Decimal].toJavaBigDecimal,
_scale))
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers ; byte, short, int, long types
+ */
+ def trunc(input: Long, position: Int): Long = {
+if (position >= 0) {
+ input
+} else {
+ // position is -ve, truncate the number by absolute value of position
+ // eg: input 123 , scale -2 , result 100
+ val pow = Math.pow(10, Math.abs(position)).toLong
+ (input / pow) * pow
+}
+ }
+
+ /**
+ * To truncate double and float type
+ */
+ def
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080690373
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
+if (_scale >= s) {
+ s
+} else {
+ _scale
+}
+ } else {
+0
+ }
+DecimalType(p - s + newPosition, newPosition)
+ case t => t
+}
+ }
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Byte].toLong, _scale).toByte
+ case ShortType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Short].toLong, _scale).shortValue
+ case IntegerType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Int].toLong, _scale).intValue
+ case LongType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Long], _scale).longValue
+ case FloatType =>
+TruncNumber.trunc(input1.asInstanceOf[Float], _scale).floatValue
+ case DoubleType =>
+TruncNumber.trunc(input1.asInstanceOf[Double], _scale).doubleValue
+ case DecimalType.Fixed(p, s) =>
+
Decimal(TruncNumber.trunc(input1.asInstanceOf[Decimal].toJavaBigDecimal,
_scale))
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers ; byte, short, int, long types
+ */
+ def trunc(input: Long, position: Int): Long = {
+if (position >= 0) {
+ input
+} else {
+ // position is -ve, truncate the number by absolute value of position
+ // eg: input 123 , scale -2 , result 100
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080690176
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -1432,6 +1695,53 @@ case class Logarithm(left: Expression, right: Expression)
newLeft: Expression, newRight: Expression): Logarithm = copy(left =
newLeft, right = newRight)
}
+trait BaseBinaryExpression extends BinaryExpression
+ with ExpectsInputTypes
+ with Serializable
+ with ImplicitCastInputTypes {
+ val child: Expression
+ val scale: Expression
+ override def left: Expression = child
+ override def right: Expression = scale
+ override def nullable: Boolean = true
+ override def foldable: Boolean = child.foldable
+
+ /**
+ * Expected input types from child expressions. The i-th position in the
returned seq indicates
+ * the type requirement for the i-th child.
+ *
+ * The possible values at each position are:
+ * 1. a specific data type, e.g. LongType, StringType.
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080690016
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
+if (_scale >= s) {
+ s
+} else {
+ _scale
+}
+ } else {
+0
+ }
+DecimalType(p - s + newPosition, newPosition)
+ case t => t
+}
+ }
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Byte].toLong, _scale).toByte
+ case ShortType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Short].toLong, _scale).shortValue
+ case IntegerType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Int].toLong, _scale).intValue
+ case LongType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Long], _scale).longValue
+ case FloatType =>
+TruncNumber.trunc(input1.asInstanceOf[Float], _scale).floatValue
+ case DoubleType =>
+TruncNumber.trunc(input1.asInstanceOf[Double], _scale).doubleValue
+ case DecimalType.Fixed(p, s) =>
+
Decimal(TruncNumber.trunc(input1.asInstanceOf[Decimal].toJavaBigDecimal,
_scale))
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers ; byte, short, int, long types
+ */
+ def trunc(input: Long, position: Int): Long = {
+if (position >= 0) {
+ input
+} else {
+ // position is -ve, truncate the number by absolute value of position
+ // eg: input 123 , scale -2 , result 100
+ val pow = Math.pow(10, Math.abs(position)).toLong
+ (input / pow) * pow
+}
+ }
+
+ /**
+ * To truncate double and float type
+ */
+ def
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080689945
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
+if (_scale >= s) {
+ s
+} else {
+ _scale
+}
+ } else {
+0
+ }
+DecimalType(p - s + newPosition, newPosition)
+ case t => t
+}
+ }
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Byte].toLong, _scale).toByte
+ case ShortType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Short].toLong, _scale).shortValue
+ case IntegerType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Int].toLong, _scale).intValue
+ case LongType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Long], _scale).longValue
+ case FloatType =>
+TruncNumber.trunc(input1.asInstanceOf[Float], _scale).floatValue
+ case DoubleType =>
+TruncNumber.trunc(input1.asInstanceOf[Double], _scale).doubleValue
+ case DecimalType.Fixed(p, s) =>
+
Decimal(TruncNumber.trunc(input1.asInstanceOf[Decimal].toJavaBigDecimal,
_scale))
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers ; byte, short, int, long types
+ */
+ def trunc(input: Long, position: Int): Long = {
+if (position >= 0) {
+ input
+} else {
+ // position is -ve, truncate the number by absolute value of position
+ // eg: input 123 , scale -2 , result 100
+ val pow = Math.pow(10, Math.abs(position)).toLong
+ (input / pow) * pow
+}
+ }
+
+ /**
+ * To truncate double and float type
+ */
+ def
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080689787
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
+if (_scale >= s) {
+ s
+} else {
+ _scale
+}
+ } else {
+0
+ }
+DecimalType(p - s + newPosition, newPosition)
+ case t => t
+}
+ }
+
+ /**
+ * Called by default [[eval]] implementation. If subclass of
BinaryExpression keep the default
+ * nullability, they can override this method to save null-check code. If we
need full control
+ * of evaluation process, we should override [[eval]].
+ */
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Byte].toLong, _scale).toByte
+ case ShortType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Short].toLong, _scale).shortValue
+ case IntegerType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Int].toLong, _scale).intValue
+ case LongType if (_scale <= 0) =>
+TruncNumber.trunc(input1.asInstanceOf[Long], _scale).longValue
+ case FloatType =>
+TruncNumber.trunc(input1.asInstanceOf[Float], _scale).floatValue
+ case DoubleType =>
+TruncNumber.trunc(input1.asInstanceOf[Double], _scale).doubleValue
+ case DecimalType.Fixed(p, s) =>
+
Decimal(TruncNumber.trunc(input1.asInstanceOf[Decimal].toJavaBigDecimal,
_scale))
+ case _ => input1
+}
+ }
+}
+
+object TruncNumber {
+ /**
+ * To truncate whole numbers ; byte, short, int, long types
Review Comment:
Done
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080689672
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}))""".stripMargin
+ case FloatType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).floatValue()""".stripMargin
+ case DoubleType if (_scale <= 0) =>
+s"""org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input, ${_scale}).doubleValue()""".stripMargin
+ case DecimalType.Fixed(_, _) =>
+s"""Decimal.apply(
+ |org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |${input}.toJavaBigDecimal(), ${_scale}))""".stripMargin
+ case _ => s"$input"
+}
+ })
+
+ /**
+ * Returns the [[DataType]] of the result of evaluating this expression. It
is invalid to query
+ * the dataType of an unresolved expression (i.e., when `resolved` == false).
+ */
+ override lazy val dataType: DataType = {
+child.dataType match {
+ case DecimalType.Fixed(p, s) =>
+val newPosition =
+ if (_scale > 0) {
Review Comment:
Done
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -331,6 +332,268 @@ case class RoundCeil(child: Expression, scale: Expression)
copy(child = newLeft, scale = newRight)
}
+case class TruncNumber(child: Expression, scale: Expression)
+ extends BaseBinaryExpression with NullIntolerant {
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): TruncNumber = copy(child = newLeft, scale =
newRight)
+
+ /**
+ * Returns Java source code that can be compiled to evaluate this
expression. The default
+ * behavior is to call the eval method of the expression. Concrete
expression implementations
+ * should override this to do actual code generation.
+ *
+ * @param ctx
+ * a [[CodegenContext]]
+ * @param ev
+ * an [[ExprCode]] with unique terms.
+ * @return
+ * an [[ExprCode]] containing the Java source code to generate the given
expression
+ */
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
+defineCodeGen(
+ ctx,
+ ev,
+ (input, _) => {
+dataType match {
+ case ByteType if (_scale <= 0) =>
+
s"""(byte)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case ShortType if (_scale <= 0) =>
+
s"""(short)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case IntegerType if (_scale <= 0) =>
+
s"""(int)(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |(long)$input, ${_scale}))""".stripMargin
+ case LongType if (_scale <= 0) =>
+s"""(org.apache.spark.sql.catalyst.expressions.TruncNumber.trunc(
+ |$input,
[GitHub] [spark] vinodkc commented on a diff in pull request #38419: [SPARK-40945][SQL] Support built-in function to truncate numbers
vinodkc commented on code in PR #38419:
URL: https://github.com/apache/spark/pull/38419#discussion_r1080689566
##
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala:
##
@@ -1432,6 +1695,53 @@ case class Logarithm(left: Expression, right: Expression)
newLeft: Expression, newRight: Expression): Logarithm = copy(left =
newLeft, right = newRight)
}
+trait BaseBinaryExpression extends BinaryExpression
+ with ExpectsInputTypes
+ with Serializable
+ with ImplicitCastInputTypes {
+ val child: Expression
+ val scale: Expression
+ override def left: Expression = child
+ override def right: Expression = scale
+ override def nullable: Boolean = true
+ override def foldable: Boolean = child.foldable
+
+ /**
+ * Expected input types from child expressions. The i-th position in the
returned seq indicates
+ * the type requirement for the i-th child.
+ *
+ * The possible values at each position are:
+ * 1. a specific data type, e.g. LongType, StringType.
+ * 2. a non-leaf abstract data type,
+ * e.g.NumericType, IntegralType, FractionalType.
+ */
+ override def inputTypes: Seq[AbstractDataType] = Seq(NumericType,
IntegerType)
+
+ // Avoid repeated evaluation since `scale` is a constant int,
+ // avoid unnecessary `child` evaluation in both codegen and non-codegen eval
+ // by checking if scaleV == null as well.
+ protected lazy val scaleV: Any = scale.eval(EmptyRow)
+
+ protected lazy val _scale: Int = scaleV.asInstanceOf[Int]
Review Comment:
`scaleV` is used in some other udfs `ceil`, `floor`, so just renamed
`_scale` to `scaleValue`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
