[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yjshen commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35615752
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,508 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
--- End diff --
cast expression will be inserted during physical operator
transformation,for example `Average`, in which case its possible we add a `cast
to self` expression.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yjshen commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35615521
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +420,506 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val eval = child.gen(ctx)
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+eval.code +
+ castCode(ctx, eval.primitive, eval.isNull, ev.primitive, ev.isNull,
dataType, nullSafeCast)
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+ }
+
+ // Since we need to cast child expressions recursively inside
ComplexTypes, such as Map's
+ // Key and Value, Struct's field, we need to name out all the variable
names involved in a cast.
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
+
+ private[this] def castToDateCode(
+ from: DataType,
+ ctx: CodeGenContext): CastFunction = from match {
+case StringType =>
+ val intOpt = ctx.freshName("intOpt")
+ (c, evPrim, evNull) => s"""
+scala.Option $intOpt =
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c);
+if ($intOpt.isDefined()) {
+ $evPrim = ((Integer) $intOpt.get()).intValue();
+} else {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, decimalType: DecimalType,
+ evPrim: String, evNull: String): String = {
+decimalType match {
+ case Decimal
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35614341
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +420,506 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val eval = child.gen(ctx)
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+eval.code +
+ castCode(ctx, eval.primitive, eval.isNull, ev.primitive, ev.isNull,
dataType, nullSafeCast)
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+ }
+
+ // Since we need to cast child expressions recursively inside
ComplexTypes, such as Map's
+ // Key and Value, Struct's field, we need to name out all the variable
names involved in a cast.
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
+
+ private[this] def castToDateCode(
+ from: DataType,
+ ctx: CodeGenContext): CastFunction = from match {
+case StringType =>
+ val intOpt = ctx.freshName("intOpt")
+ (c, evPrim, evNull) => s"""
+scala.Option $intOpt =
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c);
+if ($intOpt.isDefined()) {
+ $evPrim = ((Integer) $intOpt.get()).intValue();
+} else {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, decimalType: DecimalType,
+ evPrim: String, evNull: String): String = {
+decimalType match {
+ case Deci
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user asfgit closed the pull request at: https://github.com/apache/spark/pull/7365 --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user davies commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123996924 Merging into master, thanks! --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user davies commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123996872 LGTM --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123979558 [Test build #1183 has finished](https://amplab.cs.berkeley.edu/jenkins/job/NewSparkPullRequestBuilder/1183/console) for PR 7365 at commit [`ef6e8b5`](https://github.com/apache/spark/commit/ef6e8b51ecfc1f286f402b9fbaa50e3e0c3f1749). * This patch **passes all tests**. * This patch merges cleanly. * This patch adds no public classes. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123957683 [Test build #1183 has started](https://amplab.cs.berkeley.edu/jenkins/job/NewSparkPullRequestBuilder/1183/consoleFull) for PR 7365 at commit [`ef6e8b5`](https://github.com/apache/spark/commit/ef6e8b51ecfc1f286f402b9fbaa50e3e0c3f1749). --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yjshen commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123925541 build error caused by :`Error: Invalid or corrupt jarfile build/sbt-launch-0.13.7.jar` please trigger the test again? --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123648000 [Test build #38067 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/38067/consoleFull) for PR 7365 at commit [`ef6e8b5`](https://github.com/apache/spark/commit/ef6e8b51ecfc1f286f402b9fbaa50e3e0c3f1749). --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123648077 Merged build finished. Test FAILed. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123648070 [Test build #38067 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/38067/console) for PR 7365 at commit [`ef6e8b5`](https://github.com/apache/spark/commit/ef6e8b51ecfc1f286f402b9fbaa50e3e0c3f1749). * This patch **fails to build**. * This patch merges cleanly. * This patch adds no public classes. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123647122 Merged build started. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123647096 Merged build triggered. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yjshen commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35196231
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,515 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val eval = child.gen(ctx)
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+eval.code +
+ castCode(ctx, eval.primitive, eval.isNull, ev.primitive, ev.isNull,
dataType, nullSafeCast)
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+ }
+
+ // Since we need to cast child expressions recursively inside
ComplexTypes, such as Map's
+ // Key and Value, Struct's field, we need to name out all the variable
names involved in a cast.
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
- case (BinaryType, StringType) =>
-defineCodeGen (ctx, ev, c =>
- s"UTF8String.fromBytes($c)")
+ private[this] def castToDateCode(
+ from: DataType,
+ ctx: CodeGenContext): CastFunction = from match {
+case StringType =>
+ val intOpt = ctx.freshName("intOpt")
+ (c, evPrim, evNull) => s"""
+scala.Option $intOpt =
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c);
+if ($intOpt.isDefined()) {
+ $evPrim = ((Integer) $intOpt.get()).intValue();
+} else {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, dec
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yjshen commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35194617
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,515 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val eval = child.gen(ctx)
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+eval.code +
+ castCode(ctx, eval.primitive, eval.isNull, ev.primitive, ev.isNull,
dataType, nullSafeCast)
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+ }
+
+ // Since we need to cast child expressions recursively inside
ComplexTypes, such as Map's
+ // Key and Value, Struct's field, we need to name out all the variable
names involved in a cast.
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
- case (BinaryType, StringType) =>
-defineCodeGen (ctx, ev, c =>
- s"UTF8String.fromBytes($c)")
+ private[this] def castToDateCode(
+ from: DataType,
+ ctx: CodeGenContext): CastFunction = from match {
+case StringType =>
+ val intOpt = ctx.freshName("intOpt")
+ (c, evPrim, evNull) => s"""
+scala.Option $intOpt =
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c);
+if ($intOpt.isDefined()) {
+ $evPrim = ((Integer) $intOpt.get()).intValue();
+} else {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, dec
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123600650 [Test build #1153 has finished](https://amplab.cs.berkeley.edu/jenkins/job/NewSparkPullRequestBuilder/1153/console) for PR 7365 at commit [`eaece18`](https://github.com/apache/spark/commit/eaece189f577d9ff9afe111dba7473a58a55baec). * This patch **passes all tests**. * This patch merges cleanly. * This patch adds no public classes. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123573084 [Test build #1153 has started](https://amplab.cs.berkeley.edu/jenkins/job/NewSparkPullRequestBuilder/1153/consoleFull) for PR 7365 at commit [`eaece18`](https://github.com/apache/spark/commit/eaece189f577d9ff9afe111dba7473a58a55baec). --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user davies commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35183762
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,515 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val eval = child.gen(ctx)
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+eval.code +
+ castCode(ctx, eval.primitive, eval.isNull, ev.primitive, ev.isNull,
dataType, nullSafeCast)
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+ }
+
+ // Since we need to cast child expressions recursively inside
ComplexTypes, such as Map's
+ // Key and Value, Struct's field, we need to name out all the variable
names involved in a cast.
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
- case (BinaryType, StringType) =>
-defineCodeGen (ctx, ev, c =>
- s"UTF8String.fromBytes($c)")
+ private[this] def castToDateCode(
+ from: DataType,
+ ctx: CodeGenContext): CastFunction = from match {
+case StringType =>
+ val intOpt = ctx.freshName("intOpt")
+ (c, evPrim, evNull) => s"""
+scala.Option $intOpt =
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c);
+if ($intOpt.isDefined()) {
+ $evPrim = ((Integer) $intOpt.get()).intValue();
+} else {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, dec
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user davies commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35183717
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,515 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val eval = child.gen(ctx)
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+eval.code +
+ castCode(ctx, eval.primitive, eval.isNull, ev.primitive, ev.isNull,
dataType, nullSafeCast)
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+ }
+
+ // Since we need to cast child expressions recursively inside
ComplexTypes, such as Map's
+ // Key and Value, Struct's field, we need to name out all the variable
names involved in a cast.
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
- case (BinaryType, StringType) =>
-defineCodeGen (ctx, ev, c =>
- s"UTF8String.fromBytes($c)")
+ private[this] def castToDateCode(
+ from: DataType,
+ ctx: CodeGenContext): CastFunction = from match {
+case StringType =>
+ val intOpt = ctx.freshName("intOpt")
+ (c, evPrim, evNull) => s"""
+scala.Option $intOpt =
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c);
+if ($intOpt.isDefined()) {
+ $evPrim = ((Integer) $intOpt.get()).intValue();
+} else {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, dec
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user davies commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35183348
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,515 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val eval = child.gen(ctx)
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+eval.code +
+ castCode(ctx, eval.primitive, eval.isNull, ev.primitive, ev.isNull,
dataType, nullSafeCast)
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+ }
+
+ // Since we need to cast child expressions recursively inside
ComplexTypes, such as Map's
+ // Key and Value, Struct's field, we need to name out all the variable
names involved in a cast.
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
- case (BinaryType, StringType) =>
-defineCodeGen (ctx, ev, c =>
- s"UTF8String.fromBytes($c)")
+ private[this] def castToDateCode(
+ from: DataType,
+ ctx: CodeGenContext): CastFunction = from match {
+case StringType =>
+ val intOpt = ctx.freshName("intOpt")
+ (c, evPrim, evNull) => s"""
+scala.Option $intOpt =
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c);
+if ($intOpt.isDefined()) {
+ $evPrim = ((Integer) $intOpt.get()).intValue();
+} else {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, dec
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user davies commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35183319
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,515 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val eval = child.gen(ctx)
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+eval.code +
+ castCode(ctx, eval.primitive, eval.isNull, ev.primitive, ev.isNull,
dataType, nullSafeCast)
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+ }
+
+ // Since we need to cast child expressions recursively inside
ComplexTypes, such as Map's
+ // Key and Value, Struct's field, we need to name out all the variable
names involved in a cast.
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
- case (BinaryType, StringType) =>
-defineCodeGen (ctx, ev, c =>
- s"UTF8String.fromBytes($c)")
+ private[this] def castToDateCode(
+ from: DataType,
+ ctx: CodeGenContext): CastFunction = from match {
+case StringType =>
+ val intOpt = ctx.freshName("intOpt")
+ (c, evPrim, evNull) => s"""
+scala.Option $intOpt =
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c);
+if ($intOpt.isDefined()) {
+ $evPrim = ((Integer) $intOpt.get()).intValue();
+} else {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, dec
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user davies commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35183240
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,518 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
+ }
+
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
+
+ private[this] def castToDateCode(
+ from: DataType,
+ ctx: CodeGenContext): CastFunction = from match {
+case StringType =>
+ val intOpt = ctx.freshName("intOpt")
+ (c, evPrim, evNull) => s"""
+scala.Option $intOpt =
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c);
+if ($intOpt.isDefined()) {
+ $evPrim = ((Integer) $intOpt.get()).intValue();
+} else {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, decimalType: DecimalType,
+ evPrim: String, evNull: String): String = {
+decimalType match {
+ case DecimalType.Unlimited =>
+s"$evPrim = $d;"
+ case Decim
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35182581
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,518 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
+ }
+
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
+
+ private[this] def castToDateCode(
+ from: DataType,
+ ctx: CodeGenContext): CastFunction = from match {
+case StringType =>
+ val intOpt = ctx.freshName("intOpt")
+ (c, evPrim, evNull) => s"""
+scala.Option $intOpt =
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c);
+if ($intOpt.isDefined()) {
+ $evPrim = ((Integer) $intOpt.get()).intValue();
+} else {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, decimalType: DecimalType,
+ evPrim: String, evNull: String): String = {
+decimalType match {
+ case DecimalType.Unlimited =>
+s"$evPrim = $d;"
+ case De
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user davies commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35181654
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,518 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
+ }
+
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
+
+ private[this] def castToDateCode(
+ from: DataType,
+ ctx: CodeGenContext): CastFunction = from match {
+case StringType =>
+ val intOpt = ctx.freshName("intOpt")
+ (c, evPrim, evNull) => s"""
+scala.Option $intOpt =
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c);
+if ($intOpt.isDefined()) {
+ $evPrim = ((Integer) $intOpt.get()).intValue();
+} else {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, decimalType: DecimalType,
+ evPrim: String, evNull: String): String = {
+decimalType match {
+ case DecimalType.Unlimited =>
+s"$evPrim = $d;"
+ case Decim
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user davies commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35181626
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,518 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
+ }
+
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
+
+ private[this] def castToDateCode(
+ from: DataType,
+ ctx: CodeGenContext): CastFunction = from match {
+case StringType =>
+ val intOpt = ctx.freshName("intOpt")
+ (c, evPrim, evNull) => s"""
+scala.Option $intOpt =
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c);
+if ($intOpt.isDefined()) {
+ $evPrim = ((Integer) $intOpt.get()).intValue();
+} else {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, decimalType: DecimalType,
+ evPrim: String, evNull: String): String = {
+decimalType match {
+ case DecimalType.Unlimited =>
+s"$evPrim = $d;"
+ case Decim
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yjshen commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123537451 @davies , could you review this again? thanks! --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yjshen commented on the pull request:
https://github.com/apache/spark/pull/7365#issuecomment-123532881
It's a `run-test.py` error:
```
File "./dev/run-tests.py", line 66, in
__main__.identify_changed_files_from_git_commits
Failed example:
[x.name for x in determine_modules_for_files(
identify_changed_files_from_git_commits("fc0a1475ef", target_ref="5da21f07"))]
Exception raised:
```
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yjshen commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35175062
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,518 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
+ }
+
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
+
+ private[this] def castToDateCode(
+ from: DataType,
+ ctx: CodeGenContext): CastFunction = from match {
+case StringType =>
+ val intOpt = ctx.freshName("intOpt")
+ (c, evPrim, evNull) => s"""
+scala.Option $intOpt =
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c);
+if ($intOpt.isDefined()) {
+ $evPrim = ((Integer) $intOpt.get()).intValue();
+} else {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, decimalType: DecimalType,
+ evPrim: String, evNull: String): String = {
+decimalType match {
+ case DecimalType.Unlimited =>
+s"$evPrim = $d;"
+ case Decim
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123531499 [Test build #38013 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/38013/console) for PR 7365 at commit [`eaece18`](https://github.com/apache/spark/commit/eaece189f577d9ff9afe111dba7473a58a55baec). * This patch **fails some tests**. * This patch merges cleanly. * This patch adds no public classes. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123531509 Merged build finished. Test FAILed. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123531395 Merged build started. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123531347 Merged build triggered. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123531494 [Test build #38013 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/38013/consoleFull) for PR 7365 at commit [`eaece18`](https://github.com/apache/spark/commit/eaece189f577d9ff9afe111dba7473a58a55baec). --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yjshen commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35129721
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,518 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
--- End diff --
I think it should be removed, this was from the origin implementation when
only limited cast codegen exists, since I have all the cast with interpreted
eval, I think just remove this is reasonable
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user davies commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35128998
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,518 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
+ }
+
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
+
+ private[this] def castToDateCode(
+ from: DataType,
+ ctx: CodeGenContext): CastFunction = from match {
+case StringType =>
+ val intOpt = ctx.freshName("intOpt")
+ (c, evPrim, evNull) => s"""
+scala.Option $intOpt =
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c);
+if ($intOpt.isDefined()) {
+ $evPrim = ((Integer) $intOpt.get()).intValue();
+} else {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, decimalType: DecimalType,
+ evPrim: String, evNull: String): String = {
+decimalType match {
+ case DecimalType.Unlimited =>
+s"$evPrim = $d;"
+ case Decim
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user davies commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35128855
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,518 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
+ }
+
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
+
+ private[this] def castToDateCode(
+ from: DataType,
+ ctx: CodeGenContext): CastFunction = from match {
+case StringType =>
+ val intOpt = ctx.freshName("intOpt")
+ (c, evPrim, evNull) => s"""
+scala.Option $intOpt =
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c);
+if ($intOpt.isDefined()) {
+ $evPrim = ((Integer) $intOpt.get()).intValue();
+} else {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, decimalType: DecimalType,
+ evPrim: String, evNull: String): String = {
+decimalType match {
+ case DecimalType.Unlimited =>
+s"$evPrim = $d;"
+ case Decim
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user davies commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35128409
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,518 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
+ }
+
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
--- End diff --
It's used by `castStructCode`, never mind the above comments.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yjshen commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35127758
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,518 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
+ }
+
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
--- End diff --
castCode is intendedly take all these parameters because we have to do cast
for `ComplexType` as well, so we need call castCode recursively in these casts.
I think I should add a comment in code here.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user davies commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35127337
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,518 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
--- End diff --
What's other? UDT?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user davies commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35127256
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,518 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
+ }
+
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
--- End diff --
This function is short, I'd like to inline it into `genCode()`, it will be
easier to understand.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user davies commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r35127117
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,518 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from, ctx)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from, ctx)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
+ }
+
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
--- End diff --
Nit: we can use `result` as GeneratedExpressionCode for resultPrim,
resultNull, child for childPrim and childNull. resultType is `this.dataType`.
Right now, it has two many parameters, hard to read.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yjshen commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-123396367 More comments on this? --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122773507 Merged build finished. Test PASSed. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122773454 [Test build #37800 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/37800/console) for PR 7365 at commit [`fd7eba4`](https://github.com/apache/spark/commit/fd7eba4d812ff43307ae766953a33afdfa6f8038). * This patch **passes all tests**. * This patch merges cleanly. * This patch adds no public classes. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122760458 [Test build #37800 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/37800/consoleFull) for PR 7365 at commit [`fd7eba4`](https://github.com/apache/spark/commit/fd7eba4d812ff43307ae766953a33afdfa6f8038). --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122758917 Merged build triggered. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122758958 Merged build started. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yjshen commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r34967631
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,508 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
+ }
+
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
+
+ private[this] def castToDateCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"""
+try {
+ $evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDateTE($c);
+} catch (java.lang.IllegalArgumentException e) {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, decimalType: DecimalType,
+ evPrim: String, evNull: String): String = {
+decimalType match {
+ case DecimalType.Unlimited =>
+s"$evPrim = $d;"
+ case DecimalType.Fixed(precision, scale) =>
+s"""
+ if ($d.changePrecision($precision, $scale)) {
+$evPrim = $d;
+ } else {
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user rxin commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r34967532
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,508 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
+ }
+
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
+
+ private[this] def castToDateCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"""
+try {
+ $evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDateTE($c);
+} catch (java.lang.IllegalArgumentException e) {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, decimalType: DecimalType,
+ evPrim: String, evNull: String): String = {
+decimalType match {
+ case DecimalType.Unlimited =>
+s"$evPrim = $d;"
+ case DecimalType.Fixed(precision, scale) =>
+s"""
+ if ($d.changePrecision($precision, $scale)) {
+$evPrim = $d;
+ } else {
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user rxin commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r34967490
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,508 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
+ }
+
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
+
+ private[this] def castToDateCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"""
+try {
+ $evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDateTE($c);
+} catch (java.lang.IllegalArgumentException e) {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, decimalType: DecimalType,
+ evPrim: String, evNull: String): String = {
+decimalType match {
+ case DecimalType.Unlimited =>
+s"$evPrim = $d;"
+ case DecimalType.Fixed(precision, scale) =>
+s"""
+ if ($d.changePrecision($precision, $scale)) {
+$evPrim = $d;
+ } else {
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yjshen commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r34967457
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,508 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
+ }
+
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
+
+ private[this] def castToDateCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"""
+try {
+ $evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDateTE($c);
+} catch (java.lang.IllegalArgumentException e) {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, decimalType: DecimalType,
+ evPrim: String, evNull: String): String = {
+decimalType match {
+ case DecimalType.Unlimited =>
+s"$evPrim = $d;"
+ case DecimalType.Fixed(precision, scale) =>
+s"""
+ if ($d.changePrecision($precision, $scale)) {
+$evPrim = $d;
+ } else {
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user rxin commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r34967370
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,508 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
+ }
+
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
+
+ private[this] def castToDateCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"""
+try {
+ $evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDateTE($c);
+} catch (java.lang.IllegalArgumentException e) {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, decimalType: DecimalType,
+ evPrim: String, evNull: String): String = {
+decimalType match {
+ case DecimalType.Unlimited =>
+s"$evPrim = $d;"
+ case DecimalType.Fixed(precision, scale) =>
+s"""
+ if ($d.changePrecision($precision, $scale)) {
+$evPrim = $d;
+ } else {
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yjshen commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r34967292
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,508 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
--- End diff --
I guess its reasonable to remove unnecessary cast introduced in
`DateExpressionSuite` and remove the interpreted version as well?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yjshen commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r34967174
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,508 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
+ }
+
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
+
+ private[this] def castToDateCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"""
+try {
+ $evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDateTE($c);
+} catch (java.lang.IllegalArgumentException e) {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, decimalType: DecimalType,
+ evPrim: String, evNull: String): String = {
+decimalType match {
+ case DecimalType.Unlimited =>
+s"$evPrim = $d;"
+ case DecimalType.Fixed(precision, scale) =>
+s"""
+ if ($d.changePrecision($precision, $scale)) {
+$evPrim = $d;
+ } else {
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r34967097
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,508 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
--- End diff --
According to the fact that we also do cast in interpreted version when
`from == to`, let's do this in codegen version too.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yjshen commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r34966990
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,508 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
+case StringType => castToStringCode(from, ctx)
+case BinaryType => castToBinaryCode(from)
+case DateType => castToDateCode(from)
+case decimal: DecimalType => castToDecimalCode(from, decimal)
+case TimestampType => castToTimestampCode(from)
+case IntervalType => castToIntervalCode(from)
+case BooleanType => castToBooleanCode(from)
+case ByteType => castToByteCode(from)
+case ShortType => castToShortCode(from)
+case IntegerType => castToIntCode(from)
+case FloatType => castToFloatCode(from)
+case LongType => castToLongCode(from)
+case DoubleType => castToDoubleCode(from)
+
+case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType],
array, ctx)
+case map: MapType => castMapCode(from.asInstanceOf[MapType], map, ctx)
+case struct: StructType =>
castStructCode(from.asInstanceOf[StructType], struct, ctx)
+case other => null
+ }
+
+ private[this] def castCode(ctx: CodeGenContext, childPrim: String,
childNull: String,
+resultPrim: String, resultNull: String, resultType: DataType, cast:
CastFunction): String = {
+s"""
+ boolean $resultNull = $childNull;
+ ${ctx.javaType(resultType)} $resultPrim =
${ctx.defaultValue(resultType)};
+ if (!${childNull}) {
+${cast(childPrim, resultPrim, resultNull)}
+ }
+"""
+ }
+
+ private[this] def castToStringCode(from: DataType, ctx: CodeGenContext):
CastFunction = {
+from match {
+ case BinaryType =>
+(c, evPrim, evNull) => s"$evPrim = UTF8String.fromBytes($c);"
+ case DateType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.dateToString($c));"""
+ case TimestampType =>
+(c, evPrim, evNull) => s"""$evPrim = UTF8String.fromString(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($c));"""
+ case _ =>
+(c, evPrim, evNull) => s"$evPrim =
UTF8String.fromString(String.valueOf($c));"
+}
+ }
+
+ private[this] def castToBinaryCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"$evPrim = $c.getBytes();"
+ }
+
+ private[this] def castToDateCode(from: DataType): CastFunction = from
match {
+case StringType =>
+ (c, evPrim, evNull) => s"""
+try {
+ $evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDateTE($c);
+} catch (java.lang.IllegalArgumentException e) {
+ $evNull = true;
+}
+ """
+case TimestampType =>
+ (c, evPrim, evNull) =>
+s"$evPrim =
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToDays($c / 1000L);";
+case _ =>
+ (c, evPrim, evNull) => s"$evNull = true;"
+ }
+
+ private[this] def changePrecision(d: String, decimalType: DecimalType,
+ evPrim: String, evNull: String): String = {
+decimalType match {
+ case DecimalType.Unlimited =>
+s"$evPrim = $d;"
+ case DecimalType.Fixed(precision, scale) =>
+s"""
+ if ($d.changePrecision($precision, $scale)) {
+$evPrim = $d;
+ } else {
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yjshen commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r34966970
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,508 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
--- End diff --
I get this, `checkEvaluation` in testSuite doesn't trigger optimiser at
all, does it? So I think I should remove unnecessary cast introduced in
`DateExpressionSuite`, what do you think?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yjshen commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r34966877
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -418,51 +418,508 @@ case class Cast(child: Expression, dataType:
DataType)
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ val eval = child.gen(ctx)
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
+}
+ }
+
+ // three function arguments are: child.primitive, result.primitive and
result.isNull
+ // it returns the code snippets to be put in null safe evaluation region
+ private[this] type CastFunction = (String, String, String) => String
+
+ private[this] def nullSafeCastFunction(
+ from: DataType,
+ to: DataType,
+ ctx: CodeGenContext): CastFunction = to match {
+
+case _ if from == NullType => (c, evPrim, evNull) => s"$evNull = true;"
+case _ if to == from => (c, evPrim, evNull) => s"$evPrim = $c;"
--- End diff --
Ah, let me check why optimizer didn't work for this
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122698162 [Test build #37778 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/37778/console) for PR 7365 at commit [`d01eada`](https://github.com/apache/spark/commit/d01eada620a51c7db7d422d1739481c43d526647). * This patch **fails Spark unit tests**. * This patch merges cleanly. * This patch adds no public classes. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122698174 Merged build finished. Test FAILed. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122694100 [Test build #37778 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/37778/consoleFull) for PR 7365 at commit [`d01eada`](https://github.com/apache/spark/commit/d01eada620a51c7db7d422d1739481c43d526647). --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122692181 Merged build triggered. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122672288 Merged build finished. Test FAILed. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122672279 [Test build #37773 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/37773/console) for PR 7365 at commit [`1683876`](https://github.com/apache/spark/commit/168387624363b1a493f3573dbf3cd9922b7a0ae2). * This patch **fails Spark unit tests**. * This patch merges cleanly. * This patch adds no public classes. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122664518 [Test build #37773 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/37773/consoleFull) for PR 7365 at commit [`1683876`](https://github.com/apache/spark/commit/168387624363b1a493f3573dbf3cd9922b7a0ae2). --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122663901 Merged build triggered. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122663945 Merged build started. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122633043 Merged build finished. Test FAILed. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122633040 [Test build #37752 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/37752/console) for PR 7365 at commit [`5de0a95`](https://github.com/apache/spark/commit/5de0a951371a23cd198a6cf69b9fcb238f792f0e). * This patch **fails Spark unit tests**. * This patch merges cleanly. * This patch adds no public classes. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122627510 [Test build #37752 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/37752/consoleFull) for PR 7365 at commit [`5de0a95`](https://github.com/apache/spark/commit/5de0a951371a23cd198a6cf69b9fcb238f792f0e). --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122626063 Merged build started. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122626012 Merged build triggered. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user rxin commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122625568 Jenkins, test this please. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user rxin commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122625554 Jenkins, ok to test. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user rxin commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122618912 I'm pretty sure Jenkins hates you. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user rxin commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122618881 Jenkins, ok to test. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user yijieshen commented on the pull request: https://github.com/apache/spark/pull/7365#issuecomment-122618131 @rxin , Since #7488 is merged, can we trigger the test now? --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-8935][SQL] Implement code generation fo...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/7365#discussion_r34948324
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
---
@@ -422,47 +422,487 @@ case class Cast(child: Expression, dataType:
DataType) extends UnaryExpression w
protected override def nullSafeEval(input: Any): Any = cast(input)
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
-// TODO: Add support for more data types.
-(child.dataType, dataType) match {
+val eval = child.gen(ctx)
+val nullSafeCast = nullSafeCastFunction(child.dataType, dataType, ctx)
+if (nullSafeCast != null) {
+ eval.code +
+castCode(ctx, eval.primitive, eval.isNull, ev.primitive,
ev.isNull, dataType, nullSafeCast)
+} else {
+ super.genCode(ctx, ev)
--- End diff --
ah that makes sense
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
