Repository: spark
Updated Branches:
refs/heads/master 06b6a074f -> 6d94bf6ac
[SPARK-8174] [SPARK-8175] [SQL] function unix_timestamp, from_unixtime
unix_timestamp(): long
Gets current Unix timestamp in seconds.
unix_timestamp(string|date): long
Converts time string in format yyyy-MM-dd HH:mm:ss to Unix timestamp (in
seconds), using the default timezone and the default locale, return null if
fail: unix_timestamp('2009-03-20 11:30:01') = 1237573801
unix_timestamp(string date, string pattern): long
Convert time string with given pattern (see
[http://docs.oracle.com/javase/tutorial/i18n/format/simpleDateFormat.html]) to
Unix time stamp (in seconds), return null if fail: unix_timestamp('2009-03-20',
'yyyy-MM-dd') = 1237532400.
from_unixtime(bigint unixtime[, string format]): string
Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a
string representing the timestamp of that moment in the current system time
zone in the format of "1970-01-01 00:00:00".
Jira:
https://issues.apache.org/jira/browse/SPARK-8174
https://issues.apache.org/jira/browse/SPARK-8175
Author: Daoyuan Wang <daoyuan.w...@intel.com>
Closes #7644 from adrian-wang/udfunixtime and squashes the following commits:
2fe20c4 [Daoyuan Wang] util.Date
ea2ec16 [Daoyuan Wang] use util.Date for better performance
a2cf929 [Daoyuan Wang] doc return null instead of 0
f6f070a [Daoyuan Wang] address comments from davies
6a4cbb3 [Daoyuan Wang] temp
56ded53 [Daoyuan Wang] rebase and address comments
14a8b37 [Daoyuan Wang] function unix_timestamp, from_unixtime
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6d94bf6a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6d94bf6a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6d94bf6a
Branch: refs/heads/master
Commit: 6d94bf6ac10ac851636c62439f8f2737f3526a2a
Parents: 06b6a07
Author: Daoyuan Wang <daoyuan.w...@intel.com>
Authored: Thu Jul 30 11:13:15 2015 -0700
Committer: Davies Liu <davies....@gmail.com>
Committed: Thu Jul 30 11:14:09 2015 -0700
----------------------------------------------------------------------
.../catalyst/analysis/FunctionRegistry.scala | 2 +
.../expressions/datetimeFunctions.scala | 219 ++++++++++++++++++-
.../expressions/DateExpressionsSuite.scala | 59 ++++-
.../scala/org/apache/spark/sql/functions.scala | 42 ++++
.../apache/spark/sql/DateFunctionsSuite.scala | 56 +++++
5 files changed, 374 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/6d94bf6a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index 378df4f..d663f12 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -211,6 +211,7 @@ object FunctionRegistry {
expression[DayOfMonth]("day"),
expression[DayOfYear]("dayofyear"),
expression[DayOfMonth]("dayofmonth"),
+ expression[FromUnixTime]("from_unixtime"),
expression[Hour]("hour"),
expression[LastDay]("last_day"),
expression[Minute]("minute"),
@@ -218,6 +219,7 @@ object FunctionRegistry {
expression[NextDay]("next_day"),
expression[Quarter]("quarter"),
expression[Second]("second"),
+ expression[UnixTimestamp]("unix_timestamp"),
expression[WeekOfYear]("weekofyear"),
expression[Year]("year"),
http://git-wip-us.apache.org/repos/asf/spark/blob/6d94bf6a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeFunctions.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeFunctions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeFunctions.scala
index efecb77..a5e6249 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeFunctions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeFunctions.scala
@@ -17,7 +17,6 @@
package org.apache.spark.sql.catalyst.expressions
-import java.sql.Date
import java.text.SimpleDateFormat
import java.util.{Calendar, TimeZone}
@@ -28,6 +27,8 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
+import scala.util.Try
+
/**
* Returns the current date at the start of query evaluation.
* All calls of current_date within the same query return the same value.
@@ -236,14 +237,14 @@ case class DateFormatClass(left: Expression, right:
Expression) extends BinaryEx
override protected def nullSafeEval(timestamp: Any, format: Any): Any = {
val sdf = new SimpleDateFormat(format.toString)
- UTF8String.fromString(sdf.format(new Date(timestamp.asInstanceOf[Long] /
1000)))
+ UTF8String.fromString(sdf.format(new
java.util.Date(timestamp.asInstanceOf[Long] / 1000)))
}
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
val sdf = classOf[SimpleDateFormat].getName
defineCodeGen(ctx, ev, (timestamp, format) => {
s"""UTF8String.fromString((new $sdf($format.toString()))
- .format(new java.sql.Date($timestamp / 1000)))"""
+ .format(new java.util.Date($timestamp / 1000)))"""
})
}
@@ -251,6 +252,218 @@ case class DateFormatClass(left: Expression, right:
Expression) extends BinaryEx
}
/**
+ * Converts time string with given pattern
+ * (see
[http://docs.oracle.com/javase/tutorial/i18n/format/simpleDateFormat.html])
+ * to Unix time stamp (in seconds), returns null if fail.
+ * Note that hive Language Manual says it returns 0 if fail, but in fact it
returns null.
+ * If the second parameter is missing, use "yyyy-MM-dd HH:mm:ss".
+ * If no parameters provided, the first parameter will be current_timestamp.
+ * If the first parameter is a Date or Timestamp instead of String, we will
ignore the
+ * second parameter.
+ */
+case class UnixTimestamp(timeExp: Expression, format: Expression)
+ extends BinaryExpression with ExpectsInputTypes {
+
+ override def left: Expression = timeExp
+ override def right: Expression = format
+
+ def this(time: Expression) = {
+ this(time, Literal("yyyy-MM-dd HH:mm:ss"))
+ }
+
+ def this() = {
+ this(CurrentTimestamp())
+ }
+
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(TypeCollection(StringType, DateType, TimestampType), StringType)
+
+ override def dataType: DataType = LongType
+
+ private lazy val constFormat: UTF8String =
right.eval().asInstanceOf[UTF8String]
+
+ override def eval(input: InternalRow): Any = {
+ val t = left.eval(input)
+ if (t == null) {
+ null
+ } else {
+ left.dataType match {
+ case DateType =>
+ DateTimeUtils.daysToMillis(t.asInstanceOf[Int]) / 1000L
+ case TimestampType =>
+ t.asInstanceOf[Long] / 1000000L
+ case StringType if right.foldable =>
+ if (constFormat != null) {
+ Try(new SimpleDateFormat(constFormat.toString).parse(
+ t.asInstanceOf[UTF8String].toString).getTime /
1000L).getOrElse(null)
+ } else {
+ null
+ }
+ case StringType =>
+ val f = format.eval(input)
+ if (f == null) {
+ null
+ } else {
+ val formatString = f.asInstanceOf[UTF8String].toString
+ Try(new SimpleDateFormat(formatString).parse(
+ t.asInstanceOf[UTF8String].toString).getTime /
1000L).getOrElse(null)
+ }
+ }
+ }
+ }
+
+ override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
+ left.dataType match {
+ case StringType if right.foldable =>
+ val sdf = classOf[SimpleDateFormat].getName
+ val fString = if (constFormat == null) null else constFormat.toString
+ val formatter = ctx.freshName("formatter")
+ if (fString == null) {
+ s"""
+ boolean ${ev.isNull} = true;
+ ${ctx.javaType(dataType)} ${ev.primitive} =
${ctx.defaultValue(dataType)};
+ """
+ } else {
+ val eval1 = left.gen(ctx)
+ s"""
+ ${eval1.code}
+ boolean ${ev.isNull} = ${eval1.isNull};
+ ${ctx.javaType(dataType)} ${ev.primitive} =
${ctx.defaultValue(dataType)};
+ if (!${ev.isNull}) {
+ try {
+ $sdf $formatter = new $sdf("$fString");
+ ${ev.primitive} =
+ $formatter.parse(${eval1.primitive}.toString()).getTime() /
1000L;
+ } catch (java.lang.Throwable e) {
+ ${ev.isNull} = true;
+ }
+ }
+ """
+ }
+ case StringType =>
+ val sdf = classOf[SimpleDateFormat].getName
+ nullSafeCodeGen(ctx, ev, (string, format) => {
+ s"""
+ try {
+ ${ev.primitive} =
+ (new
$sdf($format.toString())).parse($string.toString()).getTime() / 1000L;
+ } catch (java.lang.Throwable e) {
+ ${ev.isNull} = true;
+ }
+ """
+ })
+ case TimestampType =>
+ val eval1 = left.gen(ctx)
+ s"""
+ ${eval1.code}
+ boolean ${ev.isNull} = ${eval1.isNull};
+ ${ctx.javaType(dataType)} ${ev.primitive} =
${ctx.defaultValue(dataType)};
+ if (!${ev.isNull}) {
+ ${ev.primitive} = ${eval1.primitive} / 1000000L;
+ }
+ """
+ case DateType =>
+ val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
+ val eval1 = left.gen(ctx)
+ s"""
+ ${eval1.code}
+ boolean ${ev.isNull} = ${eval1.isNull};
+ ${ctx.javaType(dataType)} ${ev.primitive} =
${ctx.defaultValue(dataType)};
+ if (!${ev.isNull}) {
+ ${ev.primitive} = $dtu.daysToMillis(${eval1.primitive}) / 1000L;
+ }
+ """
+ }
+ }
+}
+
+/**
+ * Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to
a string
+ * representing the timestamp of that moment in the current system time zone
in the given
+ * format. If the format is missing, using format like "1970-01-01 00:00:00".
+ * Note that hive Language Manual says it returns 0 if fail, but in fact it
returns null.
+ */
+case class FromUnixTime(sec: Expression, format: Expression)
+ extends BinaryExpression with ImplicitCastInputTypes {
+
+ override def left: Expression = sec
+ override def right: Expression = format
+
+ def this(unix: Expression) = {
+ this(unix, Literal("yyyy-MM-dd HH:mm:ss"))
+ }
+
+ override def dataType: DataType = StringType
+
+ override def inputTypes: Seq[AbstractDataType] = Seq(LongType, StringType)
+
+ private lazy val constFormat: UTF8String =
right.eval().asInstanceOf[UTF8String]
+
+ override def eval(input: InternalRow): Any = {
+ val time = left.eval(input)
+ if (time == null) {
+ null
+ } else {
+ if (format.foldable) {
+ if (constFormat == null) {
+ null
+ } else {
+ Try(UTF8String.fromString(new
SimpleDateFormat(constFormat.toString).format(
+ new java.util.Date(time.asInstanceOf[Long] *
1000L)))).getOrElse(null)
+ }
+ } else {
+ val f = format.eval(input)
+ if (f == null) {
+ null
+ } else {
+ Try(UTF8String.fromString(new SimpleDateFormat(
+ f.asInstanceOf[UTF8String].toString).format(new java.util.Date(
+ time.asInstanceOf[Long] * 1000L)))).getOrElse(null)
+ }
+ }
+ }
+ }
+
+ override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
+ val sdf = classOf[SimpleDateFormat].getName
+ if (format.foldable) {
+ if (constFormat == null) {
+ s"""
+ boolean ${ev.isNull} = true;
+ ${ctx.javaType(dataType)} ${ev.primitive} =
${ctx.defaultValue(dataType)};
+ """
+ } else {
+ val t = left.gen(ctx)
+ s"""
+ ${t.code}
+ boolean ${ev.isNull} = ${t.isNull};
+ ${ctx.javaType(dataType)} ${ev.primitive} =
${ctx.defaultValue(dataType)};
+ if (!${ev.isNull}) {
+ try {
+ ${ev.primitive} = UTF8String.fromString(new
$sdf("${constFormat.toString}").format(
+ new java.util.Date(${t.primitive} * 1000L)));
+ } catch (java.lang.Throwable e) {
+ ${ev.isNull} = true;
+ }
+ }
+ """
+ }
+ } else {
+ nullSafeCodeGen(ctx, ev, (seconds, f) => {
+ s"""
+ try {
+ ${ev.primitive} = UTF8String.fromString((new
$sdf($f.toString())).format(
+ new java.util.Date($seconds * 1000L)));
+ } catch (java.lang.Throwable e) {
+ ${ev.isNull} = true;
+ }""".stripMargin
+ })
+ }
+ }
+
+}
+
+/**
* Returns the last day of the month which the date belongs to.
*/
case class LastDay(startDate: Expression) extends UnaryExpression with
ImplicitCastInputTypes {
http://git-wip-us.apache.org/repos/asf/spark/blob/6d94bf6a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index aca8d6e..e1387f9 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -22,8 +22,9 @@ import java.text.SimpleDateFormat
import java.util.Calendar
import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import org.apache.spark.sql.types.{StringType, TimestampType, DateType}
+import org.apache.spark.sql.types._
class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
@@ -303,4 +304,60 @@ class DateExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
checkEvaluation(
NextDay(Literal(Date.valueOf("2015-07-23")), Literal.create(null,
StringType)), null)
}
+
+ test("from_unixtime") {
+ val sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
+ val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
+ val sdf2 = new SimpleDateFormat(fmt2)
+ checkEvaluation(
+ FromUnixTime(Literal(0L), Literal("yyyy-MM-dd HH:mm:ss")),
sdf1.format(new Timestamp(0)))
+ checkEvaluation(FromUnixTime(
+ Literal(1000L), Literal("yyyy-MM-dd HH:mm:ss")), sdf1.format(new
Timestamp(1000000)))
+ checkEvaluation(
+ FromUnixTime(Literal(-1000L), Literal(fmt2)), sdf2.format(new
Timestamp(-1000000)))
+ checkEvaluation(
+ FromUnixTime(Literal.create(null, LongType), Literal.create(null,
StringType)), null)
+ checkEvaluation(
+ FromUnixTime(Literal.create(null, LongType), Literal("yyyy-MM-dd
HH:mm:ss")), null)
+ checkEvaluation(FromUnixTime(Literal(1000L), Literal.create(null,
StringType)), null)
+ checkEvaluation(
+ FromUnixTime(Literal(0L), Literal("not a valid format")), null)
+ }
+
+ test("unix_timestamp") {
+ val sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
+ val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
+ val sdf2 = new SimpleDateFormat(fmt2)
+ val fmt3 = "yy-MM-dd"
+ val sdf3 = new SimpleDateFormat(fmt3)
+ val date1 = Date.valueOf("2015-07-24")
+ checkEvaluation(
+ UnixTimestamp(Literal(sdf1.format(new Timestamp(0))),
Literal("yyyy-MM-dd HH:mm:ss")), 0L)
+ checkEvaluation(UnixTimestamp(
+ Literal(sdf1.format(new Timestamp(1000000))), Literal("yyyy-MM-dd
HH:mm:ss")), 1000L)
+ checkEvaluation(
+ UnixTimestamp(Literal(new Timestamp(1000000)), Literal("yyyy-MM-dd
HH:mm:ss")), 1000L)
+ checkEvaluation(
+ UnixTimestamp(Literal(date1), Literal("yyyy-MM-dd HH:mm:ss")),
+ DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1)) / 1000L)
+ checkEvaluation(
+ UnixTimestamp(Literal(sdf2.format(new Timestamp(-1000000))),
Literal(fmt2)), -1000L)
+ checkEvaluation(UnixTimestamp(
+ Literal(sdf3.format(Date.valueOf("2015-07-24"))), Literal(fmt3)),
+
DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")))
/ 1000L)
+ val t1 = UnixTimestamp(
+ CurrentTimestamp(), Literal("yyyy-MM-dd
HH:mm:ss")).eval().asInstanceOf[Long]
+ val t2 = UnixTimestamp(
+ CurrentTimestamp(), Literal("yyyy-MM-dd
HH:mm:ss")).eval().asInstanceOf[Long]
+ assert(t2 - t1 <= 1)
+ checkEvaluation(
+ UnixTimestamp(Literal.create(null, DateType), Literal.create(null,
StringType)), null)
+ checkEvaluation(
+ UnixTimestamp(Literal.create(null, DateType), Literal("yyyy-MM-dd
HH:mm:ss")), null)
+ checkEvaluation(UnixTimestamp(
+ Literal(date1), Literal.create(null, StringType)), date1.getTime / 1000L)
+ checkEvaluation(
+ UnixTimestamp(Literal("2015-07-24"), Literal("not a valid format")),
null)
+ }
+
}
http://git-wip-us.apache.org/repos/asf/spark/blob/6d94bf6a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index a2fece6..3f440e0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -2110,6 +2110,48 @@ object functions {
*/
def weekofyear(columnName: String): Column = weekofyear(Column(columnName))
+ /**
+ * Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC)
to a string
+ * representing the timestamp of that moment in the current system time zone
in the given
+ * format.
+ * @group datetime_funcs
+ * @since 1.5.0
+ */
+ def from_unixtime(ut: Column): Column = FromUnixTime(ut.expr,
Literal("yyyy-MM-dd HH:mm:ss"))
+
+ /**
+ * Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC)
to a string
+ * representing the timestamp of that moment in the current system time zone
in the given
+ * format.
+ * @group datetime_funcs
+ * @since 1.5.0
+ */
+ def from_unixtime(ut: Column, f: String): Column = FromUnixTime(ut.expr,
Literal(f))
+
+ /**
+ * Gets current Unix timestamp in seconds.
+ * @group datetime_funcs
+ * @since 1.5.0
+ */
+ def unix_timestamp(): Column = UnixTimestamp(CurrentTimestamp(),
Literal("yyyy-MM-dd HH:mm:ss"))
+
+ /**
+ * Converts time string in format yyyy-MM-dd HH:mm:ss to Unix timestamp (in
seconds),
+ * using the default timezone and the default locale, return null if fail.
+ * @group datetime_funcs
+ * @since 1.5.0
+ */
+ def unix_timestamp(s: Column): Column = UnixTimestamp(s.expr,
Literal("yyyy-MM-dd HH:mm:ss"))
+
+ /**
+ * Convert time string with given pattern
+ * (see
[http://docs.oracle.com/javase/tutorial/i18n/format/simpleDateFormat.html])
+ * to Unix time stamp (in seconds), return null if fail.
+ * @group datetime_funcs
+ * @since 1.5.0
+ */
+ def unix_timestamp(s: Column, p: String): Column = UnixTimestamp(s.expr,
Literal(p))
+
//////////////////////////////////////////////////////////////////////////////////////////////
// Collection functions
//////////////////////////////////////////////////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/spark/blob/6d94bf6a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
----------------------------------------------------------------------
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
index 07eb6e4..df4cb57 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
@@ -228,4 +228,60 @@ class DateFunctionsSuite extends QueryTest {
Seq(Row(Date.valueOf("2015-07-30")), Row(Date.valueOf("2015-07-30"))))
}
+ test("from_unixtime") {
+ val sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
+ val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
+ val sdf2 = new SimpleDateFormat(fmt2)
+ val fmt3 = "yy-MM-dd HH-mm-ss"
+ val sdf3 = new SimpleDateFormat(fmt3)
+ val df = Seq((1000, "yyyy-MM-dd HH:mm:ss.SSS"), (-1000, "yy-MM-dd
HH-mm-ss")).toDF("a", "b")
+ checkAnswer(
+ df.select(from_unixtime(col("a"))),
+ Seq(Row(sdf1.format(new Timestamp(1000000))), Row(sdf1.format(new
Timestamp(-1000000)))))
+ checkAnswer(
+ df.select(from_unixtime(col("a"), fmt2)),
+ Seq(Row(sdf2.format(new Timestamp(1000000))), Row(sdf2.format(new
Timestamp(-1000000)))))
+ checkAnswer(
+ df.select(from_unixtime(col("a"), fmt3)),
+ Seq(Row(sdf3.format(new Timestamp(1000000))), Row(sdf3.format(new
Timestamp(-1000000)))))
+ checkAnswer(
+ df.selectExpr("from_unixtime(a)"),
+ Seq(Row(sdf1.format(new Timestamp(1000000))), Row(sdf1.format(new
Timestamp(-1000000)))))
+ checkAnswer(
+ df.selectExpr(s"from_unixtime(a, '$fmt2')"),
+ Seq(Row(sdf2.format(new Timestamp(1000000))), Row(sdf2.format(new
Timestamp(-1000000)))))
+ checkAnswer(
+ df.selectExpr(s"from_unixtime(a, '$fmt3')"),
+ Seq(Row(sdf3.format(new Timestamp(1000000))), Row(sdf3.format(new
Timestamp(-1000000)))))
+ }
+
+ test("unix_timestamp") {
+ val date1 = Date.valueOf("2015-07-24")
+ val date2 = Date.valueOf("2015-07-25")
+ val ts1 = Timestamp.valueOf("2015-07-24 10:00:00.3")
+ val ts2 = Timestamp.valueOf("2015-07-25 02:02:02.2")
+ val s1 = "2015/07/24 10:00:00.5"
+ val s2 = "2015/07/25 02:02:02.6"
+ val ss1 = "2015-07-24 10:00:00"
+ val ss2 = "2015-07-25 02:02:02"
+ val fmt = "yyyy/MM/dd HH:mm:ss.S"
+ val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts",
"s", "ss")
+ checkAnswer(df.select(unix_timestamp(col("ts"))), Seq(
+ Row(ts1.getTime / 1000L), Row(ts2.getTime / 1000L)))
+ checkAnswer(df.select(unix_timestamp(col("ss"))), Seq(
+ Row(ts1.getTime / 1000L), Row(ts2.getTime / 1000L)))
+ checkAnswer(df.select(unix_timestamp(col("d"), fmt)), Seq(
+ Row(date1.getTime / 1000L), Row(date2.getTime / 1000L)))
+ checkAnswer(df.select(unix_timestamp(col("s"), fmt)), Seq(
+ Row(ts1.getTime / 1000L), Row(ts2.getTime / 1000L)))
+ checkAnswer(df.selectExpr("unix_timestamp(ts)"), Seq(
+ Row(ts1.getTime / 1000L), Row(ts2.getTime / 1000L)))
+ checkAnswer(df.selectExpr("unix_timestamp(ss)"), Seq(
+ Row(ts1.getTime / 1000L), Row(ts2.getTime / 1000L)))
+ checkAnswer(df.selectExpr(s"unix_timestamp(d, '$fmt')"), Seq(
+ Row(date1.getTime / 1000L), Row(date2.getTime / 1000L)))
+ checkAnswer(df.selectExpr(s"unix_timestamp(s, '$fmt')"), Seq(
+ Row(ts1.getTime / 1000L), Row(ts2.getTime / 1000L)))
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
