spark git commit: [SPARK-16588][SQL] Deprecate monotonicallyIncreasingId in Scala/Java

[rxin]

Repository: spark
Updated Branches:
  refs/heads/branch-2.0 a4bf13a74 -> 808d69aaa


[SPARK-16588][SQL] Deprecate monotonicallyIncreasingId in Scala/Java

This patch deprecates monotonicallyIncreasingId in Scala/Java, as done in 
Python.

This patch was originally written by HyukjinKwon. Closes #14236.

(cherry picked from commit 480c870644595a71102be6597146d80b1c0816e4)
Signed-off-by: Reynold Xin 


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/808d69aa
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/808d69aa
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/808d69aa

Branch: refs/heads/branch-2.0
Commit: 808d69aaaf5024c139ce9f6ab4e1f62496f7118c
Parents: a4bf13a
Author: Reynold Xin 
Authored: Sun Jul 17 22:48:00 2016 -0700
Committer: Reynold Xin 
Committed: Sun Jul 17 22:49:27 2016 -0700

--
 .../scala/org/apache/spark/ml/clustering/LDA.scala |  4 ++--
 .../main/scala/org/apache/spark/sql/functions.scala|  1 +
 .../org/apache/spark/sql/ColumnExpressionSuite.scala   | 13 ++---
 3 files changed, 9 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/808d69aa/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
--
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala 
b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
index 778cd0f..034f2c3 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
@@ -37,7 +37,7 @@ import org.apache.spark.mllib.linalg.MatrixImplicits._
 import org.apache.spark.mllib.linalg.VectorImplicits._
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
-import org.apache.spark.sql.functions.{col, monotonicallyIncreasingId, udf}
+import org.apache.spark.sql.functions.{col, monotonically_increasing_id, udf}
 import org.apache.spark.sql.types.StructType
 
 
@@ -888,7 +888,7 @@ object LDA extends DefaultParamsReadable[LDA] {
dataset: Dataset[_],
featuresCol: String): RDD[(Long, OldVector)] = {
 dataset
-  .withColumn("docId", monotonicallyIncreasingId())
+  .withColumn("docId", monotonically_increasing_id())
   .select("docId", featuresCol)
   .rdd
   .map { case Row(docId: Long, features: Vector) =>

http://git-wip-us.apache.org/repos/asf/spark/blob/808d69aa/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala 
b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index c8782df..ab09ef7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -978,6 +978,7 @@ object functions {
* @group normal_funcs
* @since 1.4.0
*/
+  @deprecated("Use monotonically_increasing_id()", "2.0.0")
   def monotonicallyIncreasingId(): Column = monotonically_increasing_id()
 
   /**

http://git-wip-us.apache.org/repos/asf/spark/blob/808d69aa/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
--
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala 
b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
index a170fae..26e1a9f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
@@ -508,18 +508,17 @@ class ColumnExpressionSuite extends QueryTest with 
SharedSQLContext {
   Row("ab", "cde"))
   }
 
-  test("monotonicallyIncreasingId") {
+  test("monotonically_increasing_id") {
 // Make sure we have 2 partitions, each with 2 records.
 val df = sparkContext.parallelize(Seq[Int](), 2).mapPartitions { _ =>
   Iterator(Tuple1(1), Tuple1(2))
 }.toDF("a")
 checkAnswer(
-  df.select(monotonicallyIncreasingId()),
-  Row(0L) :: Row(1L) :: Row((1L << 33) + 0L) :: Row((1L << 33) + 1L) :: Nil
-)
-checkAnswer(
-  df.select(expr("monotonically_increasing_id()")),
-  Row(0L) :: Row(1L) :: Row((1L << 33) + 0L) :: Row((1L << 33) + 1L) :: Nil
+  df.select(monotonically_increasing_id(), 
expr("monotonically_increasing_id()")),
+  Row(0L, 0L) ::
+Row(1L, 1L) ::
+Row((1L << 33) + 0L, (1L << 33) + 0L) ::
+Row((1L << 33) + 1L, (1L << 33) + 1L) :: Nil
 )
   }
 


-
To 

spark git commit: [SPARK-16588][SQL] Deprecate monotonicallyIncreasingId in Scala/Java

[rxin]

Repository: spark
Updated Branches:
  refs/heads/master d27fe9ba6 -> 480c87064


[SPARK-16588][SQL] Deprecate monotonicallyIncreasingId in Scala/Java

This patch deprecates monotonicallyIncreasingId in Scala/Java, as done in 
Python.

This patch was originally written by HyukjinKwon. Closes #14236.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/480c8706
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/480c8706
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/480c8706

Branch: refs/heads/master
Commit: 480c870644595a71102be6597146d80b1c0816e4
Parents: d27fe9b
Author: Reynold Xin 
Authored: Sun Jul 17 22:48:00 2016 -0700
Committer: Reynold Xin 
Committed: Sun Jul 17 22:48:00 2016 -0700

--
 .../scala/org/apache/spark/ml/clustering/LDA.scala |  4 ++--
 .../main/scala/org/apache/spark/sql/functions.scala|  1 +
 .../org/apache/spark/sql/ColumnExpressionSuite.scala   | 13 ++---
 3 files changed, 9 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/480c8706/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
--
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala 
b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
index 778cd0f..034f2c3 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
@@ -37,7 +37,7 @@ import org.apache.spark.mllib.linalg.MatrixImplicits._
 import org.apache.spark.mllib.linalg.VectorImplicits._
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
-import org.apache.spark.sql.functions.{col, monotonicallyIncreasingId, udf}
+import org.apache.spark.sql.functions.{col, monotonically_increasing_id, udf}
 import org.apache.spark.sql.types.StructType
 
 
@@ -888,7 +888,7 @@ object LDA extends DefaultParamsReadable[LDA] {
dataset: Dataset[_],
featuresCol: String): RDD[(Long, OldVector)] = {
 dataset
-  .withColumn("docId", monotonicallyIncreasingId())
+  .withColumn("docId", monotonically_increasing_id())
   .select("docId", featuresCol)
   .rdd
   .map { case Row(docId: Long, features: Vector) =>

http://git-wip-us.apache.org/repos/asf/spark/blob/480c8706/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala 
b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 45d5d05..93af845 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -978,6 +978,7 @@ object functions {
* @group normal_funcs
* @since 1.4.0
*/
+  @deprecated("Use monotonically_increasing_id()", "2.0.0")
   def monotonicallyIncreasingId(): Column = monotonically_increasing_id()
 
   /**

http://git-wip-us.apache.org/repos/asf/spark/blob/480c8706/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
--
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala 
b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
index a170fae..26e1a9f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
@@ -508,18 +508,17 @@ class ColumnExpressionSuite extends QueryTest with 
SharedSQLContext {
   Row("ab", "cde"))
   }
 
-  test("monotonicallyIncreasingId") {
+  test("monotonically_increasing_id") {
 // Make sure we have 2 partitions, each with 2 records.
 val df = sparkContext.parallelize(Seq[Int](), 2).mapPartitions { _ =>
   Iterator(Tuple1(1), Tuple1(2))
 }.toDF("a")
 checkAnswer(
-  df.select(monotonicallyIncreasingId()),
-  Row(0L) :: Row(1L) :: Row((1L << 33) + 0L) :: Row((1L << 33) + 1L) :: Nil
-)
-checkAnswer(
-  df.select(expr("monotonically_increasing_id()")),
-  Row(0L) :: Row(1L) :: Row((1L << 33) + 0L) :: Row((1L << 33) + 1L) :: Nil
+  df.select(monotonically_increasing_id(), 
expr("monotonically_increasing_id()")),
+  Row(0L, 0L) ::
+Row(1L, 1L) ::
+Row((1L << 33) + 0L, (1L << 33) + 0L) ::
+Row((1L << 33) + 1L, (1L << 33) + 1L) :: Nil
 )
   }
 


-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org