spark git commit: [SPARK-10778] [MLLIB] Implement toString for AssociationRules.Rule

2015-09-27 Thread srowen 
Repository: spark
Updated Branches:
  refs/heads/master 6f94d56a9 -> 299b43992


[SPARK-10778] [MLLIB] Implement toString for AssociationRules.Rule

I implemented toString for AssociationRules.Rule, format like `[x, y] => {z}: 
1.0`

Author: y-shimizu 

Closes #8904 from y-shimizu/master.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/299b4399
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/299b4399
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/299b4399

Branch: refs/heads/master
Commit: 299b439920f980cce4c4f4e4a8436a5145efeaa3
Parents: 6f94d56
Author: y-shimizu 
Authored: Sun Sep 27 16:36:03 2015 +0100
Committer: Sean Owen 
Committed: Sun Sep 27 16:36:03 2015 +0100

--
 .../scala/org/apache/spark/mllib/fpm/AssociationRules.scala | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/299b4399/mllib/src/main/scala/org/apache/spark/mllib/fpm/AssociationRules.scala
--
diff --git 
a/mllib/src/main/scala/org/apache/spark/mllib/fpm/AssociationRules.scala 
b/mllib/src/main/scala/org/apache/spark/mllib/fpm/AssociationRules.scala
index 95c688c..07eb750 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/fpm/AssociationRules.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/fpm/AssociationRules.scala
@@ -142,5 +142,10 @@ object AssociationRules {
 def javaConsequent: java.util.List[Item] = {
   consequent.toList.asJava
 }
+
+override def toString: String = {
+  s"${antecedent.mkString("{", ",", "}")} => " +
+s"${consequent.mkString("{", ",", "}")}: ${confidence}"
+}
   }
 }


-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org

spark git commit: [SPARK-10741] [SQL] Hive Query Having/OrderBy against Parquet table is not working

2015-09-27 Thread yhuai 
Repository: spark
Updated Branches:
  refs/heads/branch-1.5 3fb011a48 -> e0c3212a9


[SPARK-10741] [SQL] Hive Query Having/OrderBy against Parquet table is not 
working

https://issues.apache.org/jira/browse/SPARK-10741
I choose the second approach: do not change output exprIds when convert 
MetastoreRelation to LogicalRelation

Author: Wenchen Fan 

Closes #8889 from cloud-fan/hot-bug.

(cherry picked from commit 418e5e4cbdaab87addb91ac0bb2245ff0213ac81)
Signed-off-by: Yin Huai 


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e0c3212a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e0c3212a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e0c3212a

Branch: refs/heads/branch-1.5
Commit: e0c3212a9b42e3e704b070da4ac25b68c584427f
Parents: 3fb011a
Author: Wenchen Fan 
Authored: Sun Sep 27 09:08:38 2015 -0700
Committer: Yin Huai 
Committed: Sun Sep 27 09:08:51 2015 -0700

--
 .../analysis/MultiInstanceRelation.scala|  1 -
 .../catalyst/expressions/namedExpressions.scala |  8 +++
 .../scala/org/apache/spark/sql/DataFrame.scala  |  2 +-
 .../datasources/DataSourceStrategy.scala| 18 +++---
 .../execution/datasources/LogicalRelation.scala | 33 +++---
 .../spark/sql/execution/datasources/rules.scala | 14 ++---
 .../parquet/ParquetFilterSuite.scala|  2 +-
 .../ParquetPartitionDiscoverySuite.scala|  2 +-
 .../spark/sql/hive/HiveMetastoreCatalog.scala   | 64 +---
 .../spark/sql/hive/execution/commands.scala |  2 +-
 .../sql/hive/InsertIntoHiveTableSuite.scala |  4 +-
 .../sql/hive/MetastoreDataSourcesSuite.scala|  2 +-
 .../sql/hive/execution/SQLQuerySuite.scala  | 27 -
 .../apache/spark/sql/hive/parquetSuites.scala   |  8 +--
 .../sql/sources/hadoopFsRelationSuites.scala|  2 +-
 15 files changed, 103 insertions(+), 86 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/e0c3212a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/MultiInstanceRelation.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/MultiInstanceRelation.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/MultiInstanceRelation.scala
index 35b7402..394be47 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/MultiInstanceRelation.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/MultiInstanceRelation.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.catalyst.analysis
 
-import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 
 /**

http://git-wip-us.apache.org/repos/asf/spark/blob/e0c3212a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
index 6f173b5..5768c60 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
@@ -236,6 +236,14 @@ case class AttributeReference(
 }
   }
 
+  def withExprId(newExprId: ExprId): AttributeReference = {
+if (exprId == newExprId) {
+  this
+} else {
+  AttributeReference(name, dataType, nullable, metadata)(newExprId, 
qualifiers)
+}
+  }
+
   override def toString: String = s"$name#${exprId.id}$typeSuffix"
 }
 

http://git-wip-us.apache.org/repos/asf/spark/blob/e0c3212a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala 
b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
index 49b40a3..f024db7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
@@ -1562,7 +1562,7 @@ class DataFrame private[sql](
*/
   def inputFiles: Array[String] = {
 val files: Seq[String] = logicalPlan.collect {
-  case LogicalRelation(fsBasedRelation: FileRelation) =>
+  case LogicalRelation(fsBasedRelation: FileRelation, _) =>
 fsBasedRelation.inputFiles
   case fr: FileRelation =>
 fr.inputFiles

svn commit: r10668 - /dev/spark/spark-1.5.1-rc1/

2015-09-27 Thread rxin 
Author: rxin
Date: Sun Sep 27 21:45:30 2015
New Revision: 10668

Log:
Add spark-1.5.1-rc1

Added:
dev/spark/spark-1.5.1-rc1/
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-cdh4.tgz   (with props)
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-cdh4.tgz.asc
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-cdh4.tgz.md5
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-cdh4.tgz.sha
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop1-scala2.11.tgz   (with 
props)
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop1-scala2.11.tgz.asc
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop1-scala2.11.tgz.md5
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop1-scala2.11.tgz.sha
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop2.3.tgz   (with props)
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop2.3.tgz.asc
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop2.3.tgz.md5
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop2.3.tgz.sha
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop2.4.tgz   (with props)
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop2.4.tgz.asc
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop2.4.tgz.md5
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop2.4.tgz.sha
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop2.6.tgz   (with props)
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop2.6.tgz.asc
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop2.6.tgz.md5.txt
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop2.6.tgz.sha
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-without-hadoop.tgz   (with props)
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-without-hadoop.tgz.asc
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-without-hadoop.tgz.md5.txt
dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-without-hadoop.tgz.sha
dev/spark/spark-1.5.1-rc1/spark-1.5.1.tgz   (with props)
dev/spark/spark-1.5.1-rc1/spark-1.5.1.tgz.asc
dev/spark/spark-1.5.1-rc1/spark-1.5.1.tgz.md5.txt
dev/spark/spark-1.5.1-rc1/spark-1.5.1.tgz.sha

Added: dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-cdh4.tgz
==
Binary file - no diff available.

Propchange: dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-cdh4.tgz
--
svn:mime-type = application/octet-stream

Added: dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-cdh4.tgz.asc
==
--- dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-cdh4.tgz.asc (added)
+++ dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-cdh4.tgz.asc Sun Sep 27 21:45:30 
2015
@@ -0,0 +1,11 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v2.0.14 (GNU/Linux)
+
+iQEcBAABAgAGBQJWA5RhAAoJEHxsEF/8jtCJ6awH/RNikGEyrSCW7xeKSXJoUbRr
+uATtgH1yF9bRnQomHw0oZr1rIXLeuUWJ64IvEAJ6fexnuuPiI7xFOEawLEXJd6hZ
+QI96SMOhS5VgVBz8Ox9TfsXushR0MFbuRgvY/TbUrX0ySvxQNMhFdb4ygZOq377O
+gwz4qIkOMQvBCzXW9tW47HpMIgHy+JDScHhd+R8Mf6EK4utbB086qKkpr8FUx7wH
+ODm6rVbTBYu+HdGPRnZIUjQNEhO6q4/HtHmSegQQziNFDWMy1I5X2uFsJF9H5O4K
+rsAIUzNDP8Xgy/i9rbanIVsjLBC6SA8mQUrnFga+FeOmqirOfpnIuDjJtn4QwEY=
+=klAd
+-END PGP SIGNATURE-

Added: dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-cdh4.tgz.md5
==
--- dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-cdh4.tgz.md5 (added)
+++ dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-cdh4.tgz.md5 Sun Sep 27 21:45:30 
2015
@@ -0,0 +1 @@
+spark-1.5.1-bin-cdh4.tgz: 87 9B DD 56 34 D5 E0 B0  5E 85 54 7D E6 10 A8 37

Added: dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-cdh4.tgz.sha
==
--- dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-cdh4.tgz.sha (added)
+++ dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-cdh4.tgz.sha Sun Sep 27 21:45:30 
2015
@@ -0,0 +1,3 @@
+spark-1.5.1-bin-cdh4.tgz: 0ED48B83 96B8E4A2 15ED9E31 E3459F7C EA743C9C 1882AD30
+  81645904 E4E356C6 C5ECDF2A 97791F5C 50A8A8C6 06FD1F22
+  C8BB5EFE AF8164B3 52B3CC0F 17A39756

Added: dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop1-scala2.11.tgz
==
Binary file - no diff available.

Propchange: dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop1-scala2.11.tgz
--
svn:mime-type = application/octet-stream

Added: dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop1-scala2.11.tgz.asc
==
--- dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop1-scala2.11.tgz.asc (added)
+++ dev/spark/spark-1.5.1-rc1/spark-1.5.1-bin-hadoop1-scala2.11.tgz.asc Sun Sep 
27 21:45:30 2015
@@ -0,0 +1,11 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v2.0.14 (GNU/Linux)
+
+iQEcBAABAgAGBQJWA5QdAAoJEHxsEF/8jtCJl/gH+wbrAAEivL+tjbp+4zZ9b9Ko

svn commit: r10669 - /dev/spark/spark-1.5.1-rc1/ /release/spark/spark-1.5.1/

2015-09-27 Thread rxin 
Author: rxin
Date: Sun Sep 27 21:47:42 2015
New Revision: 10669

Log:
Spark 1.5.1 binary

Added:
release/spark/spark-1.5.1/
  - copied from r10668, dev/spark/spark-1.5.1-rc1/
Removed:
dev/spark/spark-1.5.1-rc1/


-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org

svn commit: r1705570 - in /spark: documentation.md site/documentation.html

2015-09-27 Thread pwendell 
Author: pwendell
Date: Sun Sep 27 20:22:29 2015
New Revision: 1705570

URL: http://svn.apache.org/viewvc?rev=1705570=rev
Log:
Adding new Spark books to documentation page.

Modified:
spark/documentation.md
spark/site/documentation.html

Modified: spark/documentation.md
URL: 
http://svn.apache.org/viewvc/spark/documentation.md?rev=1705570=1705569=1705570=diff
==
--- spark/documentation.md (original)
+++ spark/documentation.md Sun Sep 27 20:22:29 2015
@@ -146,6 +146,8 @@ Slides, videos and EC2-based exercises f
   http://shop.oreilly.com/product/0636920035091.do;>Advanced 
Analytics with Spark, by Sandy Ryza, Uri Laserson, Sean Owen and Josh Wills 
(O'Reilly Media)
   http://manning.com/malak/;>Spark GraphX in Action, by 
Michael Malak (Manning)
   https://www.packtpub.com/big-data-and-business-intelligence/fast-data-processing-spark-second-edition;>Fast
 Data Processing with Spark, by Krishna Sankar and Holden Karau (Packt 
Publishing)
+  https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-spark;>Machine
 Learning with Spark, by Nick Pentreath (Packt Publishing)
+  https://www.packtpub.com/big-data-and-business-intelligence/spark-cookbook;>Spark
 Cookbook, by Rishi Yadav (Packt Publishing)
 
 
 Examples

Modified: spark/site/documentation.html
URL: 
http://svn.apache.org/viewvc/spark/site/documentation.html?rev=1705570=1705569=1705570=diff
==
--- spark/site/documentation.html (original)
+++ spark/site/documentation.html Sun Sep 27 20:22:29 2015
@@ -304,6 +304,8 @@ Slides, videos and EC2-based exercises f
   http://shop.oreilly.com/product/0636920035091.do;>Advanced 
Analytics with Spark, by Sandy Ryza, Uri Laserson, Sean Owen and Josh Wills 
(O'Reilly Media)
   http://manning.com/malak/;>Spark GraphX in Action, by 
Michael Malak (Manning)
   https://www.packtpub.com/big-data-and-business-intelligence/fast-data-processing-spark-second-edition;>Fast
 Data Processing with Spark, by Krishna Sankar and Holden Karau (Packt 
Publishing)
+  https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-spark;>Machine
 Learning with Spark, by Nick Pentreath (Packt Publishing)
+  https://www.packtpub.com/big-data-and-business-intelligence/spark-cookbook;>Spark
 Cookbook, by Rishi Yadav (Packt Publishing)
 
 
 Examples



-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org

spark git commit: [SPARK-10720] [SQL] [JAVA] Add a java wrapper to create a dataframe from a local list of java beans

2015-09-27 Thread srowen 
Repository: spark
Updated Branches:
  refs/heads/master 418e5e4cb -> 8ecba3e86


[SPARK-10720] [SQL] [JAVA] Add a java wrapper to create a dataframe from a 
local list of java beans

Similar to SPARK-10630 it would be nice if Java users didn't have to 
parallelize there data explicitly (as Scala users already can skip). Issue came 
up in 
http://stackoverflow.com/questions/32613413/apache-spark-machine-learning-cant-get-estimator-example-to-work

Author: Holden Karau 

Closes #8879 from 
holdenk/SPARK-10720-add-a-java-wrapper-to-create-a-dataframe-from-a-local-list-of-java-beans.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8ecba3e8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8ecba3e8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8ecba3e8

Branch: refs/heads/master
Commit: 8ecba3e86e53834413da8b4299f5791545cae12e
Parents: 418e5e4
Author: Holden Karau 
Authored: Sun Sep 27 21:16:15 2015 +0100
Committer: Sean Owen 
Committed: Sun Sep 27 21:16:15 2015 +0100

--
 .../scala/org/apache/spark/sql/SQLContext.scala | 51 +++-
 .../apache/spark/sql/JavaDataFrameSuite.java| 22 +++--
 2 files changed, 56 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/8ecba3e8/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala 
b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 1bd4e26..cb0a3e3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql
 
-import java.beans.Introspector
+import java.beans.{BeanInfo, Introspector}
 import java.util.Properties
 import java.util.concurrent.atomic.AtomicReference
 
@@ -499,21 +499,12 @@ class SQLContext(@transient val sparkContext: 
SparkContext)
* @since 1.3.0
*/
   def createDataFrame(rdd: RDD[_], beanClass: Class[_]): DataFrame = {
-val attributeSeq = getSchema(beanClass)
+val attributeSeq: Seq[AttributeReference] = getSchema(beanClass)
 val className = beanClass.getName
 val rowRdd = rdd.mapPartitions { iter =>
   // BeanInfo is not serializable so we must rediscover it remotely for 
each partition.
   val localBeanInfo = 
Introspector.getBeanInfo(Utils.classForName(className))
-  val extractors =
-localBeanInfo.getPropertyDescriptors.filterNot(_.getName == 
"class").map(_.getReadMethod)
-  val methodsToConverts = extractors.zip(attributeSeq).map { case (e, 
attr) =>
-(e, CatalystTypeConverters.createToCatalystConverter(attr.dataType))
-  }
-  iter.map { row =>
-new GenericInternalRow(
-  methodsToConverts.map { case (e, convert) => convert(e.invoke(row)) 
}.toArray[Any]
-): InternalRow
-  }
+  SQLContext.beansToRows(iter, localBeanInfo, attributeSeq)
 }
 DataFrame(this, LogicalRDD(attributeSeq, rowRdd)(this))
   }
@@ -531,6 +522,23 @@ class SQLContext(@transient val sparkContext: SparkContext)
   }
 
   /**
+   * Applies a schema to an List of Java Beans.
+   *
+   * WARNING: Since there is no guaranteed ordering for fields in a Java Bean,
+   *  SELECT * queries will return the columns in an undefined order.
+   * @group dataframes
+   * @since 1.6.0
+   */
+  def createDataFrame(data: java.util.List[_], beanClass: Class[_]): DataFrame 
= {
+val attrSeq = getSchema(beanClass)
+val className = beanClass.getName
+val beanInfo = Introspector.getBeanInfo(beanClass)
+val rows = SQLContext.beansToRows(data.asScala.iterator, beanInfo, attrSeq)
+DataFrame(self, LocalRelation(attrSeq, rows.toSeq))
+  }
+
+
+  /**
* :: Experimental ::
* Returns a [[DataFrameReader]] that can be used to read data in as a 
[[DataFrame]].
* {{{
@@ -1229,4 +1237,23 @@ object SQLContext {
   lastInstantiatedContext.set(sqlContext)
 }
   }
+
+  /**
+   * Converts an iterator of Java Beans to InternalRow using the provided
+   * bean info & schema. This is not related to the singleton, but is a static
+   * method for internal use.
+   */
+  private def beansToRows(data: Iterator[_], beanInfo: BeanInfo, attrs: 
Seq[AttributeReference]):
+  Iterator[InternalRow] = {
+val extractors =
+  beanInfo.getPropertyDescriptors.filterNot(_.getName == 
"class").map(_.getReadMethod)
+val methodsToConverts = extractors.zip(attrs).map { case (e, attr) =>
+  (e, CatalystTypeConverters.createToCatalystConverter(attr.dataType))
+}
+data.map{ element =>
+  new GenericInternalRow(
+

Git Push Summary

2015-09-27 Thread rxin 
Repository: spark
Updated Tags:  refs/tags/v1.5.1-rc1 [deleted] 4df97937d

-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org

Git Push Summary

2015-09-27 Thread rxin 
Repository: spark
Updated Tags:  refs/tags/v1.5.1 [created] 4f894dd69

-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org

spark git commit: add doc for spark.streaming.stopGracefullyOnShutdown

2015-09-27 Thread srowen 
Repository: spark
Updated Branches:
  refs/heads/master 8ecba3e86 -> fb4c7be74


add doc for spark.streaming.stopGracefullyOnShutdown

Author: Bin Wang 

Closes #8898 from wb14123/doc.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fb4c7be7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fb4c7be7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fb4c7be7

Branch: refs/heads/master
Commit: fb4c7be747a35ea733df6010194cd76db449adc9
Parents: 8ecba3e
Author: Bin Wang 
Authored: Sun Sep 27 21:26:54 2015 +0100
Committer: Sean Owen 
Committed: Sun Sep 27 21:26:54 2015 +0100

--
 docs/configuration.md | 8 
 1 file changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/fb4c7be7/docs/configuration.md
--
diff --git a/docs/configuration.md b/docs/configuration.md
index 284f97a..d99092e 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1491,6 +1491,14 @@ Apart from these, the following properties are also 
available, and may be useful
   
 
 
+  spark.streaming.stopGracefullyOnShutdown
+  false
+  
+If true, Spark shuts down the StreamingContext 
gracefully on JVM 
+shutdown rather than immediately.
+  
+
+
   spark.streaming.kafka.maxRatePerPartition
   not set
   


-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org

svn commit: r1705591 - in /spark/site/docs: ./ 1.5.1/ 1.5.1/api/ 1.5.1/api/R/ 1.5.1/api/java/ 1.5.1/api/java/lib/ 1.5.1/api/java/org/ 1.5.1/api/java/org/apache/ 1.5.1/api/java/org/apache/spark/ 1.5.1/

2015-09-27 Thread rxin 
Author: rxin
Date: Sun Sep 27 22:59:57 2015
New Revision: 1705591

URL: http://svn.apache.org/viewvc?rev=1705591=rev
Log:
Add 1.5.1 docs


[This commit notification would consist of 805 parts, 
which exceeds the limit of 50 ones, so it was shortened to the summary.]

-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org

spark git commit: [SPARK-10741] [SQL] Hive Query Having/OrderBy against Parquet table is not working

2015-09-27 Thread yhuai 
Repository: spark
Updated Branches:
  refs/heads/master 299b43992 -> 418e5e4cb


[SPARK-10741] [SQL] Hive Query Having/OrderBy against Parquet table is not 
working

https://issues.apache.org/jira/browse/SPARK-10741
I choose the second approach: do not change output exprIds when convert 
MetastoreRelation to LogicalRelation

Author: Wenchen Fan 

Closes #8889 from cloud-fan/hot-bug.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/418e5e4c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/418e5e4c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/418e5e4c

Branch: refs/heads/master
Commit: 418e5e4cbdaab87addb91ac0bb2245ff0213ac81
Parents: 299b439
Author: Wenchen Fan 
Authored: Sun Sep 27 09:08:38 2015 -0700
Committer: Yin Huai 
Committed: Sun Sep 27 09:08:38 2015 -0700

--
 .../analysis/MultiInstanceRelation.scala|  1 -
 .../catalyst/expressions/namedExpressions.scala |  8 +++
 .../scala/org/apache/spark/sql/DataFrame.scala  |  2 +-
 .../datasources/DataSourceStrategy.scala| 18 +++---
 .../execution/datasources/LogicalRelation.scala | 33 +++---
 .../spark/sql/execution/datasources/rules.scala | 14 ++---
 .../parquet/ParquetFilterSuite.scala|  2 +-
 .../ParquetPartitionDiscoverySuite.scala|  2 +-
 .../spark/sql/hive/HiveMetastoreCatalog.scala   | 64 +---
 .../spark/sql/hive/execution/commands.scala |  2 +-
 .../sql/hive/InsertIntoHiveTableSuite.scala |  4 +-
 .../sql/hive/MetastoreDataSourcesSuite.scala|  2 +-
 .../sql/hive/execution/SQLQuerySuite.scala  | 27 -
 .../apache/spark/sql/hive/parquetSuites.scala   |  8 +--
 .../sql/sources/hadoopFsRelationSuites.scala|  2 +-
 15 files changed, 103 insertions(+), 86 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/418e5e4c/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/MultiInstanceRelation.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/MultiInstanceRelation.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/MultiInstanceRelation.scala
index 35b7402..394be47 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/MultiInstanceRelation.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/MultiInstanceRelation.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.catalyst.analysis
 
-import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 
 /**

http://git-wip-us.apache.org/repos/asf/spark/blob/418e5e4c/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
index 6f173b5..5768c60 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
@@ -236,6 +236,14 @@ case class AttributeReference(
 }
   }
 
+  def withExprId(newExprId: ExprId): AttributeReference = {
+if (exprId == newExprId) {
+  this
+} else {
+  AttributeReference(name, dataType, nullable, metadata)(newExprId, 
qualifiers)
+}
+  }
+
   override def toString: String = s"$name#${exprId.id}$typeSuffix"
 }
 

http://git-wip-us.apache.org/repos/asf/spark/blob/418e5e4c/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala 
b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
index f9995da..9c67ad1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
@@ -1595,7 +1595,7 @@ class DataFrame private[sql](
*/
   def inputFiles: Array[String] = {
 val files: Seq[String] = logicalPlan.collect {
-  case LogicalRelation(fsBasedRelation: FileRelation) =>
+  case LogicalRelation(fsBasedRelation: FileRelation, _) =>
 fsBasedRelation.inputFiles
   case fr: FileRelation =>
 fr.inputFiles

http://git-wip-us.apache.org/repos/asf/spark/blob/418e5e4c/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
--
diff --git