spark git commit: [SPARK-24190][SQL] Allow saving of JSON files in UTF-16 and UTF-32
Repository: spark Updated Branches: refs/heads/master 4e7d8678a -> c7e2742f9 [SPARK-24190][SQL] Allow saving of JSON files in UTF-16 and UTF-32 ## What changes were proposed in this pull request? Currently, restrictions in JSONOptions for `encoding` and `lineSep` are the same for read and for write. For example, a requirement for `lineSep` in the code: ``` df.write.option("encoding", "UTF-32BE").json(file) ``` doesn't allow to skip `lineSep` and use its default value `\n` because it throws the exception: ``` equirement failed: The lineSep option must be specified for the UTF-32BE encoding java.lang.IllegalArgumentException: requirement failed: The lineSep option must be specified for the UTF-32BE encoding ``` In the PR, I propose to separate JSONOptions in read and write, and make JSONOptions in write less restrictive. ## How was this patch tested? Added new test for blacklisted encodings in read. And the `lineSep` option was removed in write for some tests. Author: Maxim Gekk Author: Maxim Gekk Closes #21247 from MaxGekk/json-options-in-write. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c7e2742f Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c7e2742f Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c7e2742f Branch: refs/heads/master Commit: c7e2742f9bce2fcb7c717df80761939272beff54 Parents: 4e7d867 Author: Maxim Gekk Authored: Sat Jun 23 17:40:20 2018 -0700 Committer: Xiao Li Committed: Sat Jun 23 17:40:20 2018 -0700 -- .../spark/sql/catalyst/json/JSONOptions.scala | 71 ++-- .../datasources/json/JsonFileFormat.scala | 13 ++-- .../execution/datasources/json/JsonSuite.scala | 56 +-- 3 files changed, 95 insertions(+), 45 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/c7e2742f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala index c081772..47eeb70 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala @@ -97,32 +97,16 @@ private[sql] class JSONOptions( sep } + protected def checkedEncoding(enc: String): String = enc + /** * Standard encoding (charset) name. For example UTF-8, UTF-16LE and UTF-32BE. - * If the encoding is not specified (None), it will be detected automatically - * when the multiLine option is set to `true`. + * If the encoding is not specified (None) in read, it will be detected automatically + * when the multiLine option is set to `true`. If encoding is not specified in write, + * UTF-8 is used by default. */ val encoding: Option[String] = parameters.get("encoding") -.orElse(parameters.get("charset")).map { enc => - // The following encodings are not supported in per-line mode (multiline is false) - // because they cause some problems in reading files with BOM which is supposed to - // present in the files with such encodings. After splitting input files by lines, - // only the first lines will have the BOM which leads to impossibility for reading - // the rest lines. Besides of that, the lineSep option must have the BOM in such - // encodings which can never present between lines. - val blacklist = Seq(Charset.forName("UTF-16"), Charset.forName("UTF-32")) - val isBlacklisted = blacklist.contains(Charset.forName(enc)) - require(multiLine || !isBlacklisted, -s"""The $enc encoding in the blacklist is not allowed when multiLine is disabled. - |Blacklist: ${blacklist.mkString(", ")}""".stripMargin) - - val isLineSepRequired = -multiLine || Charset.forName(enc) == StandardCharsets.UTF_8 || lineSeparator.nonEmpty - - require(isLineSepRequired, s"The lineSep option must be specified for the $enc encoding") - - enc - } +.orElse(parameters.get("charset")).map(checkedEncoding) val lineSeparatorInRead: Option[Array[Byte]] = lineSeparator.map { lineSep => lineSep.getBytes(encoding.getOrElse("UTF-8")) @@ -141,3 +125,46 @@ private[sql] class JSONOptions( factory.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, allowUnquotedControlChars) } } + +private[sql] class JSONOptionsInRead( +@transient override val parameters: CaseInsensitiveMap[String], +defaultTimeZoneId: String, +defaultColumnNameOfCorruptRecord: String) + extends JSONOptions(parameters, defaultTimeZoneId, defaultColumnNameOfCorruptRecord) { + + def thi
spark git commit: [SPARK-24206][SQL] Improve FilterPushdownBenchmark benchmark code
Repository: spark Updated Branches: refs/heads/master c7e2742f9 -> 98f363b77 [SPARK-24206][SQL] Improve FilterPushdownBenchmark benchmark code ## What changes were proposed in this pull request? This pr added benchmark code `FilterPushdownBenchmark` for string pushdown and updated performance results on the AWS `r3.xlarge`. ## How was this patch tested? N/A Author: Takeshi Yamamuro Closes #21288 from maropu/UpdateParquetBenchmark. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/98f363b7 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/98f363b7 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/98f363b7 Branch: refs/heads/master Commit: 98f363b77488792009f5b97bf831ede280f232e2 Parents: c7e2742 Author: Takeshi Yamamuro Authored: Sat Jun 23 17:51:18 2018 -0700 Committer: Xiao Li Committed: Sat Jun 23 17:51:18 2018 -0700 -- .../spark/sql/FilterPushdownBenchmark.scala | 243 -- .../benchmark/FilterPushdownBenchmark.scala | 442 +++ 2 files changed, 442 insertions(+), 243 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/98f363b7/sql/core/src/test/scala/org/apache/spark/sql/FilterPushdownBenchmark.scala -- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/FilterPushdownBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/FilterPushdownBenchmark.scala deleted file mode 100644 index c6dd7da..000 --- a/sql/core/src/test/scala/org/apache/spark/sql/FilterPushdownBenchmark.scala +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - *http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql - -import java.io.File - -import scala.util.{Random, Try} - -import org.apache.spark.SparkConf -import org.apache.spark.sql.functions.monotonically_increasing_id -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.util.{Benchmark, Utils} - - -/** - * Benchmark to measure read performance with Filter pushdown. - */ -object FilterPushdownBenchmark { - val conf = new SparkConf() - conf.set("orc.compression", "snappy") - conf.set("spark.sql.parquet.compression.codec", "snappy") - - private val spark = SparkSession.builder() -.master("local[1]") -.appName("FilterPushdownBenchmark") -.config(conf) -.getOrCreate() - - def withTempPath(f: File => Unit): Unit = { -val path = Utils.createTempDir() -path.delete() -try f(path) finally Utils.deleteRecursively(path) - } - - def withTempTable(tableNames: String*)(f: => Unit): Unit = { -try f finally tableNames.foreach(spark.catalog.dropTempView) - } - - def withSQLConf(pairs: (String, String)*)(f: => Unit): Unit = { -val (keys, values) = pairs.unzip -val currentValues = keys.map(key => Try(spark.conf.get(key)).toOption) -(keys, values).zipped.foreach(spark.conf.set) -try f finally { - keys.zip(currentValues).foreach { -case (key, Some(value)) => spark.conf.set(key, value) -case (key, None) => spark.conf.unset(key) - } -} - } - - private def prepareTable(dir: File, numRows: Int, width: Int): Unit = { -import spark.implicits._ -val selectExpr = (1 to width).map(i => s"CAST(value AS STRING) c$i") -val df = spark.range(numRows).map(_ => Random.nextLong).selectExpr(selectExpr: _*) - .withColumn("id", monotonically_increasing_id()) - -val dirORC = dir.getCanonicalPath + "/orc" -val dirParquet = dir.getCanonicalPath + "/parquet" - -df.write.mode("overwrite").orc(dirORC) -df.write.mode("overwrite").parquet(dirParquet) - -spark.read.orc(dirORC).createOrReplaceTempView("orcTable") -spark.read.parquet(dirParquet).createOrReplaceTempView("parquetTable") - } - - def filterPushDownBenchmark( - values: Int, - title: String, - whereExpr: String, - selectExpr: String = "*"): Unit = { -val benchmark = new Benchmark(title, values, minNumIters = 5) - -Seq(false, true).foreach { pushDownEnabled => - val name = s"
spark git commit: [SPARK-24327][SQL] Verify and normalize a partition column name based on the JDBC resolved schema
Repository: spark Updated Branches: refs/heads/master a5849ad9a -> f596ebe4d [SPARK-24327][SQL] Verify and normalize a partition column name based on the JDBC resolved schema ## What changes were proposed in this pull request? This pr modified JDBC datasource code to verify and normalize a partition column based on the JDBC resolved schema before building `JDBCRelation`. Closes #20370 ## How was this patch tested? Added tests in `JDBCSuite`. Author: Takeshi Yamamuro Closes #21379 from maropu/SPARK-24327. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f596ebe4 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f596ebe4 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f596ebe4 Branch: refs/heads/master Commit: f596ebe4d3170590b6fce34c179e51ee80c965d3 Parents: a5849ad Author: Takeshi Yamamuro Authored: Sun Jun 24 23:14:42 2018 -0700 Committer: Xiao Li Committed: Sun Jun 24 23:14:42 2018 -0700 -- .../scala/org/apache/spark/util/Utils.scala | 2 +- .../datasources/jdbc/JDBCRelation.scala | 76 .../datasources/jdbc/JdbcRelationProvider.scala | 6 +- .../org/apache/spark/sql/jdbc/JDBCSuite.scala | 51 - 4 files changed, 118 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/f596ebe4/core/src/main/scala/org/apache/spark/util/Utils.scala -- diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index c139db4..a6fd363 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -100,7 +100,7 @@ private[spark] object Utils extends Logging { */ val DEFAULT_MAX_TO_STRING_FIELDS = 25 - private def maxNumToStringFields = { + private[spark] def maxNumToStringFields = { if (SparkEnv.get != null) { SparkEnv.get.conf.getInt("spark.debug.maxToStringFields", DEFAULT_MAX_TO_STRING_FIELDS) } else { http://git-wip-us.apache.org/repos/asf/spark/blob/f596ebe4/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala index b23e5a7..b84543c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala @@ -22,10 +22,12 @@ import scala.collection.mutable.ArrayBuffer import org.apache.spark.Partition import org.apache.spark.internal.Logging import org.apache.spark.rdd.RDD -import org.apache.spark.sql.{DataFrame, Row, SaveMode, SparkSession, SQLContext} +import org.apache.spark.sql.{AnalysisException, DataFrame, Row, SaveMode, SparkSession, SQLContext} +import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.jdbc.JdbcDialects import org.apache.spark.sql.sources._ import org.apache.spark.sql.types.StructType +import org.apache.spark.util.Utils /** * Instructions on how to partition the table among workers. @@ -48,10 +50,17 @@ private[sql] object JDBCRelation extends Logging { * Null value predicate is added to the first partition where clause to include * the rows with null value for the partitions column. * + * @param schema resolved schema of a JDBC table * @param partitioning partition information to generate the where clause for each partition + * @param resolver function used to determine if two identifiers are equal + * @param jdbcOptions JDBC options that contains url * @return an array of partitions with where clause for each partition */ - def columnPartition(partitioning: JDBCPartitioningInfo): Array[Partition] = { + def columnPartition( + schema: StructType, + partitioning: JDBCPartitioningInfo, + resolver: Resolver, + jdbcOptions: JDBCOptions): Array[Partition] = { if (partitioning == null || partitioning.numPartitions <= 1 || partitioning.lowerBound == partitioning.upperBound) { return Array[Partition](JDBCPartition(null, 0)) @@ -78,7 +87,10 @@ private[sql] object JDBCRelation extends Logging { // Overflow and silliness can happen if you subtract then divide. // Here we get a little roundoff, but that's (hopefully) OK. val stride: Long = upperBound / numPartitions - lowerBound / numPartitions -val column = partitioning.column + +val column = verifyAndGetNormalizedColumnName( + schema, partitioning.column, reso
spark git commit: [SPARK-23931][SQL][FOLLOW-UP] Make `arrays_zip` in function.scala `@scala.annotation.varargs`.
Repository: spark Updated Branches: refs/heads/master f596ebe4d -> 6e0596e26 [SPARK-23931][SQL][FOLLOW-UP] Make `arrays_zip` in function.scala `@scala.annotation.varargs`. ## What changes were proposed in this pull request? This is a follow-up pr of #21045 which added `arrays_zip`. The `arrays_zip` in functions.scala should've been `scala.annotation.varargs`. This pr makes it `scala.annotation.varargs`. ## How was this patch tested? Existing tests. Author: Takuya UESHIN Closes #21630 from ueshin/issues/SPARK-23931/fup1. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6e0596e2 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6e0596e2 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6e0596e2 Branch: refs/heads/master Commit: 6e0596e2639117d7a0a58b644b0600086f45c7f9 Parents: f596ebe Author: Takuya UESHIN Authored: Sun Jun 24 23:56:47 2018 -0700 Committer: Xiao Li Committed: Sun Jun 24 23:56:47 2018 -0700 -- sql/core/src/main/scala/org/apache/spark/sql/functions.scala | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/6e0596e2/sql/core/src/main/scala/org/apache/spark/sql/functions.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index f792a6f..40c40e7 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -3539,6 +3539,7 @@ object functions { * @group collection_funcs * @since 2.4.0 */ + @scala.annotation.varargs def arrays_zip(e: Column*): Column = withExpr { ArraysZip(e.map(_.expr)) } // - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24596][SQL] Non-cascading Cache Invalidation
Repository: spark Updated Branches: refs/heads/master 8ab8ef773 -> bac50aa37 [SPARK-24596][SQL] Non-cascading Cache Invalidation ## What changes were proposed in this pull request? 1. Add parameter 'cascade' in CacheManager.uncacheQuery(). Under 'cascade=false' mode, only invalidate the current cache, and for other dependent caches, rebuild execution plan and reuse cached buffer. 2. Pass true/false from callers in different uncache scenarios: - Drop tables and regular (persistent) views: regular mode - Drop temporary views: non-cascading mode - Modify table contents (INSERT/UPDATE/MERGE/DELETE): regular mode - Call `DataSet.unpersist()`: non-cascading mode - Call `Catalog.uncacheTable()`: follow the same convention as drop tables/view, which is, use non-cascading mode for temporary views and regular mode for the rest Note that a regular (persistent) view is a database object just like a table, so after dropping a regular view (whether cached or not cached), any query referring to that view should no long be valid. Hence if a cached persistent view is dropped, we need to invalidate the all dependent caches so that exceptions will be thrown for any later reference. On the other hand, a temporary view is in fact equivalent to an unnamed DataSet, and dropping a temporary view should have no impact on queries referencing that view. Thus we should do non-cascading uncaching for temporary views, which also guarantees a consistent uncaching behavior between temporary views and unnamed DataSets. ## How was this patch tested? New tests in CachedTableSuite and DatasetCacheSuite. Author: Maryann Xue Closes #21594 from maryannxue/noncascading-cache. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bac50aa3 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bac50aa3 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bac50aa3 Branch: refs/heads/master Commit: bac50aa37168a7612702a4503750a78ed5d59c78 Parents: 8ab8ef7 Author: Maryann Xue Authored: Mon Jun 25 07:17:30 2018 -0700 Committer: Xiao Li Committed: Mon Jun 25 07:17:30 2018 -0700 -- docs/sql-programming-guide.md | 1 + .../scala/org/apache/spark/sql/Dataset.scala| 4 +- .../spark/sql/execution/CacheManager.scala | 50 +++--- .../execution/columnar/InMemoryRelation.scala | 10 +++ .../spark/sql/execution/command/ddl.scala | 8 ++- .../spark/sql/execution/command/tables.scala| 2 +- .../apache/spark/sql/internal/CatalogImpl.scala | 12 ++-- .../org/apache/spark/sql/CachedTableSuite.scala | 66 +- .../apache/spark/sql/DatasetCacheSuite.scala| 70 ++-- 9 files changed, 197 insertions(+), 26 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/bac50aa3/docs/sql-programming-guide.md -- diff --git a/docs/sql-programming-guide.md b/docs/sql-programming-guide.md index d2db067..196b814 100644 --- a/docs/sql-programming-guide.md +++ b/docs/sql-programming-guide.md @@ -1827,6 +1827,7 @@ working with timestamps in `pandas_udf`s to get the best performance, see - Since Spark 2.4, Spark compares a DATE type with a TIMESTAMP type after promotes both sides to TIMESTAMP. To set `false` to `spark.sql.hive.compareDateTimestampInTimestamp` restores the previous behavior. This option will be removed in Spark 3.0. - Since Spark 2.4, creating a managed table with nonempty location is not allowed. An exception is thrown when attempting to create a managed table with nonempty location. To set `true` to `spark.sql.allowCreatingManagedTableUsingNonemptyLocation` restores the previous behavior. This option will be removed in Spark 3.0. - Since Spark 2.4, the type coercion rules can automatically promote the argument types of the variadic SQL functions (e.g., IN/COALESCE) to the widest common type, no matter how the input arguments order. In prior Spark versions, the promotion could fail in some specific orders (e.g., TimestampType, IntegerType and StringType) and throw an exception. + - Since Spark 2.4, Spark has enabled non-cascading SQL cache invalidation in addition to the traditional cache invalidation mechanism. The non-cascading cache invalidation mechanism allows users to remove a cache without impacting its dependent caches. This new cache invalidation mechanism is used in scenarios where the data of the cache to be removed is still valid, e.g., calling unpersist() on a Dataset, or dropping a temporary view. This allows users to free up memory and keep the desired caches valid at the same time. - In version 2.3 and earlier, `to_utc_timestamp` and `from_utc_timestamp` respect the timezone in the input timestamp string, which breaks the assumption
spark git commit: [SPARK-24658][SQL] Remove workaround for ANTLR bug
Repository: spark Updated Branches: refs/heads/master e07aee216 -> dcaa49ff1 [SPARK-24658][SQL] Remove workaround for ANTLR bug ## What changes were proposed in this pull request? Issue antlr/antlr4#781 has already been fixed, so the workaround of extracting the pattern into a separate rule is no longer needed. The presto already removed it: https://github.com/prestodb/presto/pull/10744. ## How was this patch tested? Existing tests Author: Yuming Wang Closes #21641 from wangyum/ANTLR-780. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/dcaa49ff Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/dcaa49ff Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/dcaa49ff Branch: refs/heads/master Commit: dcaa49ff1edd7fcf0f000c6f93ae0e30bd5b6464 Parents: e07aee2 Author: Yuming Wang Authored: Tue Jun 26 14:33:04 2018 -0700 Committer: Xiao Li Committed: Tue Jun 26 14:33:04 2018 -0700 -- .../antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 | 9 + 1 file changed, 1 insertion(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/dcaa49ff/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 -- diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 3fe00ee..dc95751 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -539,18 +539,11 @@ expression booleanExpression : NOT booleanExpression#logicalNot | EXISTS '(' query ')' #exists -| predicated #booleanDefault +| valueExpression predicate? #predicated | left=booleanExpression operator=AND right=booleanExpression #logicalBinary | left=booleanExpression operator=OR right=booleanExpression #logicalBinary ; -// workaround for: -// https://github.com/antlr/antlr4/issues/780 -// https://github.com/antlr/antlr4/issues/781 -predicated -: valueExpression predicate? -; - predicate : NOT? kind=BETWEEN lower=valueExpression AND upper=valueExpression | NOT? kind=IN '(' expression (',' expression)* ')' - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24423][SQL] Add a new option for JDBC sources
Repository: spark Updated Branches: refs/heads/master dcaa49ff1 -> 02f8781fa [SPARK-24423][SQL] Add a new option for JDBC sources ## What changes were proposed in this pull request? Here is the description in the JIRA - Currently, our JDBC connector provides the option `dbtable` for users to specify the to-be-loaded JDBC source table. ```SQL val jdbcDf = spark.read .format("jdbc") .option("dbtable", "dbName.tableName") .options(jdbcCredentials: Map) .load() ``` Normally, users do not fetch the whole JDBC table due to the poor performance/throughput of JDBC. Thus, they normally just fetch a small set of tables. For advanced users, they can pass a subquery as the option. ```SQL val query = """ (select * from tableName limit 10) as tmp """ val jdbcDf = spark.read .format("jdbc") .option("dbtable", query) .options(jdbcCredentials: Map) .load() ``` However, this is straightforward to end users. We should simply allow users to specify the query by a new option `query`. We will handle the complexity for them. ```SQL val query = """select * from tableName limit 10""" val jdbcDf = spark.read .format("jdbc") .option("query", query) .options(jdbcCredentials: Map) .load() ``` ## How was this patch tested? Added tests in JDBCSuite and JDBCWriterSuite. Also tested against MySQL, Postgress, Oracle, DB2 (using docker infrastructure) to make sure there are no syntax issues. Author: Dilip Biswal Closes #21590 from dilipbiswal/SPARK-24423. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/02f8781f Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/02f8781f Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/02f8781f Branch: refs/heads/master Commit: 02f8781fa2649cf1d3a5cb932e1c8408790974ff Parents: dcaa49f Author: Dilip Biswal Authored: Tue Jun 26 15:17:00 2018 -0700 Committer: Xiao Li Committed: Tue Jun 26 15:17:00 2018 -0700 -- docs/sql-programming-guide.md | 30 ++- .../datasources/jdbc/JDBCOptions.scala | 66 +- .../execution/datasources/jdbc/JDBCRDD.scala| 4 +- .../datasources/jdbc/JDBCRelation.scala | 4 +- .../datasources/jdbc/JdbcRelationProvider.scala | 5 +- .../execution/datasources/jdbc/JdbcUtils.scala | 10 +-- .../org/apache/spark/sql/jdbc/JDBCSuite.scala | 94 +++- .../apache/spark/sql/jdbc/JDBCWriteSuite.scala | 14 ++- 8 files changed, 204 insertions(+), 23 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/02f8781f/docs/sql-programming-guide.md -- diff --git a/docs/sql-programming-guide.md b/docs/sql-programming-guide.md index 196b814..7c4ef41 100644 --- a/docs/sql-programming-guide.md +++ b/docs/sql-programming-guide.md @@ -1302,9 +1302,33 @@ the following case-insensitive options: dbtable - The JDBC table that should be read. Note that anything that is valid in a FROM clause of - a SQL query can be used. For example, instead of a full table you could also use a - subquery in parentheses. + The JDBC table that should be read from or written into. Note that when using it in the read + path anything that is valid in a FROM clause of a SQL query can be used. + For example, instead of a full table you could also use a subquery in parentheses. It is not + allowed to specify `dbtable` and `query` options at the same time. + + + +query + + A query that will be used to read data into Spark. The specified query will be parenthesized and used + as a subquery in the FROM clause. Spark will also assign an alias to the subquery clause. + As an example, spark will issue a query of the following form to the JDBC Source. + SELECTFROM ( ) spark_gen_alias + Below are couple of restrictions while using this option. + + It is not allowed to specify `dbtable` and `query` options at the same time. + It is not allowed to spcify `query` and `partitionColumn` options at the same time. When specifying +`partitionColumn` option is required, the subquery can be specified using `dbtable` option instead and +partition columns can be qualified using the subquery alias provided as part of `dbtable`. +Example: + + spark.read.format("jdbc") + .option("dbtable", "(select c1, c2 from t1) as subq") + .option("partitionColumn", "subq.c1" + .load() + + http://git-wip-us.apache.org/repos/asf/spark/blob/02f8781f/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala
spark git commit: [SPARK-24215][PYSPARK][FOLLOW UP] Implement eager evaluation for DataFrame APIs in PySpark
Repository: spark Updated Branches: refs/heads/master a1a64e358 -> 6a0b77a55 [SPARK-24215][PYSPARK][FOLLOW UP] Implement eager evaluation for DataFrame APIs in PySpark ## What changes were proposed in this pull request? Address comments in #21370 and add more test. ## How was this patch tested? Enhance test in pyspark/sql/test.py and DataFrameSuite Author: Yuanjian Li Closes #21553 from xuanyuanking/SPARK-24215-follow. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6a0b77a5 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6a0b77a5 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6a0b77a5 Branch: refs/heads/master Commit: 6a0b77a55d53e74ac0a0892556c3a7a933474948 Parents: a1a64e3 Author: Yuanjian Li Authored: Wed Jun 27 10:43:06 2018 -0700 Committer: Xiao Li Committed: Wed Jun 27 10:43:06 2018 -0700 -- docs/configuration.md | 27 - python/pyspark/sql/dataframe.py | 3 +- python/pyspark/sql/tests.py | 46 ++- .../org/apache/spark/sql/internal/SQLConf.scala | 23 .../scala/org/apache/spark/sql/Dataset.scala| 11 ++-- .../org/apache/spark/sql/DataFrameSuite.scala | 59 6 files changed, 131 insertions(+), 38 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/6a0b77a5/docs/configuration.md -- diff --git a/docs/configuration.md b/docs/configuration.md index 6aa7878..0c7c447 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -457,33 +457,6 @@ Apart from these, the following properties are also available, and may be useful - spark.sql.repl.eagerEval.enabled - false - -Enable eager evaluation or not. If true and the REPL you are using supports eager evaluation, -Dataset will be ran automatically. The HTML table which generated by _repl_html_ -called by notebooks like Jupyter will feedback the queries user have defined. For plain Python -REPL, the output will be shown like dataframe.show() -(see https://issues.apache.org/jira/browse/SPARK-24215";>SPARK-24215 for more details). - - - - spark.sql.repl.eagerEval.maxNumRows - 20 - -Default number of rows in eager evaluation output HTML table generated by _repr_html_ or plain text, -this only take effect when spark.sql.repl.eagerEval.enabled is set to true. - - - - spark.sql.repl.eagerEval.truncate - 20 - -Default number of truncate in eager evaluation output HTML table generated by _repr_html_ or -plain text, this only take effect when spark.sql.repl.eagerEval.enabled set to true. - - - spark.files http://git-wip-us.apache.org/repos/asf/spark/blob/6a0b77a5/python/pyspark/sql/dataframe.py -- diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py index 1e6a1ac..cb3fe44 100644 --- a/python/pyspark/sql/dataframe.py +++ b/python/pyspark/sql/dataframe.py @@ -393,9 +393,8 @@ class DataFrame(object): self._support_repr_html = True if self._eager_eval: max_num_rows = max(self._max_num_rows, 0) -vertical = False sock_info = self._jdf.getRowsToPython( -max_num_rows, self._truncate, vertical) +max_num_rows, self._truncate) rows = list(_load_from_socket(sock_info, BatchedSerializer(PickleSerializer( head = rows[0] row_data = rows[1:] http://git-wip-us.apache.org/repos/asf/spark/blob/6a0b77a5/python/pyspark/sql/tests.py -- diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py index 35a0636..8d73806 100644 --- a/python/pyspark/sql/tests.py +++ b/python/pyspark/sql/tests.py @@ -3351,11 +3351,41 @@ class SQLTests(ReusedSQLTestCase): finally: shutil.rmtree(path) -def test_repr_html(self): +def test_repr_behaviors(self): import re pattern = re.compile(r'^ *\|', re.MULTILINE) df = self.spark.createDataFrame([(1, "1"), (2, "2")], ("key", "value")) -self.assertEquals(None, df._repr_html_()) + +# test when eager evaluation is enabled and _repr_html_ will not be called +with self.sql_conf({"spark.sql.repl.eagerEval.enabled": True}): +expected1 = """+-+-+ +|| key|value| +|+-+-+ +||1|1| +||2|2| +|+-+-+ +|""" +self.assertEquals(re.sub(pattern, '', expected1), df.__repr__()) +with self.sql_conf({"s
spark git commit: [SPARK-21687][SQL] Spark SQL should set createTime for Hive partition
Repository: spark Updated Branches: refs/heads/master 78ecb6d45 -> c04cb2d1b [SPARK-21687][SQL] Spark SQL should set createTime for Hive partition ## What changes were proposed in this pull request? Set createTime for every hive partition created in Spark SQL, which could be used to manage data lifecycle in Hive warehouse. We found that almost every partition modified by spark sql has not been set createTime. ``` mysql> select * from partitions where create_time=0 limit 1\G; *** 1. row *** PART_ID: 1028584 CREATE_TIME: 0 LAST_ACCESS_TIME: 1502203611 PART_NAME: date=20170130 SD_ID: 1543605 TBL_ID: 211605 LINK_TARGET_ID: NULL 1 row in set (0.27 sec) ``` ## How was this patch tested? N/A Author: debugger87 Author: Chaozhong Yang Closes #18900 from debugger87/fix/set-create-time-for-hive-partition. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c04cb2d1 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c04cb2d1 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c04cb2d1 Branch: refs/heads/master Commit: c04cb2d1b72b1edaddf684755f5a9d6aaf00e03b Parents: 78ecb6d Author: debugger87 Authored: Wed Jun 27 11:34:28 2018 -0700 Committer: Xiao Li Committed: Wed Jun 27 11:34:28 2018 -0700 -- .../apache/spark/sql/catalyst/catalog/interface.scala | 6 ++ .../sql/catalyst/catalog/SessionCatalogSuite.scala| 6 -- .../results/describe-part-after-analyze.sql.out | 14 ++ .../test/resources/sql-tests/results/describe.sql.out | 4 .../resources/sql-tests/results/show-tables.sql.out | 2 ++ .../apache/spark/sql/hive/client/HiveClientImpl.scala | 4 6 files changed, 34 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/c04cb2d1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index f3e67dc..c6105c5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -93,12 +93,16 @@ object CatalogStorageFormat { * @param spec partition spec values indexed by column name * @param storage storage format of the partition * @param parameters some parameters for the partition + * @param createTime creation time of the partition, in milliseconds + * @param lastAccessTime last access time, in milliseconds * @param stats optional statistics (number of rows, total size, etc.) */ case class CatalogTablePartition( spec: CatalogTypes.TablePartitionSpec, storage: CatalogStorageFormat, parameters: Map[String, String] = Map.empty, +createTime: Long = System.currentTimeMillis, +lastAccessTime: Long = -1, stats: Option[CatalogStatistics] = None) { def toLinkedHashMap: mutable.LinkedHashMap[String, String] = { @@ -109,6 +113,8 @@ case class CatalogTablePartition( if (parameters.nonEmpty) { map.put("Partition Parameters", s"{${parameters.map(p => p._1 + "=" + p._2).mkString(", ")}}") } +map.put("Created Time", new Date(createTime).toString) +map.put("Last Access", new Date(lastAccessTime).toString) stats.foreach(s => map.put("Partition Statistics", s.simpleString)) map } http://git-wip-us.apache.org/repos/asf/spark/blob/c04cb2d1/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala -- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala index 6abab00..6a7375e 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala @@ -1114,11 +1114,13 @@ abstract class SessionCatalogSuite extends AnalysisTest { // And for hive serde table, hive metastore will set some values(e.g.transient_lastDdlTime) // in table's parameters and storage's properties, here we also ignore them. val actualPartsNormalize = actualParts.map(p => - p.copy(parameters = Map.empty, storage = p.storage.copy( + p.copy(parameters = Map.empty, createTime = -1, lastAccessTime = -1, +storage = p.storage.copy( properties = Map.empty, locationUri = None, serde = None))).t
spark git commit: [SPARK-24613][SQL] Cache with UDF could not be matched with subsequent dependent caches
Repository: spark Updated Branches: refs/heads/branch-2.3 db538b25a -> 6e1f5e018 [SPARK-24613][SQL] Cache with UDF could not be matched with subsequent dependent caches Wrap the logical plan with a `AnalysisBarrier` for execution plan compilation in CacheManager, in order to avoid the plan being analyzed again. Add one test in `DatasetCacheSuite` Author: Maryann Xue Closes #21602 from maryannxue/cache-mismatch. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6e1f5e01 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6e1f5e01 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6e1f5e01 Branch: refs/heads/branch-2.3 Commit: 6e1f5e0182e1d111f7252c24c576674d1d2c7b91 Parents: db538b2 Author: Maryann Xue Authored: Thu Jun 21 11:45:30 2018 -0700 Committer: Xiao Li Committed: Wed Jun 27 13:20:57 2018 -0700 -- .../apache/spark/sql/execution/CacheManager.scala | 6 +++--- .../org/apache/spark/sql/DatasetCacheSuite.scala| 16 2 files changed, 19 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/6e1f5e01/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala index d68aeb2..dbdda27 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala @@ -26,7 +26,7 @@ import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.internal.Logging import org.apache.spark.sql.{Dataset, SparkSession} import org.apache.spark.sql.catalyst.expressions.SubqueryExpression -import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, ResolvedHint} +import org.apache.spark.sql.catalyst.plans.logical.{AnalysisBarrier, LogicalPlan, ResolvedHint} import org.apache.spark.sql.execution.columnar.InMemoryRelation import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} import org.apache.spark.storage.StorageLevel @@ -97,7 +97,7 @@ class CacheManager extends Logging { val inMemoryRelation = InMemoryRelation( sparkSession.sessionState.conf.useCompression, sparkSession.sessionState.conf.columnBatchSize, storageLevel, -sparkSession.sessionState.executePlan(planToCache).executedPlan, + sparkSession.sessionState.executePlan(AnalysisBarrier(planToCache)).executedPlan, tableName, planToCache.stats) cachedData.add(CachedData(planToCache, inMemoryRelation)) @@ -146,7 +146,7 @@ class CacheManager extends Logging { useCompression = cd.cachedRepresentation.useCompression, batchSize = cd.cachedRepresentation.batchSize, storageLevel = cd.cachedRepresentation.storageLevel, - child = spark.sessionState.executePlan(cd.plan).executedPlan, + child = spark.sessionState.executePlan(AnalysisBarrier(cd.plan)).executedPlan, tableName = cd.cachedRepresentation.tableName, statsOfPlanToCache = cd.plan.stats) needToRecache += cd.copy(cachedRepresentation = newCache) http://git-wip-us.apache.org/repos/asf/spark/blob/6e1f5e01/sql/core/src/test/scala/org/apache/spark/sql/DatasetCacheSuite.scala -- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetCacheSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetCacheSuite.scala index e0561ee..f6c760e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetCacheSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetCacheSuite.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql +import org.apache.spark.sql.execution.columnar.{InMemoryRelation, InMemoryTableScanExec} import org.apache.spark.sql.functions._ import org.apache.spark.sql.test.SharedSQLContext import org.apache.spark.storage.StorageLevel @@ -96,4 +97,19 @@ class DatasetCacheSuite extends QueryTest with SharedSQLContext { agged.unpersist() assert(agged.storageLevel == StorageLevel.NONE, "The Dataset agged should not be cached.") } + + test("SPARK-24613 Cache with UDF could not be matched with subsequent dependent caches") { +val udf1 = udf({x: Int => x + 1}) +val df = spark.range(0, 10).toDF("a").withColumn("b", udf1($"a")) +val df2 = df.agg(sum(df("b"))) + +df.cache() +df.count() +df2.cache() + +val plan = df2.queryExecution.withCachedData +assert(plan.isInstanceOf[InMemoryRelation]) +val internalPlan = plan.asInstanceOf[InMemoryRelation].child +assert
spark git commit: [SPARK-24204][SQL] Verify a schema in Json/Orc/ParquetFileFormat
Repository: spark Updated Branches: refs/heads/master 221d03acc -> 893ea224c [SPARK-24204][SQL] Verify a schema in Json/Orc/ParquetFileFormat ## What changes were proposed in this pull request? This pr added code to verify a schema in Json/Orc/ParquetFileFormat along with CSVFileFormat. ## How was this patch tested? Added verification tests in `FileBasedDataSourceSuite` and `HiveOrcSourceSuite`. Author: Takeshi Yamamuro Closes #21389 from maropu/SPARK-24204. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/893ea224 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/893ea224 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/893ea224 Branch: refs/heads/master Commit: 893ea224cc738766be207c87f4b913fe8fea4c94 Parents: 221d03a Author: Takeshi Yamamuro Authored: Wed Jun 27 15:25:51 2018 -0700 Committer: Xiao Li Committed: Wed Jun 27 15:25:51 2018 -0700 -- .../execution/datasources/DataSourceUtils.scala | 106 + .../datasources/csv/CSVFileFormat.scala | 4 +- .../execution/datasources/csv/CSVUtils.scala| 19 -- .../datasources/json/JsonFileFormat.scala | 4 + .../datasources/orc/OrcFileFormat.scala | 4 + .../datasources/parquet/ParquetFileFormat.scala | 3 + .../spark/sql/FileBasedDataSourceSuite.scala| 213 ++- .../execution/datasources/csv/CSVSuite.scala| 33 --- .../spark/sql/hive/orc/OrcFileFormat.scala | 4 + .../spark/sql/hive/orc/HiveOrcSourceSuite.scala | 49 - 10 files changed, 383 insertions(+), 56 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/893ea224/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala new file mode 100644 index 000..c534721 --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.datasources + +import org.apache.spark.sql.execution.datasources.csv.CSVFileFormat +import org.apache.spark.sql.execution.datasources.json.JsonFileFormat +import org.apache.spark.sql.execution.datasources.orc.OrcFileFormat +import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat +import org.apache.spark.sql.types._ + + +object DataSourceUtils { + + /** + * Verify if the schema is supported in datasource in write path. + */ + def verifyWriteSchema(format: FileFormat, schema: StructType): Unit = { +verifySchema(format, schema, isReadPath = false) + } + + /** + * Verify if the schema is supported in datasource in read path. + */ + def verifyReadSchema(format: FileFormat, schema: StructType): Unit = { +verifySchema(format, schema, isReadPath = true) + } + + /** + * Verify if the schema is supported in datasource. This verification should be done + * in a driver side, e.g., `prepareWrite`, `buildReader`, and `buildReaderWithPartitionValues` + * in `FileFormat`. + * + * Unsupported data types of csv, json, orc, and parquet are as follows; + * csv -> R/W: Interval, Null, Array, Map, Struct + * json -> W: Interval + * orc -> W: Interval, Null + * parquet -> R/W: Interval, Null + */ + private def verifySchema(format: FileFormat, schema: StructType, isReadPath: Boolean): Unit = { +def throwUnsupportedException(dataType: DataType): Unit = { + throw new UnsupportedOperationException( +s"$format data source does not support ${dataType.simpleString} data type.") +} + +def verifyType(dataType: DataType): Unit = dataType match { + case BooleanType | ByteType | ShortType | IntegerType | LongType | FloatType | DoubleType | + StringType | BinaryType | DateType | Timestam
spark git commit: [SPARK-24553][WEB-UI] http 302 fixes for href redirect
Repository: spark Updated Branches: refs/heads/master 893ea224c -> c5aa54d54 [SPARK-24553][WEB-UI] http 302 fixes for href redirect ## What changes were proposed in this pull request? Updated URL/href links to include a '/' before '?id' to make links consistent and avoid http 302 redirect errors within UI port 4040 tabs. ## How was this patch tested? Built a runnable distribution and executed jobs. Validated that http 302 redirects are no longer encountered when clicking on links within UI port 4040 tabs. Author: Steven Kallman Author: Kallman, Steven Closes #21600 from SJKallman/{Spark-24553}{WEB-UI}-redirect-href-fixes. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c5aa54d5 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c5aa54d5 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c5aa54d5 Branch: refs/heads/master Commit: c5aa54d54b301555bad1ff0653df11293f0033ed Parents: 893ea22 Author: Kallman, Steven Authored: Wed Jun 27 15:36:59 2018 -0700 Committer: Xiao Li Committed: Wed Jun 27 15:36:59 2018 -0700 -- core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala | 2 +- core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala | 2 +- core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala| 2 +- .../org/apache/spark/sql/execution/ui/AllExecutionsPage.scala| 4 ++-- .../scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala | 2 +- .../apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala | 4 ++-- .../spark/sql/hive/thriftserver/ui/ThriftServerSessionPage.scala | 2 +- .../src/main/scala/org/apache/spark/streaming/ui/BatchPage.scala | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/c5aa54d5/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala -- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 178d2c8..90e9a7a 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -464,7 +464,7 @@ private[ui] class JobDataSource( val jobDescription = UIUtils.makeDescription(lastStageDescription, basePath, plainText = false) -val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId) +val detailUrl = "%s/jobs/job/?id=%s".format(basePath, jobData.jobId) new JobTableRowData( jobData, http://git-wip-us.apache.org/repos/asf/spark/blob/c5aa54d5/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala -- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala index d4e6a7b..55eb989 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala @@ -282,7 +282,7 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We val _taskTable = new TaskPagedTable( stageData, UIUtils.prependBaseUri(request, parent.basePath) + - s"/stages/stage?id=${stageId}&attempt=${stageAttemptId}", + s"/stages/stage/?id=${stageId}&attempt=${stageAttemptId}", currentTime, pageSize = taskPageSize, sortColumn = taskSortColumn, http://git-wip-us.apache.org/repos/asf/spark/blob/c5aa54d5/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala -- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala index 56e4d68..d01acda 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala @@ -370,7 +370,7 @@ private[ui] class StagePagedTable( Seq.empty } -val nameLinkUri = s"$basePathUri/stages/stage?id=${s.stageId}&attempt=${s.attemptId}" +val nameLinkUri = s"$basePathUri/stages/stage/?id=${s.stageId}&attempt=${s.attemptId}" val nameLink = {s.name} val cachedRddInfos = store.rddList().filter { rdd => s.rddIds.contains(rdd.id) } http://git-wip-us.apache.org/repos/asf/spark/blob/c5aa54d5/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala ind
[2/2] spark git commit: simplify rand in dsl/package.scala
simplify rand in dsl/package.scala Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d54d8b86 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d54d8b86 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d54d8b86 Branch: refs/heads/master Commit: d54d8b86301581142293341af25fd78b3278a2e8 Parents: 797971e Author: Xiao Li Authored: Fri Jun 29 23:51:13 2018 -0700 Committer: Xiao Li Committed: Fri Jun 29 23:51:13 2018 -0700 -- .../src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/d54d8b86/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala index 8cf69c6..89e8c99 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala @@ -149,7 +149,7 @@ package object dsl { } } -def rand(e: Long): Expression = Rand(Literal.create(e, LongType)) +def rand(e: Long): Expression = Rand(e) def sum(e: Expression): Expression = Sum(e).toAggregateExpression() def sumDistinct(e: Expression): Expression = Sum(e).toAggregateExpression(isDistinct = true) def count(e: Expression): Expression = Count(e).toAggregateExpression() - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[1/2] spark git commit: [SPARK-24696][SQL] ColumnPruning rule fails to remove extra Project
Repository: spark Updated Branches: refs/heads/master 03545ce6d -> d54d8b863 [SPARK-24696][SQL] ColumnPruning rule fails to remove extra Project ## What changes were proposed in this pull request? The ColumnPruning rule tries adding an extra Project if an input node produces fields more than needed, but as a post-processing step, it needs to remove the lower Project in the form of "Project - Filter - Project" otherwise it would conflict with PushPredicatesThroughProject and would thus cause a infinite optimization loop. The current post-processing method is defined as: ``` private def removeProjectBeforeFilter(plan: LogicalPlan): LogicalPlan = plan transform { case p1 Project(_, f Filter(_, p2 Project(_, child))) if p2.outputSet.subsetOf(child.outputSet) => p1.copy(child = f.copy(child = child)) } ``` This method works well when there is only one Filter but would not if there's two or more Filters. In this case, there is a deterministic filter and a non-deterministic filter so they stay as separate filter nodes and cannot be combined together. An simplified illustration of the optimization process that forms the infinite loop is shown below (F1 stands for the 1st filter, F2 for the 2nd filter, P for project, S for scan of relation, PredicatePushDown as abbrev. of PushPredicatesThroughProject): ``` F1 - F2 - P - S PredicatePushDown =>F1 - P - F2 - S ColumnPruning =>F1 - P - F2 - P - S =>F1 - P - F2 - S(Project removed) PredicatePushDown =>P - F1 - F2 - S ColumnPruning =>P - F1 - P - F2 - S =>P - F1 - P - F2 - P - S =>P - F1 - F2 - P - S(only one Project removed) RemoveRedundantProject =>F1 - F2 - P - S(goes back to the loop start) ``` So the problem is the ColumnPruning rule adds a Project under a Filter (and fails to remove it in the end), and that new Project triggers PushPredicateThroughProject. Once the filters have been push through the Project, a new Project will be added by the ColumnPruning rule and this goes on and on. The fix should be when adding Projects, the rule applies top-down, but later when removing extra Projects, the process should go bottom-up to ensure all extra Projects can be matched. ## How was this patch tested? Added a optimization rule test in ColumnPruningSuite; and a end-to-end test in SQLQuerySuite. Author: maryannxue Closes #21674 from maryannxue/spark-24696. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/797971ed Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/797971ed Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/797971ed Branch: refs/heads/master Commit: 797971ed42cab41cbc3d039c0af4b26199bff783 Parents: 03545ce Author: maryannxue Authored: Fri Jun 29 23:46:12 2018 -0700 Committer: Xiao Li Committed: Fri Jun 29 23:46:12 2018 -0700 -- .../apache/spark/sql/catalyst/dsl/package.scala | 1 + .../sql/catalyst/optimizer/Optimizer.scala | 5 +++-- .../catalyst/optimizer/ColumnPruningSuite.scala | 9 - .../org/apache/spark/sql/SQLQuerySuite.scala| 21 4 files changed, 33 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/797971ed/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala index efb2eba..8cf69c6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala @@ -149,6 +149,7 @@ package object dsl { } } +def rand(e: Long): Expression = Rand(Literal.create(e, LongType)) def sum(e: Expression): Expression = Sum(e).toAggregateExpression() def sumDistinct(e: Expression): Expression = Sum(e).toAggregateExpression(isDistinct = true) def count(e: Expression): Expression = Count(e).toAggregateExpression() http://git-wip-us.apache.org/repos/asf/spark/blob/797971ed/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index aa992de..2cc27d8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/or
spark git commit: simplify rand in dsl/package.scala
Repository: spark Updated Branches: refs/heads/branch-2.3 0f534d3da -> 8ff4b9727 simplify rand in dsl/package.scala (cherry picked from commit d54d8b86301581142293341af25fd78b3278a2e8) Signed-off-by: Xiao Li Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8ff4b972 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8ff4b972 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8ff4b972 Branch: refs/heads/branch-2.3 Commit: 8ff4b97274e58f5944506b25481c6eb44238a4cd Parents: 0f534d3 Author: Xiao Li Authored: Fri Jun 29 23:51:13 2018 -0700 Committer: Xiao Li Committed: Fri Jun 29 23:53:00 2018 -0700 -- .../src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/8ff4b972/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala index efb2eba..89e8c99 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala @@ -149,6 +149,7 @@ package object dsl { } } +def rand(e: Long): Expression = Rand(e) def sum(e: Expression): Expression = Sum(e).toAggregateExpression() def sumDistinct(e: Expression): Expression = Sum(e).toAggregateExpression(isDistinct = true) def count(e: Expression): Expression = Count(e).toAggregateExpression() - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24696][SQL] ColumnPruning rule fails to remove extra Project
Repository: spark Updated Branches: refs/heads/branch-2.3 8ff4b9727 -> 3c0af793f [SPARK-24696][SQL] ColumnPruning rule fails to remove extra Project The ColumnPruning rule tries adding an extra Project if an input node produces fields more than needed, but as a post-processing step, it needs to remove the lower Project in the form of "Project - Filter - Project" otherwise it would conflict with PushPredicatesThroughProject and would thus cause a infinite optimization loop. The current post-processing method is defined as: ``` private def removeProjectBeforeFilter(plan: LogicalPlan): LogicalPlan = plan transform { case p1 Project(_, f Filter(_, p2 Project(_, child))) if p2.outputSet.subsetOf(child.outputSet) => p1.copy(child = f.copy(child = child)) } ``` This method works well when there is only one Filter but would not if there's two or more Filters. In this case, there is a deterministic filter and a non-deterministic filter so they stay as separate filter nodes and cannot be combined together. An simplified illustration of the optimization process that forms the infinite loop is shown below (F1 stands for the 1st filter, F2 for the 2nd filter, P for project, S for scan of relation, PredicatePushDown as abbrev. of PushPredicatesThroughProject): ``` F1 - F2 - P - S PredicatePushDown =>F1 - P - F2 - S ColumnPruning =>F1 - P - F2 - P - S =>F1 - P - F2 - S(Project removed) PredicatePushDown =>P - F1 - F2 - S ColumnPruning =>P - F1 - P - F2 - S =>P - F1 - P - F2 - P - S =>P - F1 - F2 - P - S(only one Project removed) RemoveRedundantProject =>F1 - F2 - P - S(goes back to the loop start) ``` So the problem is the ColumnPruning rule adds a Project under a Filter (and fails to remove it in the end), and that new Project triggers PushPredicateThroughProject. Once the filters have been push through the Project, a new Project will be added by the ColumnPruning rule and this goes on and on. The fix should be when adding Projects, the rule applies top-down, but later when removing extra Projects, the process should go bottom-up to ensure all extra Projects can be matched. Added a optimization rule test in ColumnPruningSuite; and a end-to-end test in SQLQuerySuite. Author: maryannxue Closes #21674 from maryannxue/spark-24696. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3c0af793 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3c0af793 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3c0af793 Branch: refs/heads/branch-2.3 Commit: 3c0af793f9e050f5d8dfb2f5daab6c0043c39748 Parents: 8ff4b97 Author: maryannxue Authored: Fri Jun 29 23:46:12 2018 -0700 Committer: Xiao Li Committed: Fri Jun 29 23:57:09 2018 -0700 -- .../sql/catalyst/optimizer/Optimizer.scala | 5 +++-- .../catalyst/optimizer/ColumnPruningSuite.scala | 9 - .../org/apache/spark/sql/SQLQuerySuite.scala| 21 3 files changed, 32 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/3c0af793/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index c77e0f8..38799c1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -524,9 +524,10 @@ object ColumnPruning extends Rule[LogicalPlan] { /** * The Project before Filter is not necessary but conflict with PushPredicatesThroughProject, - * so remove it. + * so remove it. Since the Projects have been added top-down, we need to remove in bottom-up + * order, otherwise lower Projects can be missed. */ - private def removeProjectBeforeFilter(plan: LogicalPlan): LogicalPlan = plan transform { + private def removeProjectBeforeFilter(plan: LogicalPlan): LogicalPlan = plan transformUp { case p1 @ Project(_, f @ Filter(_, p2 @ Project(_, child))) if p2.outputSet.subsetOf(child.outputSet) => p1.copy(child = f.copy(child = child)) http://git-wip-us.apache.org/repos/asf/spark/blob/3c0af793/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala -- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite
spark git commit: [SPARK-22384][SQL][FOLLOWUP] Refine partition pruning when attribute is wrapped in Cast
Repository: spark Updated Branches: refs/heads/master ca8243f30 -> bf764a33b [SPARK-22384][SQL][FOLLOWUP] Refine partition pruning when attribute is wrapped in Cast ## What changes were proposed in this pull request? As mentioned in https://github.com/apache/spark/pull/21586 , `Cast.mayTruncate` is not 100% safe, string to boolean is allowed. Since changing `Cast.mayTruncate` also changes the behavior of Dataset, here I propose to add a new `Cast.canSafeCast` for partition pruning. ## How was this patch tested? new test cases Author: Wenchen Fan Closes #21712 from cloud-fan/safeCast. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bf764a33 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bf764a33 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bf764a33 Branch: refs/heads/master Commit: bf764a33bef617aa9bae535a5ea73d6a3e278d42 Parents: ca8243f Author: Wenchen Fan Authored: Wed Jul 4 18:36:09 2018 -0700 Committer: Xiao Li Committed: Wed Jul 4 18:36:09 2018 -0700 -- .../spark/sql/catalyst/expressions/Cast.scala | 20 .../apache/spark/sql/hive/client/HiveShim.scala | 5 +++-- .../spark/sql/hive/client/HiveClientSuite.scala | 20 ++-- 3 files changed, 41 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/bf764a33/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 699ea53..7971ae6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -134,6 +134,26 @@ object Cast { toPrecedence > 0 && fromPrecedence > toPrecedence } + /** + * Returns true iff we can safely cast the `from` type to `to` type without any truncating or + * precision lose, e.g. int -> long, date -> timestamp. + */ + def canSafeCast(from: AtomicType, to: AtomicType): Boolean = (from, to) match { +case _ if from == to => true +case (from: NumericType, to: DecimalType) if to.isWiderThan(from) => true +case (from: DecimalType, to: NumericType) if from.isTighterThan(to) => true +case (from, to) if legalNumericPrecedence(from, to) => true +case (DateType, TimestampType) => true +case (_, StringType) => true +case _ => false + } + + private def legalNumericPrecedence(from: DataType, to: DataType): Boolean = { +val fromPrecedence = TypeCoercion.numericPrecedence.indexOf(from) +val toPrecedence = TypeCoercion.numericPrecedence.indexOf(to) +fromPrecedence >= 0 && fromPrecedence < toPrecedence + } + def forceNullable(from: DataType, to: DataType): Boolean = (from, to) match { case (NullType, _) => true case (_, _) if from == to => false http://git-wip-us.apache.org/repos/asf/spark/blob/bf764a33/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala -- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala index 8620f3f..933384e 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala @@ -45,7 +45,7 @@ import org.apache.spark.sql.catalyst.analysis.NoSuchPermanentFunctionException import org.apache.spark.sql.catalyst.catalog.{CatalogFunction, CatalogTablePartition, CatalogUtils, FunctionResource, FunctionResourceType} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{IntegralType, StringType} +import org.apache.spark.sql.types.{AtomicType, IntegralType, StringType} import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.util.Utils @@ -660,7 +660,8 @@ private[client] class Shim_v0_13 extends Shim_v0_12 { def unapply(expr: Expression): Option[Attribute] = { expr match { case attr: Attribute => Some(attr) - case Cast(child, dt, _) if !Cast.mayTruncate(child.dataType, dt) => unapply(child) + case Cast(child @ AtomicType(), dt: AtomicType, _) + if Cast.canSafeCast(child.dataType.asInstanceOf[AtomicType], dt) => unapply(child) case _ => None } } http://git-wip-us.apache.org/repos/asf/spark/blob/bf764a33/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala ---
spark git commit: [SPARK-24675][SQL] Rename table: validate existence of new location
Repository: spark Updated Branches: refs/heads/master ac78bcce0 -> 33952cfa8 [SPARK-24675][SQL] Rename table: validate existence of new location ## What changes were proposed in this pull request? If table is renamed to a existing new location, data won't show up. ``` scala> Seq("hello").toDF("a").write.format("parquet").saveAsTable("t") scala> sql("select * from t").show() +-+ |a| +-+ |hello| +-+ scala> sql("alter table t rename to test") res2: org.apache.spark.sql.DataFrame = [] scala> sql("select * from test").show() +---+ | a| +---+ +---+ ``` The file layout is like ``` $ tree test test âââ gabage âââ t âââ _SUCCESS âââ part-0-856b0f10-08f1-42d6-9eb3-7719261f3d5e-c000.snappy.parquet ``` In Hive, if the new location exists, the renaming will fail even the location is empty. We should have the same validation in Catalog, in case of unexpected bugs. ## How was this patch tested? New unit test. Author: Gengliang Wang Closes #21655 from gengliangwang/validate_rename_table. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/33952cfa Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/33952cfa Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/33952cfa Branch: refs/heads/master Commit: 33952cfa8182c1e925083e18c63c6152dcc3c8b4 Parents: ac78bcc Author: Gengliang Wang Authored: Thu Jul 5 09:25:19 2018 -0700 Committer: Xiao Li Committed: Thu Jul 5 09:25:19 2018 -0700 -- docs/sql-programming-guide.md | 1 + .../sql/catalyst/catalog/SessionCatalog.scala | 20 .../spark/sql/execution/command/DDLSuite.scala | 18 ++ 3 files changed, 39 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/33952cfa/docs/sql-programming-guide.md -- diff --git a/docs/sql-programming-guide.md b/docs/sql-programming-guide.md index cd7329b..ad23dae 100644 --- a/docs/sql-programming-guide.md +++ b/docs/sql-programming-guide.md @@ -1850,6 +1850,7 @@ working with timestamps in `pandas_udf`s to get the best performance, see - Since Spark 2.4, writing a dataframe with an empty or nested empty schema using any file formats (parquet, orc, json, text, csv etc.) is not allowed. An exception is thrown when attempting to write dataframes with empty schema. - Since Spark 2.4, Spark compares a DATE type with a TIMESTAMP type after promotes both sides to TIMESTAMP. To set `false` to `spark.sql.hive.compareDateTimestampInTimestamp` restores the previous behavior. This option will be removed in Spark 3.0. - Since Spark 2.4, creating a managed table with nonempty location is not allowed. An exception is thrown when attempting to create a managed table with nonempty location. To set `true` to `spark.sql.allowCreatingManagedTableUsingNonemptyLocation` restores the previous behavior. This option will be removed in Spark 3.0. + - Since Spark 2.4, renaming a managed table to existing location is not allowed. An exception is thrown when attempting to rename a managed table to existing location. - Since Spark 2.4, the type coercion rules can automatically promote the argument types of the variadic SQL functions (e.g., IN/COALESCE) to the widest common type, no matter how the input arguments order. In prior Spark versions, the promotion could fail in some specific orders (e.g., TimestampType, IntegerType and StringType) and throw an exception. - Since Spark 2.4, Spark has enabled non-cascading SQL cache invalidation in addition to the traditional cache invalidation mechanism. The non-cascading cache invalidation mechanism allows users to remove a cache without impacting its dependent caches. This new cache invalidation mechanism is used in scenarios where the data of the cache to be removed is still valid, e.g., calling unpersist() on a Dataset, or dropping a temporary view. This allows users to free up memory and keep the desired caches valid at the same time. - In version 2.3 and earlier, `to_utc_timestamp` and `from_utc_timestamp` respect the timezone in the input timestamp string, which breaks the assumption that the input timestamp is in a specific timezone. Therefore, these 2 functions can return unexpected results. In version 2.4 and later, this problem has been fixed. `to_utc_timestamp` and `from_utc_timestamp` will return null if the input timestamp string contains timezone. As an example, `from_utc_timestamp('2000-10-10 00:00:00', 'GMT+1')` will return `2000-10-10 01:00:00` in both Spark 2.3 and 2.4. However, `from_utc_timestamp('2000-10-10 00:00:00+00:00', 'GMT+1')`, assuming a local timezone of GMT+8, will return `2000-10-10 09:00:00` in Spark 2.3 but `null` in 2.4. For people w
spark git commit: Revert "[SPARK-24268][SQL] Use datatype.simpleString in error messages"
Repository: spark Updated Branches: refs/heads/master 1bd3d61f4 -> aec966b05 Revert "[SPARK-24268][SQL] Use datatype.simpleString in error messages" This reverts commit 1bd3d61f4191767a94b71b42f4d00706b703e84f. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/aec966b0 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/aec966b0 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/aec966b0 Branch: refs/heads/master Commit: aec966b05e8df9d459dae88d091de1923e50e2dc Parents: 1bd3d61 Author: Xiao Li Authored: Mon Jul 9 14:24:23 2018 -0700 Committer: Xiao Li Committed: Mon Jul 9 14:24:23 2018 -0700 -- .../apache/spark/sql/kafka010/KafkaWriteTask.scala | 6 +++--- .../org/apache/spark/sql/kafka010/KafkaWriter.scala | 6 +++--- .../sql/kafka010/KafkaContinuousSinkSuite.scala | 4 ++-- .../apache/spark/sql/kafka010/KafkaSinkSuite.scala | 4 ++-- .../scala/org/apache/spark/ml/feature/DCT.scala | 3 +-- .../org/apache/spark/ml/feature/FeatureHasher.scala | 5 ++--- .../org/apache/spark/ml/feature/HashingTF.scala | 2 +- .../org/apache/spark/ml/feature/Interaction.scala | 3 +-- .../scala/org/apache/spark/ml/feature/NGram.scala | 2 +- .../org/apache/spark/ml/feature/OneHotEncoder.scala | 3 +-- .../org/apache/spark/ml/feature/RFormula.scala | 2 +- .../apache/spark/ml/feature/StopWordsRemover.scala | 4 ++-- .../org/apache/spark/ml/feature/Tokenizer.scala | 3 +-- .../apache/spark/ml/feature/VectorAssembler.scala | 2 +- .../scala/org/apache/spark/ml/fpm/FPGrowth.scala| 2 +- .../org/apache/spark/ml/util/SchemaUtils.scala | 11 --- .../BinaryClassificationEvaluatorSuite.scala| 4 ++-- .../org/apache/spark/ml/feature/RFormulaSuite.scala | 2 +- .../spark/ml/feature/VectorAssemblerSuite.scala | 6 +++--- .../apache/spark/ml/recommendation/ALSSuite.scala | 2 +- .../ml/regression/AFTSurvivalRegressionSuite.scala | 2 +- .../org/apache/spark/ml/util/MLTestingUtils.scala | 6 +++--- .../catalyst/expressions/complexTypeCreator.scala | 4 ++-- .../sql/catalyst/expressions/jsonExpressions.scala | 2 +- .../catalyst/expressions/stringExpressions.scala| 5 ++--- .../spark/sql/catalyst/json/JacksonGenerator.scala | 4 ++-- .../spark/sql/catalyst/json/JacksonParser.scala | 6 ++ .../spark/sql/catalyst/json/JsonInferSchema.scala | 6 ++ .../apache/spark/sql/catalyst/util/TypeUtils.scala | 5 ++--- .../apache/spark/sql/types/AbstractDataType.scala | 9 - .../org/apache/spark/sql/types/ArrayType.scala | 5 ++--- .../org/apache/spark/sql/types/DecimalType.scala| 3 +-- .../org/apache/spark/sql/types/ObjectType.scala | 3 +-- .../org/apache/spark/sql/types/StructType.scala | 5 ++--- .../sql/catalyst/analysis/AnalysisErrorSuite.scala | 2 +- .../analysis/ExpressionTypeCheckingSuite.scala | 16 .../sql/catalyst/parser/ExpressionParserSuite.scala | 2 +- .../org/apache/spark/sql/types/DataTypeSuite.scala | 2 +- .../datasources/parquet/VectorizedColumnReader.java | 2 +- .../apache/spark/sql/RelationalGroupedDataset.scala | 2 +- .../spark/sql/execution/arrow/ArrowUtils.scala | 3 +-- .../sql/execution/datasources/orc/OrcFilters.scala | 2 +- .../parquet/ParquetSchemaConverter.scala| 2 +- .../spark/sql/execution/stat/StatFunctions.scala| 2 +- .../sql-tests/results/json-functions.sql.out| 4 ++-- .../resources/sql-tests/results/literals.sql.out| 6 +++--- .../datasources/parquet/ParquetSchemaSuite.scala| 4 ++-- .../sql/hive/execution/HiveTableScanExec.scala | 6 +++--- 48 files changed, 88 insertions(+), 108 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/aec966b0/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaWriteTask.scala -- diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaWriteTask.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaWriteTask.scala index 59a8470..d90630a 100644 --- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaWriteTask.scala +++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaWriteTask.scala @@ -110,7 +110,7 @@ private[kafka010] abstract class KafkaRowWriter( case t => throw new IllegalStateException(s"${KafkaWriter.TOPIC_ATTRIBUTE_NAME} " + s"attribute unsupported type $t. ${KafkaWriter.TOPIC_ATTRIBUTE_NAME} " + - s"must be a ${StringType.simpleString}") + "must be a StringType") } val keyExpression = inputSchema.find(_.name == KafkaWriter.KEY_ATTRIBUTE_NAME) .g
spark git commit: [SPARK-24759][SQL] No reordering keys for broadcast hash join
Repository: spark Updated Branches: refs/heads/master aec966b05 -> eb6e98803 [SPARK-24759][SQL] No reordering keys for broadcast hash join ## What changes were proposed in this pull request? As the implementation of the broadcast hash join is independent of the input hash partitioning, reordering keys is not necessary. Thus, we solve this issue by simply removing the broadcast hash join from the reordering rule in EnsureRequirements. ## How was this patch tested? N/A Author: Xiao Li Closes #21728 from gatorsmile/cleanER. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/eb6e9880 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/eb6e9880 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/eb6e9880 Branch: refs/heads/master Commit: eb6e9880397dbac8b0b9ebc0796150b6924fc566 Parents: aec966b Author: Xiao Li Authored: Mon Jul 9 14:53:14 2018 -0700 Committer: Xiao Li Committed: Mon Jul 9 14:53:14 2018 -0700 -- .../spark/sql/execution/exchange/EnsureRequirements.scala | 7 --- 1 file changed, 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/eb6e9880/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala index ad95879..d96ecba 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala @@ -279,13 +279,6 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] { */ private def reorderJoinPredicates(plan: SparkPlan): SparkPlan = { plan match { - case BroadcastHashJoinExec(leftKeys, rightKeys, joinType, buildSide, condition, left, -right) => -val (reorderedLeftKeys, reorderedRightKeys) = - reorderJoinKeys(leftKeys, rightKeys, left.outputPartitioning, right.outputPartitioning) -BroadcastHashJoinExec(reorderedLeftKeys, reorderedRightKeys, joinType, buildSide, condition, - left, right) - case ShuffledHashJoinExec(leftKeys, rightKeys, joinType, buildSide, condition, left, right) => val (reorderedLeftKeys, reorderedRightKeys) = reorderJoinKeys(leftKeys, rightKeys, left.outputPartitioning, right.outputPartitioning) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24208][SQL] Fix attribute deduplication for FlatMapGroupsInPandas
Repository: spark Updated Branches: refs/heads/master 592cc8458 -> ebf4bfb96 [SPARK-24208][SQL] Fix attribute deduplication for FlatMapGroupsInPandas ## What changes were proposed in this pull request? A self-join on a dataset which contains a `FlatMapGroupsInPandas` fails because of duplicate attributes. This happens because we are not dealing with this specific case in our `dedupAttr` rules. The PR fix the issue by adding the management of the specific case ## How was this patch tested? added UT + manual tests Author: Marco Gaido Author: Marco Gaido Closes #21737 from mgaido91/SPARK-24208. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ebf4bfb9 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ebf4bfb9 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ebf4bfb9 Branch: refs/heads/master Commit: ebf4bfb966389342bfd9bdb8e3b612828c18730c Parents: 592cc84 Author: Marco Gaido Authored: Wed Jul 11 09:29:19 2018 -0700 Committer: Xiao Li Committed: Wed Jul 11 09:29:19 2018 -0700 -- python/pyspark/sql/tests.py | 16 .../spark/sql/catalyst/analysis/Analyzer.scala | 4 .../org/apache/spark/sql/GroupedDatasetSuite.scala | 12 3 files changed, 32 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/ebf4bfb9/python/pyspark/sql/tests.py -- diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py index 8d73806..4404dbe 100644 --- a/python/pyspark/sql/tests.py +++ b/python/pyspark/sql/tests.py @@ -5925,6 +5925,22 @@ class GroupedAggPandasUDFTests(ReusedSQLTestCase): 'mixture.*aggregate function.*group aggregate pandas UDF'): df.groupby(df.id).agg(mean_udf(df.v), mean(df.v)).collect() +def test_self_join_with_pandas(self): +import pyspark.sql.functions as F + +@F.pandas_udf('key long, col string', F.PandasUDFType.GROUPED_MAP) +def dummy_pandas_udf(df): +return df[['key', 'col']] + +df = self.spark.createDataFrame([Row(key=1, col='A'), Row(key=1, col='B'), + Row(key=2, col='C')]) +dfWithPandas = df.groupBy('key').apply(dummy_pandas_udf) + +# this was throwing an AnalysisException before SPARK-24208 +res = dfWithPandas.alias('temp0').join(dfWithPandas.alias('temp1'), + F.col('temp0.key') == F.col('temp1.key')) +self.assertEquals(res.count(), 5) + @unittest.skipIf( not _have_pandas or not _have_pyarrow, http://git-wip-us.apache.org/repos/asf/spark/blob/ebf4bfb9/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index e187133..c078efd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -738,6 +738,10 @@ class Analyzer( if findAliases(aggregateExpressions).intersect(conflictingAttributes).nonEmpty => (oldVersion, oldVersion.copy(aggregateExpressions = newAliases(aggregateExpressions))) +case oldVersion @ FlatMapGroupsInPandas(_, _, output, _) +if oldVersion.outputSet.intersect(conflictingAttributes).nonEmpty => + (oldVersion, oldVersion.copy(output = output.map(_.newInstance( + case oldVersion: Generate if oldVersion.producedAttributes.intersect(conflictingAttributes).nonEmpty => val newOutput = oldVersion.generatorOutput.map(_.newInstance()) http://git-wip-us.apache.org/repos/asf/spark/blob/ebf4bfb9/sql/core/src/test/scala/org/apache/spark/sql/GroupedDatasetSuite.scala -- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/GroupedDatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/GroupedDatasetSuite.scala index 147c0b6..bd54ea4 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/GroupedDatasetSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/GroupedDatasetSuite.scala @@ -93,4 +93,16 @@ class GroupedDatasetSuite extends QueryTest with SharedSQLContext { } datasetWithUDF.unpersist(true) } + + test("SPARK-24208: analysis fails on self-join with FlatMapGroupsInPandas") { +val df = datasetWithUDF.groupBy("s").flatMapGroupsInPandas(PythonUDF( + "pyUDF", + null, + StructType
spark git commit: [SPARK-24208][SQL] Fix attribute deduplication for FlatMapGroupsInPandas
Repository: spark Updated Branches: refs/heads/branch-2.3 86457a16d -> 32429256f [SPARK-24208][SQL] Fix attribute deduplication for FlatMapGroupsInPandas A self-join on a dataset which contains a `FlatMapGroupsInPandas` fails because of duplicate attributes. This happens because we are not dealing with this specific case in our `dedupAttr` rules. The PR fix the issue by adding the management of the specific case added UT + manual tests Author: Marco Gaido Author: Marco Gaido Closes #21737 from mgaido91/SPARK-24208. (cherry picked from commit ebf4bfb966389342bfd9bdb8e3b612828c18730c) Signed-off-by: Xiao Li Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/32429256 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/32429256 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/32429256 Branch: refs/heads/branch-2.3 Commit: 32429256f3e659c648462e5b2740747645740c97 Parents: 86457a1 Author: Marco Gaido Authored: Wed Jul 11 09:29:19 2018 -0700 Committer: Xiao Li Committed: Wed Jul 11 09:35:44 2018 -0700 -- python/pyspark/sql/tests.py | 16 .../spark/sql/catalyst/analysis/Analyzer.scala | 4 .../org/apache/spark/sql/GroupedDatasetSuite.scala | 12 3 files changed, 32 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/32429256/python/pyspark/sql/tests.py -- diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py index aa7d8eb..6bfb329 100644 --- a/python/pyspark/sql/tests.py +++ b/python/pyspark/sql/tests.py @@ -4691,6 +4691,22 @@ class GroupedMapPandasUDFTests(ReusedSQLTestCase): result = df.groupby('time').apply(foo_udf).sort('time') self.assertPandasEqual(df.toPandas(), result.toPandas()) +def test_self_join_with_pandas(self): +import pyspark.sql.functions as F + +@F.pandas_udf('key long, col string', F.PandasUDFType.GROUPED_MAP) +def dummy_pandas_udf(df): +return df[['key', 'col']] + +df = self.spark.createDataFrame([Row(key=1, col='A'), Row(key=1, col='B'), + Row(key=2, col='C')]) +dfWithPandas = df.groupBy('key').apply(dummy_pandas_udf) + +# this was throwing an AnalysisException before SPARK-24208 +res = dfWithPandas.alias('temp0').join(dfWithPandas.alias('temp1'), + F.col('temp0.key') == F.col('temp1.key')) +self.assertEquals(res.count(), 5) + if __name__ == "__main__": from pyspark.sql.tests import * http://git-wip-us.apache.org/repos/asf/spark/blob/32429256/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 8597d83..a584cb8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -723,6 +723,10 @@ class Analyzer( if findAliases(aggregateExpressions).intersect(conflictingAttributes).nonEmpty => (oldVersion, oldVersion.copy(aggregateExpressions = newAliases(aggregateExpressions))) +case oldVersion @ FlatMapGroupsInPandas(_, _, output, _) +if oldVersion.outputSet.intersect(conflictingAttributes).nonEmpty => + (oldVersion, oldVersion.copy(output = output.map(_.newInstance( + case oldVersion: Generate if oldVersion.producedAttributes.intersect(conflictingAttributes).nonEmpty => val newOutput = oldVersion.generatorOutput.map(_.newInstance()) http://git-wip-us.apache.org/repos/asf/spark/blob/32429256/sql/core/src/test/scala/org/apache/spark/sql/GroupedDatasetSuite.scala -- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/GroupedDatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/GroupedDatasetSuite.scala index 218a1b7..9699fad 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/GroupedDatasetSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/GroupedDatasetSuite.scala @@ -93,4 +93,16 @@ class GroupedDatasetSuite extends QueryTest with SharedSQLContext { } datasetWithUDF.unpersist(true) } + + test("SPARK-24208: analysis fails on self-join with FlatMapGroupsInPandas") { +val df = datasetWithUDF.groupBy("s").flatMapGroupsInPandas(PythonUDF( + "pyUDF", + null, + StructType(Seq(St
spark git commit: [SPARK-24782][SQL] Simplify conf retrieval in SQL expressions
Repository: spark Updated Branches: refs/heads/master ff7f6ef75 -> e008ad175 [SPARK-24782][SQL] Simplify conf retrieval in SQL expressions ## What changes were proposed in this pull request? The PR simplifies the retrieval of config in `size`, as we can access them from tasks too thanks to SPARK-24250. ## How was this patch tested? existing UTs Author: Marco Gaido Closes #21736 from mgaido91/SPARK-24605_followup. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e008ad17 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e008ad17 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e008ad17 Branch: refs/heads/master Commit: e008ad175256a3192fdcbd2c4793044d52f46d57 Parents: ff7f6ef Author: Marco Gaido Authored: Wed Jul 11 17:30:43 2018 -0700 Committer: Xiao Li Committed: Wed Jul 11 17:30:43 2018 -0700 -- .../expressions/collectionOperations.scala | 10 +-- .../catalyst/expressions/jsonExpressions.scala | 16 ++--- .../spark/sql/catalyst/plans/QueryPlan.scala| 2 - .../CollectionExpressionsSuite.scala| 27 .../expressions/JsonExpressionsSuite.scala | 65 ++-- .../scala/org/apache/spark/sql/functions.scala | 4 +- 6 files changed, 57 insertions(+), 67 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/e008ad17/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index 879603b..e217d37 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -89,15 +89,9 @@ trait BinaryArrayExpressionWithImplicitCast extends BinaryExpression > SELECT _FUNC_(NULL); -1 """) -case class Size( -child: Expression, -legacySizeOfNull: Boolean) - extends UnaryExpression with ExpectsInputTypes { +case class Size(child: Expression) extends UnaryExpression with ExpectsInputTypes { - def this(child: Expression) = -this( - child, - legacySizeOfNull = SQLConf.get.getConf(SQLConf.LEGACY_SIZE_OF_NULL)) + val legacySizeOfNull = SQLConf.get.legacySizeOfNull override def dataType: DataType = IntegerType override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(ArrayType, MapType)) http://git-wip-us.apache.org/repos/asf/spark/blob/e008ad17/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala index 8cd8605..63943b1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala @@ -514,10 +514,11 @@ case class JsonToStructs( schema: DataType, options: Map[String, String], child: Expression, -timeZoneId: Option[String], -forceNullableSchema: Boolean) +timeZoneId: Option[String] = None) extends UnaryExpression with TimeZoneAwareExpression with CodegenFallback with ExpectsInputTypes { + val forceNullableSchema = SQLConf.get.getConf(SQLConf.FROM_JSON_FORCE_NULLABLE_SCHEMA) + // The JSON input data might be missing certain fields. We force the nullability // of the user-provided schema to avoid data corruptions. In particular, the parquet-mr encoder // can generate incorrect files if values are missing in columns declared as non-nullable. @@ -531,8 +532,7 @@ case class JsonToStructs( schema = JsonExprUtils.evalSchemaExpr(schema), options = options, child = child, - timeZoneId = None, - forceNullableSchema = SQLConf.get.getConf(SQLConf.FROM_JSON_FORCE_NULLABLE_SCHEMA)) + timeZoneId = None) def this(child: Expression, schema: Expression) = this(child, schema, Map.empty[String, String]) @@ -541,13 +541,7 @@ case class JsonToStructs( schema = JsonExprUtils.evalSchemaExpr(schema), options = JsonExprUtils.convertToMapData(options), child = child, - timeZoneId = None, - forceNullableSchema = SQLConf.get.getConf(SQLConf.FROM_JSON_FORCE_NULLABLE_SCHEMA)) - - // Used in `org.apache.spark.sql.functions` - def this(schema: DataType, options: Map[String,
spark git commit: [SPARK-24761][SQL] Adding of isModifiable() to RuntimeConfig
Repository: spark Updated Branches: refs/heads/master e008ad175 -> 3ab48f985 [SPARK-24761][SQL] Adding of isModifiable() to RuntimeConfig ## What changes were proposed in this pull request? In the PR, I propose to extend `RuntimeConfig` by new method `isModifiable()` which returns `true` if a config parameter can be modified at runtime (for current session state). For static SQL and core parameters, the method returns `false`. ## How was this patch tested? Added new test to `RuntimeConfigSuite` for checking Spark core and SQL parameters. Author: Maxim Gekk Closes #21730 from MaxGekk/is-modifiable. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3ab48f98 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3ab48f98 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3ab48f98 Branch: refs/heads/master Commit: 3ab48f985c7f96bc9143caad99bf3df7cc984583 Parents: e008ad1 Author: Maxim Gekk Authored: Wed Jul 11 17:38:43 2018 -0700 Committer: Xiao Li Committed: Wed Jul 11 17:38:43 2018 -0700 -- python/pyspark/sql/conf.py| 8 .../scala/org/apache/spark/sql/internal/SQLConf.scala | 4 .../scala/org/apache/spark/sql/RuntimeConfig.scala| 11 +++ .../org/apache/spark/sql/RuntimeConfigSuite.scala | 14 ++ 4 files changed, 37 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/3ab48f98/python/pyspark/sql/conf.py -- diff --git a/python/pyspark/sql/conf.py b/python/pyspark/sql/conf.py index db49040..f80bf59 100644 --- a/python/pyspark/sql/conf.py +++ b/python/pyspark/sql/conf.py @@ -63,6 +63,14 @@ class RuntimeConfig(object): raise TypeError("expected %s '%s' to be a string (was '%s')" % (identifier, obj, type(obj).__name__)) +@ignore_unicode_prefix +@since(2.4) +def isModifiable(self, key): +"""Indicates whether the configuration property with the given key +is modifiable in the current session. +""" +return self._jconf.isModifiable(key) + def _test(): import os http://git-wip-us.apache.org/repos/asf/spark/blob/3ab48f98/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index ae56cc9..14dd528 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -1907,4 +1907,8 @@ class SQLConf extends Serializable with Logging { } cloned } + + def isModifiable(key: String): Boolean = { +sqlConfEntries.containsKey(key) && !staticConfKeys.contains(key) + } } http://git-wip-us.apache.org/repos/asf/spark/blob/3ab48f98/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala index b352e33..3c39579 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala @@ -133,6 +133,17 @@ class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) { } /** + * Indicates whether the configuration property with the given key + * is modifiable in the current session. + * + * @return `true` if the configuration property is modifiable. For static SQL, Spark Core, + * invalid (not existing) and other non-modifiable configuration properties, + * the returned value is `false`. + * @since 2.4.0 + */ + def isModifiable(key: String): Boolean = sqlConf.isModifiable(key) + + /** * Returns whether a particular key is set. */ protected[sql] def contains(key: String): Boolean = { http://git-wip-us.apache.org/repos/asf/spark/blob/3ab48f98/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala -- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala index cfe2e9f..cdcea09 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala @@ -54,4 +54,18 @@ class RuntimeConfigSuite extends SparkFunSuite { conf.get("k1") } } + + test("SPARK-24761: is a config
[1/2] spark git commit: [SPARK-24768][SQL] Have a built-in AVRO data source implementation
Repository: spark Updated Branches: refs/heads/master 1055c94cd -> 395860a98 http://git-wip-us.apache.org/repos/asf/spark/blob/395860a9/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala -- diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala new file mode 100644 index 000..c6c1e40 --- /dev/null +++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala @@ -0,0 +1,812 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.avro + +import java.io._ +import java.nio.file.Files +import java.sql.{Date, Timestamp} +import java.util.{TimeZone, UUID} + +import scala.collection.JavaConverters._ + +import org.apache.avro.Schema +import org.apache.avro.Schema.{Field, Type} +import org.apache.avro.file.DataFileWriter +import org.apache.avro.generic.{GenericData, GenericDatumWriter, GenericRecord} +import org.apache.avro.generic.GenericData.{EnumSymbol, Fixed} +import org.apache.commons.io.FileUtils + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql._ +import org.apache.spark.sql.avro.SchemaConverters.IncompatibleSchemaException +import org.apache.spark.sql.types._ + +class AvroSuite extends SparkFunSuite { + val episodesFile = "src/test/resources/episodes.avro" + val testFile = "src/test/resources/test.avro" + + private var spark: SparkSession = _ + + override protected def beforeAll(): Unit = { +super.beforeAll() +spark = SparkSession.builder() + .master("local[2]") + .appName("AvroSuite") + .config("spark.sql.files.maxPartitionBytes", 1024) + .getOrCreate() + } + + override protected def afterAll(): Unit = { +try { + spark.sparkContext.stop() +} finally { + super.afterAll() +} + } + + test("reading from multiple paths") { +val df = spark.read.avro(episodesFile, episodesFile) +assert(df.count == 16) + } + + test("reading and writing partitioned data") { +val df = spark.read.avro(episodesFile) +val fields = List("title", "air_date", "doctor") +for (field <- fields) { + TestUtils.withTempDir { dir => +val outputDir = s"$dir/${UUID.randomUUID}" +df.write.partitionBy(field).avro(outputDir) +val input = spark.read.avro(outputDir) +// makes sure that no fields got dropped. +// We convert Rows to Seqs in order to work around SPARK-10325 +assert(input.select(field).collect().map(_.toSeq).toSet === + df.select(field).collect().map(_.toSeq).toSet) + } +} + } + + test("request no fields") { +val df = spark.read.avro(episodesFile) +df.registerTempTable("avro_table") +assert(spark.sql("select count(*) from avro_table").collect().head === Row(8)) + } + + test("convert formats") { +TestUtils.withTempDir { dir => + val df = spark.read.avro(episodesFile) + df.write.parquet(dir.getCanonicalPath) + assert(spark.read.parquet(dir.getCanonicalPath).count() === df.count) +} + } + + test("rearrange internal schema") { +TestUtils.withTempDir { dir => + val df = spark.read.avro(episodesFile) + df.select("doctor", "title").write.avro(dir.getCanonicalPath) +} + } + + test("test NULL avro type") { +TestUtils.withTempDir { dir => + val fields = Seq(new Field("null", Schema.create(Type.NULL), "doc", null)).asJava + val schema = Schema.createRecord("name", "docs", "namespace", false) + schema.setFields(fields) + val datumWriter = new GenericDatumWriter[GenericRecord](schema) + val dataFileWriter = new DataFileWriter[GenericRecord](datumWriter) + dataFileWriter.create(schema, new File(s"$dir.avro")) + val avroRec = new GenericData.Record(schema) + avroRec.put("null", null) + dataFileWriter.append(avroRec) + dataFileWriter.flush() + dataFileWriter.close() + + intercept[IncompatibleSchemaException] { +spark.read.avro(s"$dir.avro") + } +} + } + + test("union(int, long) is read as long") { +TestUtils.withTempDir { dir => + val avroSchema
[2/2] spark git commit: [SPARK-24768][SQL] Have a built-in AVRO data source implementation
[SPARK-24768][SQL] Have a built-in AVRO data source implementation ## What changes were proposed in this pull request? Apache Avro (https://avro.apache.org) is a popular data serialization format. It is widely used in the Spark and Hadoop ecosystem, especially for Kafka-based data pipelines. Using the external package https://github.com/databricks/spark-avro, Spark SQL can read and write the avro data. Making spark-Avro built-in can provide a better experience for first-time users of Spark SQL and structured streaming. We expect the built-in Avro data source can further improve the adoption of structured streaming. The proposal is to inline code from spark-avro package (https://github.com/databricks/spark-avro). The target release is Spark 2.4. [Built-in AVRO Data Source In Spark 2.4.pdf](https://github.com/apache/spark/files/2181511/Built-in.AVRO.Data.Source.In.Spark.2.4.pdf) ## How was this patch tested? Unit test Author: Gengliang Wang Closes #21742 from gengliangwang/export_avro. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/395860a9 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/395860a9 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/395860a9 Branch: refs/heads/master Commit: 395860a986987886df6d60fd9b26afd818b2cb39 Parents: 1055c94 Author: Gengliang Wang Authored: Thu Jul 12 13:55:25 2018 -0700 Committer: Xiao Li Committed: Thu Jul 12 13:55:25 2018 -0700 -- dev/run-tests.py| 2 +- dev/sparktestsupport/modules.py | 10 + external/avro/pom.xml | 73 ++ apache.spark.sql.sources.DataSourceRegister | 1 + .../apache/spark/sql/avro/AvroFileFormat.scala | 289 +++ .../spark/sql/avro/AvroOutputWriter.scala | 164 .../sql/avro/AvroOutputWriterFactory.scala | 38 + .../spark/sql/avro/SchemaConverters.scala | 406 ++ .../org/apache/spark/sql/avro/package.scala | 39 + external/avro/src/test/resources/episodes.avro | Bin 0 -> 597 bytes .../avro/src/test/resources/log4j.properties| 49 ++ .../test-random-partitioned/part-r-0.avro | Bin 0 -> 1768 bytes .../test-random-partitioned/part-r-1.avro | Bin 0 -> 2313 bytes .../test-random-partitioned/part-r-2.avro | Bin 0 -> 1621 bytes .../test-random-partitioned/part-r-3.avro | Bin 0 -> 2117 bytes .../test-random-partitioned/part-r-4.avro | Bin 0 -> 3282 bytes .../test-random-partitioned/part-r-5.avro | Bin 0 -> 1550 bytes .../test-random-partitioned/part-r-6.avro | Bin 0 -> 1729 bytes .../test-random-partitioned/part-r-7.avro | Bin 0 -> 1897 bytes .../test-random-partitioned/part-r-8.avro | Bin 0 -> 3420 bytes .../test-random-partitioned/part-r-9.avro | Bin 0 -> 1796 bytes .../test-random-partitioned/part-r-00010.avro | Bin 0 -> 3872 bytes external/avro/src/test/resources/test.avro | Bin 0 -> 1365 bytes external/avro/src/test/resources/test.avsc | 53 ++ external/avro/src/test/resources/test.json | 42 + .../org/apache/spark/sql/avro/AvroSuite.scala | 812 +++ .../avro/SerializableConfigurationSuite.scala | 50 ++ .../org/apache/spark/sql/avro/TestUtils.scala | 156 pom.xml | 1 + project/SparkBuild.scala| 12 +- 30 files changed, 2191 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/395860a9/dev/run-tests.py -- diff --git a/dev/run-tests.py b/dev/run-tests.py index cd45908..d9d3789 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -110,7 +110,7 @@ def determine_modules_to_test(changed_modules): ['graphx', 'examples'] >>> x = [x.name for x in determine_modules_to_test([modules.sql])] >>> x # doctest: +NORMALIZE_WHITESPACE -['sql', 'hive', 'mllib', 'sql-kafka-0-10', 'examples', 'hive-thriftserver', +['sql', 'avro', 'hive', 'mllib', 'sql-kafka-0-10', 'examples', 'hive-thriftserver', 'pyspark-sql', 'repl', 'sparkr', 'pyspark-mllib', 'pyspark-ml'] """ modules_to_test = set() http://git-wip-us.apache.org/repos/asf/spark/blob/395860a9/dev/sparktestsupport/modules.py -- diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index dfea762..2aa3555 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -170,6 +170,16 @@ hive_thriftserver = Module( ] ) +avro = Module( +name="avro", +dependencies=[sql], +source_file_regexes=[ +"external/avro", +], +sbt_test_goals=[ +"avro/test", +] +) sql_kafka = Module(
spark git commit: [SPARK-23007][SQL][TEST] Add read schema suite for file-based data sources
Repository: spark Updated Branches: refs/heads/master 395860a98 -> 07704c971 [SPARK-23007][SQL][TEST] Add read schema suite for file-based data sources ## What changes were proposed in this pull request? The reader schema is said to be evolved (or projected) when it changed after the data is written. The followings are already supported in file-based data sources. Note that partition columns are not maintained in files. In this PR, `column` means `non-partition column`. 1. Add a column 2. Hide a column 3. Change a column position 4. Change a column type (upcast) This issue aims to guarantee users a backward-compatible read-schema test coverage on file-based data sources and to prevent future regressions by *adding read schema tests explicitly*. Here, we consider safe changes without data loss. For example, data type change should be from small types to larger types like `int`-to-`long`, not vice versa. As of today, in the master branch, file-based data sources have the following coverage. File Format | Coverage | Note --- | -- | TEXT | N/A| Schema consists of a single string column. CSV| 1, 2, 4| JSON | 1, 2, 3, 4| ORC| 1, 2, 3, 4| Native vectorized ORC reader has the widest coverage among ORC formats. PARQUET | 1, 2, 3| ## How was this patch tested? Pass the Jenkins with newly added test suites. Author: Dongjoon Hyun Closes #20208 from dongjoon-hyun/SPARK-SCHEMA-EVOLUTION. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/07704c97 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/07704c97 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/07704c97 Branch: refs/heads/master Commit: 07704c971cbc92bff15e15f8c42fab9afaab3ef7 Parents: 395860a Author: Dongjoon Hyun Authored: Thu Jul 12 14:08:49 2018 -0700 Committer: Xiao Li Committed: Thu Jul 12 14:08:49 2018 -0700 -- .../execution/datasources/ReadSchemaSuite.scala | 181 +++ .../execution/datasources/ReadSchemaTest.scala | 493 +++ 2 files changed, 674 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/07704c97/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaSuite.scala -- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaSuite.scala new file mode 100644 index 000..23c58e1 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaSuite.scala @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.datasources + +import org.apache.spark.sql.internal.SQLConf + +/** + * Read schema suites have the following hierarchy and aims to guarantee users + * a backward-compatible read-schema change coverage on file-based data sources, and + * to prevent future regressions. + * + * ReadSchemaSuite + * -> CSVReadSchemaSuite + * -> HeaderCSVReadSchemaSuite + * + * -> JsonReadSchemaSuite + * + * -> OrcReadSchemaSuite + * -> VectorizedOrcReadSchemaSuite + * + * -> ParquetReadSchemaSuite + * -> VectorizedParquetReadSchemaSuite + * -> MergedParquetReadSchemaSuite + */ + +/** + * All file-based data sources supports column addition and removal at the end. + */ +abstract class ReadSchemaSuite + extends AddColumnTest + with HideColumnAtTheEndTest { + + var originalConf: Boolean = _ +} + +class CSVReadSchemaSuite + extends ReadSchemaSuite + with IntegralTypeTest + with ToDoubleTypeTest + with ToDecimalTypeTest + with ToStringTypeTest { + + override val format: String = "csv" +} + +class HeaderCSVReadSchemaSuite + extends ReadSchemaSuite + with IntegralTypeTest + with ToDoubleTypeTest + with ToDecimalTypeTest + with ToStringTyp
spark git commit: [SPARK-24208][SQL][FOLLOWUP] Move test cases to proper locations
Repository: spark Updated Branches: refs/heads/master 07704c971 -> 11384893b [SPARK-24208][SQL][FOLLOWUP] Move test cases to proper locations ## What changes were proposed in this pull request? The PR is a followup to move the test cases introduced by the original PR in their proper location. ## How was this patch tested? moved UTs Author: Marco Gaido Closes #21751 from mgaido91/SPARK-24208_followup. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/11384893 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/11384893 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/11384893 Branch: refs/heads/master Commit: 11384893b6ad09c0c8bc6a350bb9540d0d704bb4 Parents: 07704c9 Author: Marco Gaido Authored: Thu Jul 12 15:13:26 2018 -0700 Committer: Xiao Li Committed: Thu Jul 12 15:13:26 2018 -0700 -- python/pyspark/sql/tests.py | 32 ++-- .../sql/catalyst/analysis/AnalysisSuite.scala | 18 +++ .../apache/spark/sql/GroupedDatasetSuite.scala | 12 3 files changed, 34 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/11384893/python/pyspark/sql/tests.py -- diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py index 4404dbe..565654e 100644 --- a/python/pyspark/sql/tests.py +++ b/python/pyspark/sql/tests.py @@ -5471,6 +5471,22 @@ class GroupedMapPandasUDFTests(ReusedSQLTestCase): self.assertEqual(r.a, 'hi') self.assertEqual(r.b, 1) +def test_self_join_with_pandas(self): +import pyspark.sql.functions as F + +@F.pandas_udf('key long, col string', F.PandasUDFType.GROUPED_MAP) +def dummy_pandas_udf(df): +return df[['key', 'col']] + +df = self.spark.createDataFrame([Row(key=1, col='A'), Row(key=1, col='B'), + Row(key=2, col='C')]) +df_with_pandas = df.groupBy('key').apply(dummy_pandas_udf) + +# this was throwing an AnalysisException before SPARK-24208 +res = df_with_pandas.alias('temp0').join(df_with_pandas.alias('temp1'), + F.col('temp0.key') == F.col('temp1.key')) +self.assertEquals(res.count(), 5) + @unittest.skipIf( not _have_pandas or not _have_pyarrow, @@ -5925,22 +5941,6 @@ class GroupedAggPandasUDFTests(ReusedSQLTestCase): 'mixture.*aggregate function.*group aggregate pandas UDF'): df.groupby(df.id).agg(mean_udf(df.v), mean(df.v)).collect() -def test_self_join_with_pandas(self): -import pyspark.sql.functions as F - -@F.pandas_udf('key long, col string', F.PandasUDFType.GROUPED_MAP) -def dummy_pandas_udf(df): -return df[['key', 'col']] - -df = self.spark.createDataFrame([Row(key=1, col='A'), Row(key=1, col='B'), - Row(key=2, col='C')]) -dfWithPandas = df.groupBy('key').apply(dummy_pandas_udf) - -# this was throwing an AnalysisException before SPARK-24208 -res = dfWithPandas.alias('temp0').join(dfWithPandas.alias('temp1'), - F.col('temp0.key') == F.col('temp1.key')) -self.assertEquals(res.count(), 5) - @unittest.skipIf( not _have_pandas or not _have_pyarrow, http://git-wip-us.apache.org/repos/asf/spark/blob/11384893/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala -- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala index cd85795..bbcdf6c 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala @@ -21,6 +21,7 @@ import java.util.TimeZone import org.scalatest.Matchers +import org.apache.spark.api.python.PythonEvalType import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.dsl.plans._ @@ -557,4 +558,21 @@ class AnalysisSuite extends AnalysisTest with Matchers { SubqueryAlias("tbl", testRelation))) assertAnalysisError(barrier, Seq("cannot resolve '`tbl.b`'")) } + + test("SPARK-24208: analysis fails on self-join with FlatMapGroupsInPandas") { +val pythonUdf = PythonUDF("pyUDF", null, + StructType(Seq(StructField("a", LongType))), + Seq.empty, + PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF, +
spark git commit: [SPARK-24790][SQL] Allow complex aggregate expressions in Pivot
Repository: spark Updated Branches: refs/heads/master 11384893b -> 75725057b [SPARK-24790][SQL] Allow complex aggregate expressions in Pivot ## What changes were proposed in this pull request? Relax the check to allow complex aggregate expressions, like `ceil(sum(col1))` or `sum(col1) + 1`, which roughly means any aggregate expression that could appear in an Aggregate plan except pandas UDF (due to the fact that it is not supported in pivot yet). ## How was this patch tested? Added 2 tests in pivot.sql Author: maryannxue Closes #21753 from maryannxue/pivot-relax-syntax. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/75725057 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/75725057 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/75725057 Branch: refs/heads/master Commit: 75725057b3ffdb0891844b10bd707bb0830f92ca Parents: 1138489 Author: maryannxue Authored: Thu Jul 12 16:54:03 2018 -0700 Committer: Xiao Li Committed: Thu Jul 12 16:54:03 2018 -0700 -- .../spark/sql/catalyst/analysis/Analyzer.scala | 24 +++--- .../test/resources/sql-tests/inputs/pivot.sql | 18 +++ .../resources/sql-tests/results/pivot.sql.out | 34 ++-- 3 files changed, 62 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/75725057/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index c078efd..9749893 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -509,12 +509,7 @@ class Analyzer( || !p.pivotColumn.resolved => p case Pivot(groupByExprsOpt, pivotColumn, pivotValues, aggregates, child) => // Check all aggregate expressions. -aggregates.foreach { e => - if (!isAggregateExpression(e)) { - throw new AnalysisException( -s"Aggregate expression required for pivot, found '$e'") - } -} +aggregates.foreach(checkValidAggregateExpression) // Group-by expressions coming from SQL are implicit and need to be deduced. val groupByExprs = groupByExprsOpt.getOrElse( (child.outputSet -- aggregates.flatMap(_.references) -- pivotColumn.references).toSeq) @@ -586,12 +581,17 @@ class Analyzer( } } -private def isAggregateExpression(expr: Expression): Boolean = { - expr match { -case Alias(e, _) => isAggregateExpression(e) -case AggregateExpression(_, _, _, _) => true -case _ => false - } +// Support any aggregate expression that can appear in an Aggregate plan except Pandas UDF. +// TODO: Support Pandas UDF. +private def checkValidAggregateExpression(expr: Expression): Unit = expr match { + case _: AggregateExpression => // OK and leave the argument check to CheckAnalysis. + case expr: PythonUDF if PythonUDF.isGroupedAggPandasUDF(expr) => +failAnalysis("Pandas UDF aggregate expressions are currently not supported in pivot.") + case e: Attribute => +failAnalysis( + s"Aggregate expression required for pivot, but '${e.sql}' " + + s"did not appear in any aggregate function.") + case e => e.children.foreach(checkValidAggregateExpression) } } http://git-wip-us.apache.org/repos/asf/spark/blob/75725057/sql/core/src/test/resources/sql-tests/inputs/pivot.sql -- diff --git a/sql/core/src/test/resources/sql-tests/inputs/pivot.sql b/sql/core/src/test/resources/sql-tests/inputs/pivot.sql index 01dea6c..b3d53ad 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/pivot.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/pivot.sql @@ -111,3 +111,21 @@ PIVOT ( sum(earnings) FOR year IN (2012, 2013) ); + +-- pivot with complex aggregate expressions +SELECT * FROM ( + SELECT year, course, earnings FROM courseSales +) +PIVOT ( + ceil(sum(earnings)), avg(earnings) + 1 as a1 + FOR course IN ('dotNET', 'Java') +); + +-- pivot with invalid arguments in aggregate expressions +SELECT * FROM ( + SELECT year, course, earnings FROM courseSales +) +PIVOT ( + sum(avg(earnings)) + FOR course IN ('dotNET', 'Java') +); http://git-wip-us.apache.org/repos/asf/spark/blob/75725057/sql/core/src/test/resources/sql-tests/results/pivot.sql.out -- diff --git a/sql
spark git commit: [SPARK-23486] cache the function name from the external catalog for lookupFunctions
Repository: spark Updated Branches: refs/heads/master e0f4f206b -> 0ce11d0e3 [SPARK-23486] cache the function name from the external catalog for lookupFunctions ## What changes were proposed in this pull request? This PR will cache the function name from external catalog, it is used by lookupFunctions in the analyzer, and it is cached for each query plan. The original problem is reported in the [ spark-19737](https://issues.apache.org/jira/browse/SPARK-19737) ## How was this patch tested? create new test file LookupFunctionsSuite and add test case in SessionCatalogSuite Author: Kevin Yu Closes #20795 from kevinyu98/spark-23486. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0ce11d0e Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0ce11d0e Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0ce11d0e Branch: refs/heads/master Commit: 0ce11d0e3a7c8c48d9f7305d2dd39c7b281b2a53 Parents: e0f4f20 Author: Kevin Yu Authored: Thu Jul 12 22:20:06 2018 -0700 Committer: Xiao Li Committed: Thu Jul 12 22:20:06 2018 -0700 -- .../spark/sql/catalyst/analysis/Analyzer.scala | 45 ++-- .../sql/catalyst/catalog/SessionCatalog.scala | 16 +++ .../analysis/LookupFunctionsSuite.scala | 104 +++ .../catalyst/catalog/SessionCatalogSuite.scala | 36 +++ .../spark/sql/hive/HiveSessionCatalog.scala | 4 + 5 files changed, 199 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/0ce11d0e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 9749893..960ee27 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -17,6 +17,9 @@ package org.apache.spark.sql.catalyst.analysis +import java.util.Locale + +import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.util.Random @@ -1208,16 +1211,46 @@ class Analyzer( * only performs simple existence check according to the function identifier to quickly identify * undefined functions without triggering relation resolution, which may incur potentially * expensive partition/schema discovery process in some cases. - * + * In order to avoid duplicate external functions lookup, the external function identifier will + * store in the local hash set externalFunctionNameSet. * @see [[ResolveFunctions]] * @see https://issues.apache.org/jira/browse/SPARK-19737 */ object LookupFunctions extends Rule[LogicalPlan] { -override def apply(plan: LogicalPlan): LogicalPlan = plan.transformAllExpressions { - case f: UnresolvedFunction if !catalog.functionExists(f.name) => -withPosition(f) { - throw new NoSuchFunctionException(f.name.database.getOrElse("default"), f.name.funcName) -} +override def apply(plan: LogicalPlan): LogicalPlan = { + val externalFunctionNameSet = new mutable.HashSet[FunctionIdentifier]() + plan.transformAllExpressions { +case f: UnresolvedFunction + if externalFunctionNameSet.contains(normalizeFuncName(f.name)) => f +case f: UnresolvedFunction if catalog.isRegisteredFunction(f.name) => f +case f: UnresolvedFunction if catalog.isPersistentFunction(f.name) => + externalFunctionNameSet.add(normalizeFuncName(f.name)) + f +case f: UnresolvedFunction => + withPosition(f) { +throw new NoSuchFunctionException(f.name.database.getOrElse(catalog.getCurrentDatabase), + f.name.funcName) + } + } +} + +def normalizeFuncName(name: FunctionIdentifier): FunctionIdentifier = { + val funcName = if (conf.caseSensitiveAnalysis) { +name.funcName + } else { +name.funcName.toLowerCase(Locale.ROOT) + } + + val databaseName = name.database match { +case Some(a) => formatDatabaseName(a) +case None => catalog.getCurrentDatabase + } + + FunctionIdentifier(funcName, Some(databaseName)) +} + +protected def formatDatabaseName(name: String): String = { + if (conf.caseSensitiveAnalysis) name else name.toLowerCase(Locale.ROOT) } } http://git-wip-us.apache.org/repos/asf/spark/blob/0ce11d0e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala -- diff --git a/sql/catalyst/src/main/scal
spark git commit: [SPARK-24781][SQL] Using a reference from Dataset in Filter/Sort might not work
Repository: spark Updated Branches: refs/heads/master 0f24c6f8a -> dfd7ac988 [SPARK-24781][SQL] Using a reference from Dataset in Filter/Sort might not work ## What changes were proposed in this pull request? When we use a reference from Dataset in filter or sort, which was not used in the prior select, an AnalysisException occurs, e.g., ```scala val df = Seq(("test1", 0), ("test2", 1)).toDF("name", "id") df.select(df("name")).filter(df("id") === 0).show() ``` ```scala org.apache.spark.sql.AnalysisException: Resolved attribute(s) id#6 missing from name#5 in operator !Filter (id#6 = 0).;; !Filter (id#6 = 0) +- AnalysisBarrier +- Project [name#5] +- Project [_1#2 AS name#5, _2#3 AS id#6] +- LocalRelation [_1#2, _2#3] ``` This change updates the rule `ResolveMissingReferences` so `Filter` and `Sort` with non-empty `missingInputs` will also be transformed. ## How was this patch tested? Added tests. Author: Liang-Chi Hsieh Closes #21745 from viirya/SPARK-24781. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/dfd7ac98 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/dfd7ac98 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/dfd7ac98 Branch: refs/heads/master Commit: dfd7ac9887f89b9b51b7b143ab54d01f11cfcdb5 Parents: 0f24c6f Author: Liang-Chi Hsieh Authored: Fri Jul 13 08:25:00 2018 -0700 Committer: Xiao Li Committed: Fri Jul 13 08:25:00 2018 -0700 -- .../spark/sql/catalyst/analysis/Analyzer.scala | 30 +++- .../org/apache/spark/sql/DataFrameSuite.scala | 25 2 files changed, 48 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/dfd7ac98/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 960ee27..36f14cc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -1132,7 +1132,8 @@ class Analyzer( case sa @ Sort(_, _, AnalysisBarrier(child: Aggregate)) => sa case sa @ Sort(_, _, child: Aggregate) => sa - case s @ Sort(order, _, child) if !s.resolved && child.resolved => + case s @ Sort(order, _, child) + if (!s.resolved || s.missingInput.nonEmpty) && child.resolved => val (newOrder, newChild) = resolveExprsAndAddMissingAttrs(order, child) val ordering = newOrder.map(_.asInstanceOf[SortOrder]) if (child.output == newChild.output) { @@ -1143,7 +1144,7 @@ class Analyzer( Project(child.output, newSort) } - case f @ Filter(cond, child) if !f.resolved && child.resolved => + case f @ Filter(cond, child) if (!f.resolved || f.missingInput.nonEmpty) && child.resolved => val (newCond, newChild) = resolveExprsAndAddMissingAttrs(Seq(cond), child) if (child.output == newChild.output) { f.copy(condition = newCond.head) @@ -1154,10 +1155,17 @@ class Analyzer( } } +/** + * This method tries to resolve expressions and find missing attributes recursively. Specially, + * when the expressions used in `Sort` or `Filter` contain unresolved attributes or resolved + * attributes which are missed from child output. This method tries to find the missing + * attributes out and add into the projection. + */ private def resolveExprsAndAddMissingAttrs( exprs: Seq[Expression], plan: LogicalPlan): (Seq[Expression], LogicalPlan) = { - if (exprs.forall(_.resolved)) { -// All given expressions are resolved, no need to continue anymore. + // Missing attributes can be unresolved attributes or resolved attributes which are not in + // the output attributes of the plan. + if (exprs.forall(e => e.resolved && e.references.subsetOf(plan.outputSet))) { (exprs, plan) } else { plan match { @@ -1168,15 +1176,19 @@ class Analyzer( (newExprs, AnalysisBarrier(newChild)) case p: Project => +// Resolving expressions against current plan. val maybeResolvedExprs = exprs.map(resolveExpression(_, p)) +// Recursively resolving expressions on the child of current plan. val (newExprs, newChild) = resolveExprsAndAddMissingAttrs(maybeResolvedExprs, p.child) -val missingAttrs = AttributeSet(newExprs) -- AttributeSet(maybeResolvedExprs) +// If some attributes used by expressions are resolvable on
spark git commit: [SPARK-24781][SQL] Using a reference from Dataset in Filter/Sort might not work
Repository: spark Updated Branches: refs/heads/branch-2.3 32429256f -> 9cf375f5b [SPARK-24781][SQL] Using a reference from Dataset in Filter/Sort might not work ## What changes were proposed in this pull request? When we use a reference from Dataset in filter or sort, which was not used in the prior select, an AnalysisException occurs, e.g., ```scala val df = Seq(("test1", 0), ("test2", 1)).toDF("name", "id") df.select(df("name")).filter(df("id") === 0).show() ``` ```scala org.apache.spark.sql.AnalysisException: Resolved attribute(s) id#6 missing from name#5 in operator !Filter (id#6 = 0).;; !Filter (id#6 = 0) +- AnalysisBarrier +- Project [name#5] +- Project [_1#2 AS name#5, _2#3 AS id#6] +- LocalRelation [_1#2, _2#3] ``` This change updates the rule `ResolveMissingReferences` so `Filter` and `Sort` with non-empty `missingInputs` will also be transformed. ## How was this patch tested? Added tests. Author: Liang-Chi Hsieh Closes #21745 from viirya/SPARK-24781. (cherry picked from commit dfd7ac9887f89b9b51b7b143ab54d01f11cfcdb5) Signed-off-by: Xiao Li Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9cf375f5 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9cf375f5 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9cf375f5 Branch: refs/heads/branch-2.3 Commit: 9cf375f5be3d359912bde9b6ba5766425e8cf3bb Parents: 3242925 Author: Liang-Chi Hsieh Authored: Fri Jul 13 08:25:00 2018 -0700 Committer: Xiao Li Committed: Fri Jul 13 08:25:14 2018 -0700 -- .../spark/sql/catalyst/analysis/Analyzer.scala | 30 +++- .../org/apache/spark/sql/DataFrameSuite.scala | 25 2 files changed, 48 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/9cf375f5/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index a584cb8..2858bee 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -1114,7 +1114,8 @@ class Analyzer( case sa @ Sort(_, _, AnalysisBarrier(child: Aggregate)) => sa case sa @ Sort(_, _, child: Aggregate) => sa - case s @ Sort(order, _, child) if !s.resolved && child.resolved => + case s @ Sort(order, _, child) + if (!s.resolved || s.missingInput.nonEmpty) && child.resolved => val (newOrder, newChild) = resolveExprsAndAddMissingAttrs(order, child) val ordering = newOrder.map(_.asInstanceOf[SortOrder]) if (child.output == newChild.output) { @@ -1125,7 +1126,7 @@ class Analyzer( Project(child.output, newSort) } - case f @ Filter(cond, child) if !f.resolved && child.resolved => + case f @ Filter(cond, child) if (!f.resolved || f.missingInput.nonEmpty) && child.resolved => val (newCond, newChild) = resolveExprsAndAddMissingAttrs(Seq(cond), child) if (child.output == newChild.output) { f.copy(condition = newCond.head) @@ -1136,10 +1137,17 @@ class Analyzer( } } +/** + * This method tries to resolve expressions and find missing attributes recursively. Specially, + * when the expressions used in `Sort` or `Filter` contain unresolved attributes or resolved + * attributes which are missed from child output. This method tries to find the missing + * attributes out and add into the projection. + */ private def resolveExprsAndAddMissingAttrs( exprs: Seq[Expression], plan: LogicalPlan): (Seq[Expression], LogicalPlan) = { - if (exprs.forall(_.resolved)) { -// All given expressions are resolved, no need to continue anymore. + // Missing attributes can be unresolved attributes or resolved attributes which are not in + // the output attributes of the plan. + if (exprs.forall(e => e.resolved && e.references.subsetOf(plan.outputSet))) { (exprs, plan) } else { plan match { @@ -1150,15 +1158,19 @@ class Analyzer( (newExprs, AnalysisBarrier(newChild)) case p: Project => +// Resolving expressions against current plan. val maybeResolvedExprs = exprs.map(resolveExpression(_, p)) +// Recursively resolving expressions on the child of current plan. val (newExprs, newChild) = resolveExprsAndAddMissingAttrs(maybeResolvedExprs, p.child) -val missingAttrs = AttributeSet(newExprs) -- Att
spark git commit: [SPARK-24776][SQL] Avro unit test: use SQLTestUtils and replace deprecated methods
Repository: spark Updated Branches: refs/heads/master dfd7ac988 -> c1b62e420 [SPARK-24776][SQL] Avro unit test: use SQLTestUtils and replace deprecated methods ## What changes were proposed in this pull request? Improve Avro unit test: 1. use QueryTest/SharedSQLContext/SQLTestUtils, instead of the duplicated test utils. 2. replace deprecated methods ## How was this patch tested? Unit test Author: Gengliang Wang Closes #21760 from gengliangwang/improve_avro_test. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c1b62e42 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c1b62e42 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c1b62e42 Branch: refs/heads/master Commit: c1b62e420a43aa7da36733ccdbec057d87ac1b43 Parents: dfd7ac9 Author: Gengliang Wang Authored: Fri Jul 13 08:55:46 2018 -0700 Committer: Xiao Li Committed: Fri Jul 13 08:55:46 2018 -0700 -- .../org/apache/spark/sql/avro/AvroSuite.scala | 114 +++--- .../org/apache/spark/sql/avro/TestUtils.scala | 156 --- 2 files changed, 53 insertions(+), 217 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/c1b62e42/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala -- diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala index c6c1e40..108b347 100644 --- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala +++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala @@ -31,32 +31,24 @@ import org.apache.avro.generic.{GenericData, GenericDatumWriter, GenericRecord} import org.apache.avro.generic.GenericData.{EnumSymbol, Fixed} import org.apache.commons.io.FileUtils -import org.apache.spark.SparkFunSuite import org.apache.spark.sql._ import org.apache.spark.sql.avro.SchemaConverters.IncompatibleSchemaException +import org.apache.spark.sql.test.{SharedSQLContext, SQLTestUtils} import org.apache.spark.sql.types._ -class AvroSuite extends SparkFunSuite { +class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils { val episodesFile = "src/test/resources/episodes.avro" val testFile = "src/test/resources/test.avro" - private var spark: SparkSession = _ - override protected def beforeAll(): Unit = { super.beforeAll() -spark = SparkSession.builder() - .master("local[2]") - .appName("AvroSuite") - .config("spark.sql.files.maxPartitionBytes", 1024) - .getOrCreate() - } - - override protected def afterAll(): Unit = { -try { - spark.sparkContext.stop() -} finally { - super.afterAll() -} +spark.conf.set("spark.sql.files.maxPartitionBytes", 1024) + } + + def checkReloadMatchesSaved(originalFile: String, newFile: String): Unit = { +val originalEntries = spark.read.avro(testFile).collect() +val newEntries = spark.read.avro(newFile) +checkAnswer(newEntries, originalEntries) } test("reading from multiple paths") { @@ -68,7 +60,7 @@ class AvroSuite extends SparkFunSuite { val df = spark.read.avro(episodesFile) val fields = List("title", "air_date", "doctor") for (field <- fields) { - TestUtils.withTempDir { dir => + withTempPath { dir => val outputDir = s"$dir/${UUID.randomUUID}" df.write.partitionBy(field).avro(outputDir) val input = spark.read.avro(outputDir) @@ -82,12 +74,12 @@ class AvroSuite extends SparkFunSuite { test("request no fields") { val df = spark.read.avro(episodesFile) -df.registerTempTable("avro_table") +df.createOrReplaceTempView("avro_table") assert(spark.sql("select count(*) from avro_table").collect().head === Row(8)) } test("convert formats") { -TestUtils.withTempDir { dir => +withTempPath { dir => val df = spark.read.avro(episodesFile) df.write.parquet(dir.getCanonicalPath) assert(spark.read.parquet(dir.getCanonicalPath).count() === df.count) @@ -95,15 +87,16 @@ class AvroSuite extends SparkFunSuite { } test("rearrange internal schema") { -TestUtils.withTempDir { dir => +withTempPath { dir => val df = spark.read.avro(episodesFile) df.select("doctor", "title").write.avro(dir.getCanonicalPath) } } test("test NULL avro type") { -TestUtils.withTempDir { dir => - val fields = Seq(new Field("null", Schema.create(Type.NULL), "doc", null)).asJava +withTempPath { dir => + val fields = +Seq(new Field("null", Schema.create(Type.NULL), "doc", null.asInstanceOf[Any])).asJava val schema = Schema.createRecord("name", "docs", "namespace", false) schema.setFields
spark git commit: Revert "[SPARK-24776][SQL] Avro unit test: use SQLTestUtils and replace deprecated methods"
Repository: spark Updated Branches: refs/heads/master c1b62e420 -> 3bcb1b481 Revert "[SPARK-24776][SQL] Avro unit test: use SQLTestUtils and replace deprecated methods" This reverts commit c1b62e420a43aa7da36733ccdbec057d87ac1b43. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3bcb1b48 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3bcb1b48 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3bcb1b48 Branch: refs/heads/master Commit: 3bcb1b481423aedf1ac531ad582c7cb8685f1e3c Parents: c1b62e4 Author: Xiao Li Authored: Fri Jul 13 10:06:26 2018 -0700 Committer: Xiao Li Committed: Fri Jul 13 10:06:26 2018 -0700 -- .../org/apache/spark/sql/avro/AvroSuite.scala | 114 +++--- .../org/apache/spark/sql/avro/TestUtils.scala | 156 +++ 2 files changed, 217 insertions(+), 53 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/3bcb1b48/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala -- diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala index 108b347..c6c1e40 100644 --- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala +++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala @@ -31,24 +31,32 @@ import org.apache.avro.generic.{GenericData, GenericDatumWriter, GenericRecord} import org.apache.avro.generic.GenericData.{EnumSymbol, Fixed} import org.apache.commons.io.FileUtils +import org.apache.spark.SparkFunSuite import org.apache.spark.sql._ import org.apache.spark.sql.avro.SchemaConverters.IncompatibleSchemaException -import org.apache.spark.sql.test.{SharedSQLContext, SQLTestUtils} import org.apache.spark.sql.types._ -class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils { +class AvroSuite extends SparkFunSuite { val episodesFile = "src/test/resources/episodes.avro" val testFile = "src/test/resources/test.avro" + private var spark: SparkSession = _ + override protected def beforeAll(): Unit = { super.beforeAll() -spark.conf.set("spark.sql.files.maxPartitionBytes", 1024) - } - - def checkReloadMatchesSaved(originalFile: String, newFile: String): Unit = { -val originalEntries = spark.read.avro(testFile).collect() -val newEntries = spark.read.avro(newFile) -checkAnswer(newEntries, originalEntries) +spark = SparkSession.builder() + .master("local[2]") + .appName("AvroSuite") + .config("spark.sql.files.maxPartitionBytes", 1024) + .getOrCreate() + } + + override protected def afterAll(): Unit = { +try { + spark.sparkContext.stop() +} finally { + super.afterAll() +} } test("reading from multiple paths") { @@ -60,7 +68,7 @@ class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils { val df = spark.read.avro(episodesFile) val fields = List("title", "air_date", "doctor") for (field <- fields) { - withTempPath { dir => + TestUtils.withTempDir { dir => val outputDir = s"$dir/${UUID.randomUUID}" df.write.partitionBy(field).avro(outputDir) val input = spark.read.avro(outputDir) @@ -74,12 +82,12 @@ class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils { test("request no fields") { val df = spark.read.avro(episodesFile) -df.createOrReplaceTempView("avro_table") +df.registerTempTable("avro_table") assert(spark.sql("select count(*) from avro_table").collect().head === Row(8)) } test("convert formats") { -withTempPath { dir => +TestUtils.withTempDir { dir => val df = spark.read.avro(episodesFile) df.write.parquet(dir.getCanonicalPath) assert(spark.read.parquet(dir.getCanonicalPath).count() === df.count) @@ -87,16 +95,15 @@ class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils { } test("rearrange internal schema") { -withTempPath { dir => +TestUtils.withTempDir { dir => val df = spark.read.avro(episodesFile) df.select("doctor", "title").write.avro(dir.getCanonicalPath) } } test("test NULL avro type") { -withTempPath { dir => - val fields = -Seq(new Field("null", Schema.create(Type.NULL), "doc", null.asInstanceOf[Any])).asJava +TestUtils.withTempDir { dir => + val fields = Seq(new Field("null", Schema.create(Type.NULL), "doc", null)).asJava val schema = Schema.createRecord("name", "docs", "namespace", false) schema.setFields(fields) val datumWriter = new GenericDatumWriter[GenericRecord](schema) @@ -115,11 +122,11 @@ class AvroSuite extends QueryTest wi
spark git commit: [SPARK-23831][SQL] Add org.apache.derby to IsolatedClientLoader
Repository: spark Updated Branches: refs/heads/master 3b6005b8a -> a75571b46 [SPARK-23831][SQL] Add org.apache.derby to IsolatedClientLoader ## What changes were proposed in this pull request? Add `org.apache.derby` to `IsolatedClientLoader`, otherwise it may throw an exception: ```scala ... [info] Cause: java.sql.SQLException: Failed to start database 'metastore_db' with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$12439ab23, see the next exception for details. [info] at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) [info] at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) [info] at org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source) [info] at org.apache.derby.impl.jdbc.EmbedConnection.bootDatabase(Unknown Source) [info] at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) [info] at org.apache.derby.jdbc.InternalDriver$1.run(Unknown Source) ... ``` ## How was this patch tested? unit tests and manual tests Author: Yuming Wang Closes #20944 from wangyum/SPARK-23831. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a75571b4 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a75571b4 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a75571b4 Branch: refs/heads/master Commit: a75571b46f813005a6d4b076ec39081ffab11844 Parents: 3b6005b Author: Yuming Wang Authored: Fri Jul 13 14:07:52 2018 -0700 Committer: Xiao Li Committed: Fri Jul 13 14:07:52 2018 -0700 -- .../apache/spark/sql/hive/client/IsolatedClientLoader.scala| 1 + .../org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala | 6 ++ 2 files changed, 7 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/a75571b4/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala -- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala index 2f34f69..6a90c44 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala @@ -182,6 +182,7 @@ private[hive] class IsolatedClientLoader( name.startsWith("org.slf4j") || name.startsWith("org.apache.log4j") || // log4j1.x name.startsWith("org.apache.logging.log4j") || // log4j2 +name.startsWith("org.apache.derby.") || name.startsWith("org.apache.spark.") || (sharesHadoopClasses && isHadoopClass) || name.startsWith("scala.") || http://git-wip-us.apache.org/repos/asf/spark/blob/a75571b4/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala -- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala index 0a522b6..1de258f 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala @@ -113,4 +113,10 @@ class HiveExternalCatalogSuite extends ExternalCatalogSuite { catalog.createDatabase(newDb("dbWithNullDesc").copy(description = null), ignoreIfExists = false) assert(catalog.getDatabase("dbWithNullDesc").description == "") } + + test("SPARK-23831: Add org.apache.derby to IsolatedClientLoader") { +val client1 = HiveUtils.newClientForMetadata(new SparkConf, new Configuration) +val client2 = HiveUtils.newClientForMetadata(new SparkConf, new Configuration) +assert(!client1.equals(client2)) + } } - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24776][SQL] Avro unit test: deduplicate code and replace deprecated methods
Repository: spark Updated Branches: refs/heads/master 43e4e851b -> 3e7dc8296 [SPARK-24776][SQL] Avro unit test: deduplicate code and replace deprecated methods ## What changes were proposed in this pull request? Improve Avro unit test: 1. use QueryTest/SharedSQLContext/SQLTestUtils, instead of the duplicated test utils. 2. replace deprecated methods This is a follow up PR for #21760, the PR passes pull request tests but failed in: https://amplab.cs.berkeley.edu/jenkins/view/Spark%20QA%20Compile/job/spark-master-compile-maven-hadoop-2.6/7842/ This PR is to fix it. ## How was this patch tested? Unit test. Compile with different commands: ``` ./build/mvn --force -DzincPort=3643 -DskipTests -Phadoop-2.6 -Phive-thriftserver -Pkinesis-asl -Pspark-ganglia-lgpl -Pmesos -Pyarn compile test-compile ./build/mvn --force -DzincPort=3643 -DskipTests -Phadoop-2.7 -Phive-thriftserver -Pkinesis-asl -Pspark-ganglia-lgpl -Pmesos -Pyarn compile test-compile ./build/mvn --force -DzincPort=3643 -DskipTests -Phadoop-3.1 -Phive-thriftserver -Pkinesis-asl -Pspark-ganglia-lgpl -Pmesos -Pyarn compile test-compile ``` Author: Gengliang Wang Closes #21768 from gengliangwang/improve_avro_test. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3e7dc829 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3e7dc829 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3e7dc829 Branch: refs/heads/master Commit: 3e7dc82960fd3339eee16d83df66761ae6e3fe3d Parents: 43e4e85 Author: Gengliang Wang Authored: Sat Jul 14 21:36:56 2018 -0700 Committer: Xiao Li Committed: Sat Jul 14 21:36:56 2018 -0700 -- .../org/apache/spark/sql/avro/AvroSuite.scala | 98 ++-- .../org/apache/spark/sql/avro/TestUtils.scala | 156 --- 2 files changed, 45 insertions(+), 209 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/3e7dc829/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala -- diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala index c6c1e40..4f94d82 100644 --- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala +++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala @@ -31,32 +31,24 @@ import org.apache.avro.generic.{GenericData, GenericDatumWriter, GenericRecord} import org.apache.avro.generic.GenericData.{EnumSymbol, Fixed} import org.apache.commons.io.FileUtils -import org.apache.spark.SparkFunSuite import org.apache.spark.sql._ import org.apache.spark.sql.avro.SchemaConverters.IncompatibleSchemaException +import org.apache.spark.sql.test.{SharedSQLContext, SQLTestUtils} import org.apache.spark.sql.types._ -class AvroSuite extends SparkFunSuite { +class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils { val episodesFile = "src/test/resources/episodes.avro" val testFile = "src/test/resources/test.avro" - private var spark: SparkSession = _ - override protected def beforeAll(): Unit = { super.beforeAll() -spark = SparkSession.builder() - .master("local[2]") - .appName("AvroSuite") - .config("spark.sql.files.maxPartitionBytes", 1024) - .getOrCreate() - } - - override protected def afterAll(): Unit = { -try { - spark.sparkContext.stop() -} finally { - super.afterAll() -} +spark.conf.set("spark.sql.files.maxPartitionBytes", 1024) + } + + def checkReloadMatchesSaved(originalFile: String, newFile: String): Unit = { +val originalEntries = spark.read.avro(testFile).collect() +val newEntries = spark.read.avro(newFile) +checkAnswer(newEntries, originalEntries) } test("reading from multiple paths") { @@ -68,7 +60,7 @@ class AvroSuite extends SparkFunSuite { val df = spark.read.avro(episodesFile) val fields = List("title", "air_date", "doctor") for (field <- fields) { - TestUtils.withTempDir { dir => + withTempPath { dir => val outputDir = s"$dir/${UUID.randomUUID}" df.write.partitionBy(field).avro(outputDir) val input = spark.read.avro(outputDir) @@ -82,12 +74,12 @@ class AvroSuite extends SparkFunSuite { test("request no fields") { val df = spark.read.avro(episodesFile) -df.registerTempTable("avro_table") +df.createOrReplaceTempView("avro_table") assert(spark.sql("select count(*) from avro_table").collect().head === Row(8)) } test("convert formats") { -TestUtils.withTempDir { dir => +withTempPath { dir => val df = spark.read.avro(episodesFile) df.write.parquet(dir.getCanonicalPath) assert(spark.read.parquet(dir.getCanonical
spark git commit: [SPARK-24807][CORE] Adding files/jars twice: output a warning and add a note
Repository: spark Updated Branches: refs/heads/master 3e7dc8296 -> 69993217f [SPARK-24807][CORE] Adding files/jars twice: output a warning and add a note ## What changes were proposed in this pull request? In the PR, I propose to output an warning if the `addFile()` or `addJar()` methods are callled more than once for the same path. Currently, overwriting of already added files is not supported. New comments and warning are reflected the existing behaviour. Author: Maxim Gekk Closes #21771 from MaxGekk/warning-on-adding-file. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/69993217 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/69993217 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/69993217 Branch: refs/heads/master Commit: 69993217fc4f5e5e41a297702389e86fe534dc2f Parents: 3e7dc82 Author: Maxim Gekk Authored: Sat Jul 14 22:07:49 2018 -0700 Committer: Xiao Li Committed: Sat Jul 14 22:07:49 2018 -0700 -- R/pkg/R/context.R | 2 ++ core/src/main/scala/org/apache/spark/SparkContext.scala | 12 .../org/apache/spark/api/java/JavaSparkContext.scala| 6 ++ python/pyspark/context.py | 4 4 files changed, 24 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/69993217/R/pkg/R/context.R -- diff --git a/R/pkg/R/context.R b/R/pkg/R/context.R index 8ec727d..3e996a5 100644 --- a/R/pkg/R/context.R +++ b/R/pkg/R/context.R @@ -305,6 +305,8 @@ setCheckpointDirSC <- function(sc, dirName) { #' Currently directories are only supported for Hadoop-supported filesystems. #' Refer Hadoop-supported filesystems at \url{https://wiki.apache.org/hadoop/HCFS}. #' +#' Note: A path can be added only once. Subsequent additions of the same path are ignored. +#' #' @rdname spark.addFile #' @param path The path of the file to be added #' @param recursive Whether to add files recursively from the path. Default is FALSE. http://git-wip-us.apache.org/repos/asf/spark/blob/69993217/core/src/main/scala/org/apache/spark/SparkContext.scala -- diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 74bfb5d..531384a 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -1496,6 +1496,8 @@ class SparkContext(config: SparkConf) extends Logging { * @param path can be either a local file, a file in HDFS (or other Hadoop-supported * filesystems), or an HTTP, HTTPS or FTP URI. To access the file in Spark jobs, * use `SparkFiles.get(fileName)` to find its download location. + * + * @note A path can be added only once. Subsequent additions of the same path are ignored. */ def addFile(path: String): Unit = { addFile(path, false) @@ -1516,6 +1518,8 @@ class SparkContext(config: SparkConf) extends Logging { * use `SparkFiles.get(fileName)` to find its download location. * @param recursive if true, a directory can be given in `path`. Currently directories are * only supported for Hadoop-supported filesystems. + * + * @note A path can be added only once. Subsequent additions of the same path are ignored. */ def addFile(path: String, recursive: Boolean): Unit = { val uri = new Path(path).toUri @@ -1555,6 +1559,9 @@ class SparkContext(config: SparkConf) extends Logging { Utils.fetchFile(uri.toString, new File(SparkFiles.getRootDirectory()), conf, env.securityManager, hadoopConfiguration, timestamp, useCache = false) postEnvironmentUpdate() +} else { + logWarning(s"The path $path has been added already. Overwriting of added paths " + + "is not supported in the current version.") } } @@ -1803,6 +1810,8 @@ class SparkContext(config: SparkConf) extends Logging { * * @param path can be either a local file, a file in HDFS (or other Hadoop-supported filesystems), * an HTTP, HTTPS or FTP URI, or local:/path for a file on every worker node. + * + * @note A path can be added only once. Subsequent additions of the same path are ignored. */ def addJar(path: String) { def addJarFile(file: File): String = { @@ -1849,6 +1858,9 @@ class SparkContext(config: SparkConf) extends Logging { if (addedJars.putIfAbsent(key, timestamp).isEmpty) { logInfo(s"Added JAR $path at $key with timestamp $timestamp") postEnvironmentUpdate() +} else { + logWarning(s"The jar $path has been added already. Overwriting of added jars " + +"is not supported in th
spark git commit: [SPARK-24676][SQL] Project required data from CSV parsed data when column pruning disabled
Repository: spark Updated Branches: refs/heads/master bcf7121ed -> d463533de [SPARK-24676][SQL] Project required data from CSV parsed data when column pruning disabled ## What changes were proposed in this pull request? This pr modified code to project required data from CSV parsed data when column pruning disabled. In the current master, an exception below happens if `spark.sql.csv.parser.columnPruning.enabled` is false. This is because required formats and CSV parsed formats are different from each other; ``` ./bin/spark-shell --conf spark.sql.csv.parser.columnPruning.enabled=false scala> val dir = "/tmp/spark-csv/csv" scala> spark.range(10).selectExpr("id % 2 AS p", "id").write.mode("overwrite").partitionBy("p").csv(dir) scala> spark.read.csv(dir).selectExpr("sum(p)").collect() 18/06/25 13:48:46 ERROR Executor: Exception in task 2.0 in stage 2.0 (TID 7) java.lang.ClassCastException: org.apache.spark.unsafe.types.UTF8String cannot be cast to java.lang.Integer at scala.runtime.BoxesRunTime.unboxToInt(BoxesRunTime.java:101) at org.apache.spark.sql.catalyst.expressions.BaseGenericInternalRow$class.getInt(rows.scala:41) ... ``` ## How was this patch tested? Added tests in `CSVSuite`. Author: Takeshi Yamamuro Closes #21657 from maropu/SPARK-24676. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d463533d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d463533d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d463533d Branch: refs/heads/master Commit: d463533ded89a05e9f77e590fd3de2ffa212d68b Parents: bcf7121 Author: Takeshi Yamamuro Authored: Sun Jul 15 20:22:09 2018 -0700 Committer: Xiao Li Committed: Sun Jul 15 20:22:09 2018 -0700 -- .../datasources/csv/UnivocityParser.scala | 54 +++- .../execution/datasources/csv/CSVSuite.scala| 29 +++ 2 files changed, 70 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/d463533d/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/UnivocityParser.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/UnivocityParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/UnivocityParser.scala index aa545e1..79143cc 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/UnivocityParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/UnivocityParser.scala @@ -33,29 +33,49 @@ import org.apache.spark.sql.execution.datasources.FailureSafeParser import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String + +/** + * Constructs a parser for a given schema that translates CSV data to an [[InternalRow]]. + * + * @param dataSchema The CSV data schema that is specified by the user, or inferred from underlying + * data files. + * @param requiredSchema The schema of the data that should be output for each row. This should be a + * subset of the columns in dataSchema. + * @param options Configuration options for a CSV parser. + */ class UnivocityParser( dataSchema: StructType, requiredSchema: StructType, val options: CSVOptions) extends Logging { require(requiredSchema.toSet.subsetOf(dataSchema.toSet), -"requiredSchema should be the subset of schema.") +s"requiredSchema (${requiredSchema.catalogString}) should be the subset of " + + s"dataSchema (${dataSchema.catalogString}).") def this(schema: StructType, options: CSVOptions) = this(schema, schema, options) // A `ValueConverter` is responsible for converting the given value to a desired type. private type ValueConverter = String => Any + // This index is used to reorder parsed tokens + private val tokenIndexArr = +requiredSchema.map(f => java.lang.Integer.valueOf(dataSchema.indexOf(f))).toArray + + // When column pruning is enabled, the parser only parses the required columns based on + // their positions in the data schema. + private val parsedSchema = if (options.columnPruning) requiredSchema else dataSchema + val tokenizer = { val parserSetting = options.asParserSettings -if (options.columnPruning && requiredSchema.length < dataSchema.length) { - val tokenIndexArr = requiredSchema.map(f => java.lang.Integer.valueOf(dataSchema.indexOf(f))) +// When to-be-parsed schema is shorter than the to-be-read data schema, we let Univocity CSV +// parser select a sequence of fields for reading by their positions. +// if (options.columnPruning && requiredSchema.length < dataSchema.length) { +if (parsedSchema.length < dataSchema.length)
spark git commit: [SPARK-24810][SQL] Fix paths to test files in AvroSuite
Repository: spark Updated Branches: refs/heads/master d463533de -> 9f929458f [SPARK-24810][SQL] Fix paths to test files in AvroSuite ## What changes were proposed in this pull request? In the PR, I propose to move `testFile()` to the common trait `SQLTestUtilsBase` and wrap test files in `AvroSuite` by the method `testFile()` which returns full paths to test files in the resource folder. Author: Maxim Gekk Closes #21773 from MaxGekk/test-file. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9f929458 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9f929458 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9f929458 Branch: refs/heads/master Commit: 9f929458fb0a8a106f3b5a6ed3ee2cd3faa85770 Parents: d463533 Author: Maxim Gekk Authored: Sun Jul 15 23:01:36 2018 -0700 Committer: Xiao Li Committed: Sun Jul 15 23:01:36 2018 -0700 -- .../org/apache/spark/sql/avro/AvroSuite.scala | 79 +++- .../execution/datasources/csv/CSVSuite.scala| 4 - .../execution/datasources/json/JsonSuite.scala | 4 - .../datasources/text/WholeTextFileSuite.scala | 11 +-- .../apache/spark/sql/test/SQLTestUtils.scala| 7 ++ 5 files changed, 53 insertions(+), 52 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/9f929458/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala -- diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala index 6ed6656..9c6526b 100644 --- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala +++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala @@ -36,8 +36,8 @@ import org.apache.spark.sql.test.{SharedSQLContext, SQLTestUtils} import org.apache.spark.sql.types._ class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils { - val episodesFile = "src/test/resources/episodes.avro" - val testFile = "src/test/resources/test.avro" + val episodesAvro = testFile("episodes.avro") + val testAvro = testFile("test.avro") override protected def beforeAll(): Unit = { super.beforeAll() @@ -45,18 +45,18 @@ class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils { } def checkReloadMatchesSaved(originalFile: String, newFile: String): Unit = { -val originalEntries = spark.read.avro(testFile).collect() +val originalEntries = spark.read.avro(testAvro).collect() val newEntries = spark.read.avro(newFile) checkAnswer(newEntries, originalEntries) } test("reading from multiple paths") { -val df = spark.read.avro(episodesFile, episodesFile) +val df = spark.read.avro(episodesAvro, episodesAvro) assert(df.count == 16) } test("reading and writing partitioned data") { -val df = spark.read.avro(episodesFile) +val df = spark.read.avro(episodesAvro) val fields = List("title", "air_date", "doctor") for (field <- fields) { withTempPath { dir => @@ -72,14 +72,14 @@ class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils { } test("request no fields") { -val df = spark.read.avro(episodesFile) +val df = spark.read.avro(episodesAvro) df.createOrReplaceTempView("avro_table") assert(spark.sql("select count(*) from avro_table").collect().head === Row(8)) } test("convert formats") { withTempPath { dir => - val df = spark.read.avro(episodesFile) + val df = spark.read.avro(episodesAvro) df.write.parquet(dir.getCanonicalPath) assert(spark.read.parquet(dir.getCanonicalPath).count() === df.count) } @@ -87,7 +87,7 @@ class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils { test("rearrange internal schema") { withTempPath { dir => - val df = spark.read.avro(episodesFile) + val df = spark.read.avro(episodesAvro) df.select("doctor", "title").write.avro(dir.getCanonicalPath) } } @@ -362,7 +362,7 @@ class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils { val deflateDir = s"$dir/deflate" val snappyDir = s"$dir/snappy" - val df = spark.read.avro(testFile) + val df = spark.read.avro(testAvro) spark.conf.set(AVRO_COMPRESSION_CODEC, "uncompressed") df.write.avro(uncompressDir) spark.conf.set(AVRO_COMPRESSION_CODEC, "deflate") @@ -381,49 +381,49 @@ class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils { } test("dsl test") { -val results = spark.read.avro(episodesFile).select("title").collect() +val results = spark.read.avro(episodesAvro).select("title").collect() assert(results.length === 8
spark git commit: [SPARK-23901][SQL] Removing masking functions
Repository: spark Updated Branches: refs/heads/master b045315e5 -> b0c95a1d6 [SPARK-23901][SQL] Removing masking functions The PR reverts #21246. Author: Marek Novotny Closes #21786 from mn-mikke/SPARK-23901. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b0c95a1d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b0c95a1d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b0c95a1d Branch: refs/heads/master Commit: b0c95a1d69df752bd62e49838a98268f6847 Parents: b045315 Author: Marek Novotny Authored: Mon Jul 16 14:28:35 2018 -0700 Committer: Xiao Li Committed: Mon Jul 16 14:28:35 2018 -0700 -- .../expressions/MaskExpressionsUtils.java | 80 --- .../catalyst/analysis/FunctionRegistry.scala| 8 - .../catalyst/expressions/maskExpressions.scala | 569 --- .../expressions/MaskExpressionsSuite.scala | 236 .../scala/org/apache/spark/sql/functions.scala | 119 .../spark/sql/DataFrameFunctionsSuite.scala | 107 6 files changed, 1119 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/b0c95a1d/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/MaskExpressionsUtils.java -- diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/MaskExpressionsUtils.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/MaskExpressionsUtils.java deleted file mode 100644 index 0587990..000 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/MaskExpressionsUtils.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - *http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.catalyst.expressions; - -/** - * Contains all the Utils methods used in the masking expressions. - */ -public class MaskExpressionsUtils { - static final int UNMASKED_VAL = -1; - - /** - * Returns the masking character for {@param c} or {@param c} is it should not be masked. - * @param c the character to transform - * @param maskedUpperChar the character to use instead of a uppercase letter - * @param maskedLowerChar the character to use instead of a lowercase letter - * @param maskedDigitChar the character to use instead of a digit - * @param maskedOtherChar the character to use instead of a any other character - * @return masking character for {@param c} - */ - public static int transformChar( - final int c, - int maskedUpperChar, - int maskedLowerChar, - int maskedDigitChar, - int maskedOtherChar) { -switch(Character.getType(c)) { - case Character.UPPERCASE_LETTER: -if(maskedUpperChar != UNMASKED_VAL) { - return maskedUpperChar; -} -break; - - case Character.LOWERCASE_LETTER: -if(maskedLowerChar != UNMASKED_VAL) { - return maskedLowerChar; -} -break; - - case Character.DECIMAL_DIGIT_NUMBER: -if(maskedDigitChar != UNMASKED_VAL) { - return maskedDigitChar; -} -break; - - default: -if(maskedOtherChar != UNMASKED_VAL) { - return maskedOtherChar; -} -break; -} - -return c; - } - - /** - * Returns the replacement char to use according to the {@param rep} specified by the user and - * the {@param def} default. - */ - public static int getReplacementChar(String rep, int def) { -if (rep != null && rep.length() > 0) { - return rep.codePointAt(0); -} -return def; - } -} http://git-wip-us.apache.org/repos/asf/spark/blob/b0c95a1d/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index 1d9e470..d696ce9 100644 --- a/s
spark git commit: [SPARK-24805][SQL] Do not ignore avro files without extensions by default
Repository: spark Updated Branches: refs/heads/master b0c95a1d6 -> ba437fc5c [SPARK-24805][SQL] Do not ignore avro files without extensions by default ## What changes were proposed in this pull request? In the PR, I propose to change default behaviour of AVRO datasource which currently ignores files without `.avro` extension in read by default. This PR sets the default value for `avro.mapred.ignore.inputs.without.extension` to `false` in the case if the parameter is not set by an user. ## How was this patch tested? Added a test file without extension in AVRO format, and new test for reading the file with and wihout specified schema. Author: Maxim Gekk Author: Maxim Gekk Closes #21769 from MaxGekk/avro-without-extension. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ba437fc5 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ba437fc5 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ba437fc5 Branch: refs/heads/master Commit: ba437fc5c73b95ee4c59327abf3161c58f64cb12 Parents: b0c95a1 Author: Maxim Gekk Authored: Mon Jul 16 14:35:44 2018 -0700 Committer: Xiao Li Committed: Mon Jul 16 14:35:44 2018 -0700 -- .../apache/spark/sql/avro/AvroFileFormat.scala | 14 +++--- .../org/apache/spark/sql/avro/AvroSuite.scala | 45 +--- 2 files changed, 47 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/ba437fc5/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala -- diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala index fb93033..9eb2064 100755 --- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala +++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala @@ -62,7 +62,7 @@ private[avro] class AvroFileFormat extends FileFormat with DataSourceRegister { // Schema evolution is not supported yet. Here we only pick a single random sample file to // figure out the schema of the whole dataset. val sampleFile = - if (conf.getBoolean(AvroFileFormat.IgnoreFilesWithoutExtensionProperty, true)) { + if (AvroFileFormat.ignoreFilesWithoutExtensions(conf)) { files.find(_.getPath.getName.endsWith(".avro")).getOrElse { throw new FileNotFoundException( "No Avro files found. Hadoop option \"avro.mapred.ignore.inputs.without.extension\" " + @@ -170,10 +170,7 @@ private[avro] class AvroFileFormat extends FileFormat with DataSourceRegister { // Doing input file filtering is improper because we may generate empty tasks that process no // input files but stress the scheduler. We should probably add a more general input file // filtering mechanism for `FileFormat` data sources. See SPARK-16317. - if ( -conf.getBoolean(AvroFileFormat.IgnoreFilesWithoutExtensionProperty, true) && -!file.filePath.endsWith(".avro") - ) { + if (AvroFileFormat.ignoreFilesWithoutExtensions(conf) && !file.filePath.endsWith(".avro")) { Iterator.empty } else { val reader = { @@ -278,4 +275,11 @@ private[avro] object AvroFileFormat { value.readFields(new DataInputStream(in)) } } + + def ignoreFilesWithoutExtensions(conf: Configuration): Boolean = { +// Files without .avro extensions are not ignored by default +val defaultValue = false + +conf.getBoolean(AvroFileFormat.IgnoreFilesWithoutExtensionProperty, defaultValue) + } } http://git-wip-us.apache.org/repos/asf/spark/blob/ba437fc5/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala -- diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala index 9c6526b..446b421 100644 --- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala +++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala @@ -18,7 +18,8 @@ package org.apache.spark.sql.avro import java.io._ -import java.nio.file.Files +import java.net.URL +import java.nio.file.{Files, Path, Paths} import java.sql.{Date, Timestamp} import java.util.{TimeZone, UUID} @@ -622,7 +623,12 @@ class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils { intercept[FileNotFoundException] { withTempPath { dir => FileUtils.touch(new File(dir, "test")) -spark.read.avro(dir.toString) +val hadoopConf = spark.sqlContext.sparkContext.hadoopConfiguration +try { + hadoopConf.set
spark git commit: [SPARK-24402][SQL] Optimize `In` expression when only one element in the collection or collection is empty
Repository: spark Updated Branches: refs/heads/master ba437fc5c -> 0f0d1865f [SPARK-24402][SQL] Optimize `In` expression when only one element in the collection or collection is empty ## What changes were proposed in this pull request? Two new rules in the logical plan optimizers are added. 1. When there is only one element in the **`Collection`**, the physical plan will be optimized to **`EqualTo`**, so predicate pushdown can be used. ```scala profileDF.filter( $"profileID".isInCollection(Set(6))).explain(true) """ |== Physical Plan == |*(1) Project [profileID#0] |+- *(1) Filter (isnotnull(profileID#0) && (profileID#0 = 6)) | +- *(1) FileScan parquet [profileID#0] Batched: true, Format: Parquet, | PartitionFilters: [], | PushedFilters: [IsNotNull(profileID), EqualTo(profileID,6)], | ReadSchema: struct """.stripMargin ``` 2. When the **`Collection`** is empty, and the input is nullable, the logical plan will be simplified to ```scala profileDF.filter( $"profileID".isInCollection(Set())).explain(true) """ |== Optimized Logical Plan == |Filter if (isnull(profileID#0)) null else false |+- Relation[profileID#0] parquet """.stripMargin ``` TODO: 1. For multiple conditions with numbers less than certain thresholds, we should still allow predicate pushdown. 2. Optimize the **`In`** using **`tableswitch`** or **`lookupswitch`** when the numbers of the categories are low, and they are **`Int`**, **`Long`**. 3. The default immutable hash trees set is slow for query, and we should do benchmark for using different set implementation for faster query. 4. **`filter(if (condition) null else false)`** can be optimized to false. ## How was this patch tested? Couple new tests are added. Author: DB Tsai Closes #21442 from dbtsai/optimize-in. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0f0d1865 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0f0d1865 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0f0d1865 Branch: refs/heads/master Commit: 0f0d1865f581a9158d73505471953656b173beba Parents: ba437fc Author: DB Tsai Authored: Mon Jul 16 15:33:39 2018 -0700 Committer: Xiao Li Committed: Mon Jul 16 15:33:39 2018 -0700 -- .../sql/catalyst/optimizer/expressions.scala| 13 +--- .../catalyst/optimizer/OptimizeInSuite.scala| 32 2 files changed, 41 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/0f0d1865/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala index 1d363b8..f78a0ff 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala @@ -218,15 +218,20 @@ object ReorderAssociativeOperator extends Rule[LogicalPlan] { object OptimizeIn extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan transform { case q: LogicalPlan => q transformExpressionsDown { - case In(v, list) if list.isEmpty && !v.nullable => FalseLiteral + case In(v, list) if list.isEmpty => +// When v is not nullable, the following expression will be optimized +// to FalseLiteral which is tested in OptimizeInSuite.scala +If(IsNotNull(v), FalseLiteral, Literal(null, BooleanType)) case expr @ In(v, list) if expr.inSetConvertible => val newList = ExpressionSet(list).toSeq -if (newList.size > SQLConf.get.optimizerInSetConversionThreshold) { +if (newList.length == 1 && !newList.isInstanceOf[ListQuery]) { + EqualTo(v, newList.head) +} else if (newList.length > SQLConf.get.optimizerInSetConversionThreshold) { val hSet = newList.map(e => e.eval(EmptyRow)) InSet(v, HashSet() ++ hSet) -} else if (newList.size < list.size) { +} else if (newList.length < list.length) { expr.copy(list = newList) -} else { // newList.length == list.length +} else { // newList.length == list.length && newList.length > 1 expr } } http://git-wip-us.apache.org/repos/asf/spark/blob/0f0d1865/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala -- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala b/sql/catalyst/src/t
spark git commit: [SPARK-24681][SQL] Verify nested column names in Hive metastore
Repository: spark Updated Branches: refs/heads/master 912634b00 -> 2a4dd6f06 [SPARK-24681][SQL] Verify nested column names in Hive metastore ## What changes were proposed in this pull request? This pr added code to check if nested column names do not include ',', ':', and ';' because Hive metastore can't handle these characters in nested column names; ref: https://github.com/apache/hive/blob/release-1.2.1/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java#L239 ## How was this patch tested? Added tests in `HiveDDLSuite`. Author: Takeshi Yamamuro Closes #21711 from maropu/SPARK-24681. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2a4dd6f0 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2a4dd6f0 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2a4dd6f0 Branch: refs/heads/master Commit: 2a4dd6f06cfd2f58fda9786c88809e6de695444e Parents: 912634b Author: Takeshi Yamamuro Authored: Tue Jul 17 14:15:30 2018 -0700 Committer: Xiao Li Committed: Tue Jul 17 14:15:30 2018 -0700 -- .../spark/sql/hive/HiveExternalCatalog.scala| 34 .../spark/sql/hive/execution/HiveDDLSuite.scala | 19 +++ 2 files changed, 46 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/2a4dd6f0/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala -- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index 44480ce..7f28fc4 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -138,17 +138,37 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat } /** - * Checks the validity of data column names. Hive metastore disallows the table to use comma in - * data column names. Partition columns do not have such a restriction. Views do not have such - * a restriction. + * Checks the validity of data column names. Hive metastore disallows the table to use some + * special characters (',', ':', and ';') in data column names, including nested column names. + * Partition columns do not have such a restriction. Views do not have such a restriction. */ private def verifyDataSchema( tableName: TableIdentifier, tableType: CatalogTableType, dataSchema: StructType): Unit = { if (tableType != VIEW) { - dataSchema.map(_.name).foreach { colName => -if (colName.contains(",")) { - throw new AnalysisException("Cannot create a table having a column whose name contains " + -s"commas in Hive metastore. Table: $tableName; Column: $colName") + val invalidChars = Seq(",", ":", ";") + def verifyNestedColumnNames(schema: StructType): Unit = schema.foreach { f => +f.dataType match { + case st: StructType => verifyNestedColumnNames(st) + case _ if invalidChars.exists(f.name.contains) => +val invalidCharsString = invalidChars.map(c => s"'$c'").mkString(", ") +val errMsg = "Cannot create a table having a nested column whose name contains " + + s"invalid characters ($invalidCharsString) in Hive metastore. Table: $tableName; " + + s"Column: ${f.name}" +throw new AnalysisException(errMsg) + case _ => +} + } + + dataSchema.foreach { f => +f.dataType match { + // Checks top-level column names + case _ if f.name.contains(",") => +throw new AnalysisException("Cannot create a table having a column whose name " + + s"contains commas in Hive metastore. Table: $tableName; Column: ${f.name}") + // Checks nested column names + case st: StructType => +verifyNestedColumnNames(st) + case _ => } } } http://git-wip-us.apache.org/repos/asf/spark/blob/2a4dd6f0/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala -- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index 0341c3b..31fd4c5 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -33,6 +33,7 @@ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionE
spark git commit: [SPARK-24576][BUILD] Upgrade Apache ORC to 1.5.2
Repository: spark Updated Branches: refs/heads/master fc2e18963 -> 3b59d326c [SPARK-24576][BUILD] Upgrade Apache ORC to 1.5.2 ## What changes were proposed in this pull request? This issue aims to upgrade Apache ORC library from 1.4.4 to 1.5.2 in order to bring the following benefits into Apache Spark. - [ORC-91](https://issues.apache.org/jira/browse/ORC-91) Support for variable length blocks in HDFS (The current space wasted in ORC to padding is known to be 5%.) - [ORC-344](https://issues.apache.org/jira/browse/ORC-344) Support for using Decimal64ColumnVector In addition to that, Apache Hive 3.1 and 3.2 will use ORC 1.5.1 ([HIVE-19669](https://issues.apache.org/jira/browse/HIVE-19465)) and 1.5.2 ([HIVE-19792](https://issues.apache.org/jira/browse/HIVE-19792)) respectively. This will improve the compatibility between Apache Spark and Apache Hive by sharing the common library. ## How was this patch tested? Pass the Jenkins with all existing tests. Author: Dongjoon Hyun Closes #21582 from dongjoon-hyun/SPARK-24576. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3b59d326 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3b59d326 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3b59d326 Branch: refs/heads/master Commit: 3b59d326c77bec96e5fb856d827139e0389394ba Parents: fc2e189 Author: Dongjoon Hyun Authored: Tue Jul 17 23:52:17 2018 -0700 Committer: Xiao Li Committed: Tue Jul 17 23:52:17 2018 -0700 -- dev/deps/spark-deps-hadoop-2.6 | 7 ++--- dev/deps/spark-deps-hadoop-2.7 | 7 ++--- dev/deps/spark-deps-hadoop-3.1 | 7 ++--- pom.xml | 2 +- sql/core/pom.xml| 28 .../datasources/orc/OrcFileFormat.scala | 15 ++- .../datasources/orc/OrcSerializer.scala | 2 +- 7 files changed, 56 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/3b59d326/dev/deps/spark-deps-hadoop-2.6 -- diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index f50a0aa..ff6d5c3 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -2,7 +2,7 @@ JavaEWAH-0.3.2.jar RoaringBitmap-0.5.11.jar ST4-4.0.4.jar activation-1.1.1.jar -aircompressor-0.8.jar +aircompressor-0.10.jar antlr-2.7.7.jar antlr-runtime-3.4.jar antlr4-runtime-4.7.jar @@ -157,8 +157,9 @@ objenesis-2.1.jar okhttp-3.8.1.jar okio-1.13.0.jar opencsv-2.3.jar -orc-core-1.4.4-nohive.jar -orc-mapreduce-1.4.4-nohive.jar +orc-core-1.5.2-nohive.jar +orc-mapreduce-1.5.2-nohive.jar +orc-shims-1.5.2.jar oro-2.0.8.jar osgi-resource-locator-1.0.1.jar paranamer-2.8.jar http://git-wip-us.apache.org/repos/asf/spark/blob/3b59d326/dev/deps/spark-deps-hadoop-2.7 -- diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index 774f9dc..72a94f8 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -2,7 +2,7 @@ JavaEWAH-0.3.2.jar RoaringBitmap-0.5.11.jar ST4-4.0.4.jar activation-1.1.1.jar -aircompressor-0.8.jar +aircompressor-0.10.jar antlr-2.7.7.jar antlr-runtime-3.4.jar antlr4-runtime-4.7.jar @@ -158,8 +158,9 @@ objenesis-2.1.jar okhttp-3.8.1.jar okio-1.13.0.jar opencsv-2.3.jar -orc-core-1.4.4-nohive.jar -orc-mapreduce-1.4.4-nohive.jar +orc-core-1.5.2-nohive.jar +orc-mapreduce-1.5.2-nohive.jar +orc-shims-1.5.2.jar oro-2.0.8.jar osgi-resource-locator-1.0.1.jar paranamer-2.8.jar http://git-wip-us.apache.org/repos/asf/spark/blob/3b59d326/dev/deps/spark-deps-hadoop-3.1 -- diff --git a/dev/deps/spark-deps-hadoop-3.1 b/dev/deps/spark-deps-hadoop-3.1 index 19c05ad..3409dc4 100644 --- a/dev/deps/spark-deps-hadoop-3.1 +++ b/dev/deps/spark-deps-hadoop-3.1 @@ -4,7 +4,7 @@ RoaringBitmap-0.5.11.jar ST4-4.0.4.jar accessors-smart-1.2.jar activation-1.1.1.jar -aircompressor-0.8.jar +aircompressor-0.10.jar antlr-2.7.7.jar antlr-runtime-3.4.jar antlr4-runtime-4.7.jar @@ -176,8 +176,9 @@ okhttp-2.7.5.jar okhttp-3.8.1.jar okio-1.13.0.jar opencsv-2.3.jar -orc-core-1.4.4-nohive.jar -orc-mapreduce-1.4.4-nohive.jar +orc-core-1.5.2-nohive.jar +orc-mapreduce-1.5.2-nohive.jar +orc-shims-1.5.2.jar oro-2.0.8.jar osgi-resource-locator-1.0.1.jar paranamer-2.8.jar http://git-wip-us.apache.org/repos/asf/spark/blob/3b59d326/pom.xml -- diff --git a/pom.xml b/pom.xml index 1892bbe..649221d 100644 --- a/pom.xml +++ b/pom.xml @@ -131,7 +131,7 @@ 1.2.1 10.12.1.1 1.10.0 -
spark git commit: [SPARK-24163][SPARK-24164][SQL] Support column list as the pivot column in Pivot
Repository: spark Updated Branches: refs/heads/master 1272b2034 -> cd203e0df [SPARK-24163][SPARK-24164][SQL] Support column list as the pivot column in Pivot ## What changes were proposed in this pull request? 1. Extend the Parser to enable parsing a column list as the pivot column. 2. Extend the Parser and the Pivot node to enable parsing complex expressions with aliases as the pivot value. 3. Add type check and constant check in Analyzer for Pivot node. ## How was this patch tested? Add tests in pivot.sql Author: maryannxue Closes #21720 from maryannxue/spark-24164. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cd203e0d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cd203e0d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cd203e0d Branch: refs/heads/master Commit: cd203e0dfc0758a2a90297e8c74c22a1212db846 Parents: 1272b20 Author: maryannxue Authored: Wed Jul 18 13:33:26 2018 -0700 Committer: Xiao Li Committed: Wed Jul 18 13:33:26 2018 -0700 -- .../apache/spark/sql/catalyst/parser/SqlBase.g4 | 11 +- .../spark/sql/catalyst/analysis/Analyzer.scala | 47 ++-- .../spark/sql/catalyst/parser/AstBuilder.scala | 22 +- .../plans/logical/basicLogicalOperators.scala | 2 +- .../test/resources/sql-tests/inputs/pivot.sql | 92 .../resources/sql-tests/results/pivot.sql.out | 230 +++ 6 files changed, 348 insertions(+), 56 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/cd203e0d/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 -- diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index dc95751..1b43874 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -414,7 +414,16 @@ groupingSet ; pivotClause -: PIVOT '(' aggregates=namedExpressionSeq FOR pivotColumn=identifier IN '(' pivotValues+=constant (',' pivotValues+=constant)* ')' ')' +: PIVOT '(' aggregates=namedExpressionSeq FOR pivotColumn IN '(' pivotValues+=pivotValue (',' pivotValues+=pivotValue)* ')' ')' +; + +pivotColumn +: identifiers+=identifier +| '(' identifiers+=identifier (',' identifiers+=identifier)* ')' +; + +pivotValue +: expression (AS? identifier)? ; lateralView http://git-wip-us.apache.org/repos/asf/spark/blob/cd203e0d/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 36f14cc..59c371e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -509,17 +509,39 @@ class Analyzer( def apply(plan: LogicalPlan): LogicalPlan = plan transform { case p: Pivot if !p.childrenResolved || !p.aggregates.forall(_.resolved) || (p.groupByExprsOpt.isDefined && !p.groupByExprsOpt.get.forall(_.resolved)) -|| !p.pivotColumn.resolved => p +|| !p.pivotColumn.resolved || !p.pivotValues.forall(_.resolved) => p case Pivot(groupByExprsOpt, pivotColumn, pivotValues, aggregates, child) => // Check all aggregate expressions. aggregates.foreach(checkValidAggregateExpression) +// Check all pivot values are literal and match pivot column data type. +val evalPivotValues = pivotValues.map { value => + val foldable = value match { +case Alias(v, _) => v.foldable +case _ => value.foldable + } + if (!foldable) { +throw new AnalysisException( + s"Literal expressions required for pivot values, found '$value'") + } + if (!Cast.canCast(value.dataType, pivotColumn.dataType)) { +throw new AnalysisException(s"Invalid pivot value '$value': " + + s"value data type ${value.dataType.simpleString} does not match " + + s"pivot column data type ${pivotColumn.dataType.catalogString}") + } + Cast(value, pivotColumn.dataType, Some(conf.sessionLocalTimeZone)).eval(EmptyRow) +} // Group-by expressions coming from SQL are implicit and need to be deduced. val groupByExprs = groupByExprsOpt.getOrElse( (child.outputSet -- aggregates.flatMap(_.references) --
[1/2] spark git commit: [SPARK-24268][SQL] Use datatype.catalogString in error messages
Repository: spark Updated Branches: refs/heads/master 1462b1766 -> a5925c163 http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java -- diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java index d5969b5..31ef090 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java +++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java @@ -244,7 +244,7 @@ public class VectorizedColumnReader { return new SchemaColumnConvertNotSupportedException( Arrays.toString(descriptor.getPath()), descriptor.getType().toString(), - column.dataType().toString()); + column.dataType().catalogString()); } /** http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala index c6449cd..b068493 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala @@ -452,7 +452,7 @@ class RelationalGroupedDataset protected[sql]( require(expr.evalType == PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF, "Must pass a grouped map udf") require(expr.dataType.isInstanceOf[StructType], - "The returnType of the udf must be a StructType") + s"The returnType of the udf must be a ${StructType.simpleString}") val groupingNamedExpressions = groupingExprs.map { case ne: NamedExpression => ne http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowUtils.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowUtils.scala index 93c8127..533097a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowUtils.scala @@ -47,11 +47,13 @@ object ArrowUtils { case DateType => new ArrowType.Date(DateUnit.DAY) case TimestampType => if (timeZoneId == null) { -throw new UnsupportedOperationException("TimestampType must supply timeZoneId parameter") +throw new UnsupportedOperationException( + s"${TimestampType.catalogString} must supply timeZoneId parameter") } else { new ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZoneId) } -case _ => throw new UnsupportedOperationException(s"Unsupported data type: ${dt.simpleString}") +case _ => + throw new UnsupportedOperationException(s"Unsupported data type: ${dt.catalogString}") } def fromArrowType(dt: ArrowType): DataType = dt match { http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowWriter.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowWriter.scala index 66888fc..3de6ea8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowWriter.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowWriter.scala @@ -68,7 +68,7 @@ object ArrowWriter { } new StructWriter(vector, children.toArray) case (dt, _) => -throw new UnsupportedOperationException(s"Unsupported data type: ${dt.simpleString}") +throw new UnsupportedOperationException(s"Unsupported data type: ${dt.catalogString}") } } } http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala index e9b150f..542a10f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala @@ -717,7 +717,7 @@ private[column
[2/2] spark git commit: [SPARK-24268][SQL] Use datatype.catalogString in error messages
[SPARK-24268][SQL] Use datatype.catalogString in error messages ## What changes were proposed in this pull request? As stated in https://github.com/apache/spark/pull/21321, in the error messages we should use `catalogString`. This is not the case, as SPARK-22893 used `simpleString` in order to have the same representation everywhere and it missed some places. The PR unifies the messages using alway the `catalogString` representation of the dataTypes in the messages. ## How was this patch tested? existing/modified UTs Author: Marco Gaido Closes #21804 from mgaido91/SPARK-24268_catalog. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a5925c16 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a5925c16 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a5925c16 Branch: refs/heads/master Commit: a5925c1631e25c2dcc3c2948cea31e993ce66a97 Parents: 1462b17 Author: Marco Gaido Authored: Thu Jul 19 23:29:29 2018 -0700 Committer: Xiao Li Committed: Thu Jul 19 23:29:29 2018 -0700 -- .../spark/sql/kafka010/KafkaWriteTask.scala | 6 +++--- .../apache/spark/sql/kafka010/KafkaWriter.scala | 6 +++--- .../sql/kafka010/KafkaContinuousSinkSuite.scala | 4 ++-- .../spark/sql/kafka010/KafkaSinkSuite.scala | 4 ++-- .../scala/org/apache/spark/ml/feature/DCT.scala | 3 ++- .../apache/spark/ml/feature/FeatureHasher.scala | 5 +++-- .../org/apache/spark/ml/feature/HashingTF.scala | 2 +- .../org/apache/spark/ml/feature/Interaction.scala | 3 ++- .../scala/org/apache/spark/ml/feature/NGram.scala | 3 ++- .../apache/spark/ml/feature/OneHotEncoder.scala | 3 ++- .../org/apache/spark/ml/feature/RFormula.scala| 2 +- .../spark/ml/feature/StopWordsRemover.scala | 4 ++-- .../org/apache/spark/ml/feature/Tokenizer.scala | 3 ++- .../apache/spark/ml/feature/VectorAssembler.scala | 2 +- .../scala/org/apache/spark/ml/fpm/FPGrowth.scala | 2 +- .../org/apache/spark/ml/util/SchemaUtils.scala| 11 +++ .../BinaryClassificationEvaluatorSuite.scala | 4 ++-- .../apache/spark/ml/feature/RFormulaSuite.scala | 2 +- .../spark/ml/feature/VectorAssemblerSuite.scala | 6 +++--- .../apache/spark/ml/recommendation/ALSSuite.scala | 2 +- .../regression/AFTSurvivalRegressionSuite.scala | 2 +- .../org/apache/spark/ml/util/MLTestingUtils.scala | 6 +++--- .../spark/sql/catalyst/analysis/Analyzer.scala| 8 .../sql/catalyst/analysis/CheckAnalysis.scala | 18 +- .../apache/spark/sql/catalyst/analysis/view.scala | 3 ++- .../spark/sql/catalyst/expressions/Cast.scala | 2 +- .../catalyst/expressions/ExpectsInputTypes.scala | 2 +- .../sql/catalyst/expressions/Expression.scala | 4 ++-- .../spark/sql/catalyst/expressions/ScalaUDF.scala | 5 +++-- .../sql/catalyst/expressions/SortOrder.scala | 2 +- .../aggregate/ApproximatePercentile.scala | 4 ++-- .../sql/catalyst/expressions/arithmetic.scala | 4 ++-- .../expressions/codegen/CodeGenerator.scala | 4 ++-- .../expressions/collectionOperations.scala| 14 +++--- .../catalyst/expressions/complexTypeCreator.scala | 8 .../expressions/complexTypeExtractors.scala | 2 +- .../expressions/conditionalExpressions.scala | 4 ++-- .../sql/catalyst/expressions/generators.scala | 8 .../catalyst/expressions/jsonExpressions.scala| 6 +++--- .../catalyst/expressions/objects/objects.scala| 2 +- .../sql/catalyst/expressions/predicates.scala | 2 +- .../catalyst/expressions/stringExpressions.scala | 7 --- .../catalyst/expressions/windowExpressions.scala | 6 +++--- .../sql/catalyst/json/JacksonGenerator.scala | 8 .../spark/sql/catalyst/json/JacksonParser.scala | 6 -- .../spark/sql/catalyst/json/JacksonUtils.scala| 2 +- .../spark/sql/catalyst/json/JsonInferSchema.scala | 6 -- .../spark/sql/catalyst/util/TypeUtils.scala | 7 --- .../apache/spark/sql/types/AbstractDataType.scala | 9 + .../org/apache/spark/sql/types/ArrayType.scala| 5 +++-- .../org/apache/spark/sql/types/DecimalType.scala | 3 ++- .../org/apache/spark/sql/types/ObjectType.scala | 3 ++- .../org/apache/spark/sql/types/StructType.scala | 5 +++-- .../catalyst/analysis/AnalysisErrorSuite.scala| 2 +- .../analysis/ExpressionTypeCheckingSuite.scala| 16 .../catalyst/parser/ExpressionParserSuite.scala | 2 +- .../apache/spark/sql/types/DataTypeSuite.scala| 2 +- .../parquet/VectorizedColumnReader.java | 2 +- .../spark/sql/RelationalGroupedDataset.scala | 2 +- .../spark/sql/execution/arrow/ArrowUtils.scala| 6 -- .../spark/sql/execution/arrow/ArrowWriter.scala | 2 +- .../spark/sql/execution/columnar/ColumnType.scala | 2 +- .../execution/datasou
spark git commit: [SPARK-24424][SQL] Support ANSI-SQL compliant syntax for GROUPING SET
Repository: spark Updated Branches: refs/heads/master a5925c163 -> 2b91d9918 [SPARK-24424][SQL] Support ANSI-SQL compliant syntax for GROUPING SET ## What changes were proposed in this pull request? Enhances the parser and analyzer to support ANSI compliant syntax for GROUPING SET. As part of this change we derive the grouping expressions from user supplied groupings in the grouping sets clause. ```SQL SELECT c1, c2, max(c3) FROM t1 GROUP BY GROUPING SETS ((c1), (c1, c2)) ``` ## How was this patch tested? Added tests in SQLQueryTestSuite and ResolveGroupingAnalyticsSuite. Please review http://spark.apache.org/contributing.html before opening a pull request. Author: Dilip Biswal Closes #21813 from dilipbiswal/spark-24424. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2b91d991 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2b91d991 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2b91d991 Branch: refs/heads/master Commit: 2b91d9918c8eaec6c32a502e2f08b63c475d3335 Parents: a5925c1 Author: Dilip Biswal Authored: Thu Jul 19 23:52:53 2018 -0700 Committer: Xiao Li Committed: Thu Jul 19 23:52:53 2018 -0700 -- .../apache/spark/sql/catalyst/parser/SqlBase.g4 | 1 + .../spark/sql/catalyst/analysis/Analyzer.scala | 22 +++- .../ResolveGroupingAnalyticsSuite.scala | 28 + .../resources/sql-tests/inputs/grouping_set.sql | 36 ++ .../sql-tests/results/grouping_set.sql.out | 126 ++- 5 files changed, 210 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/2b91d991/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 -- diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 1b43874..2aca10f 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -406,6 +406,7 @@ aggregation WITH kind=ROLLUP | WITH kind=CUBE | kind=GROUPING SETS '(' groupingSet (',' groupingSet)* ')')? +| GROUP BY kind=GROUPING SETS '(' groupingSet (',' groupingSet)* ')' ; groupingSet http://git-wip-us.apache.org/repos/asf/spark/blob/2b91d991/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 7c5504d..957c468 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -442,17 +442,35 @@ class Analyzer( child: LogicalPlan): LogicalPlan = { val gid = AttributeReference(VirtualColumn.groupingIdName, IntegerType, false)() + // In case of ANSI-SQL compliant syntax for GROUPING SETS, groupByExprs is optional and + // can be null. In such case, we derive the groupByExprs from the user supplied values for + // grouping sets. + val finalGroupByExpressions = if (groupByExprs == Nil) { +selectedGroupByExprs.flatten.foldLeft(Seq.empty[Expression]) { (result, currentExpr) => + // Only unique expressions are included in the group by expressions and is determined + // based on their semantic equality. Example. grouping sets ((a * b), (b * a)) results + // in grouping expression (a * b) + if (result.find(_.semanticEquals(currentExpr)).isDefined) { +result + } else { +result :+ currentExpr + } +} + } else { +groupByExprs + } + // Expand works by setting grouping expressions to null as determined by the // `selectedGroupByExprs`. To prevent these null values from being used in an aggregate // instead of the original value we need to create new aliases for all group by expressions // that will only be used for the intended purpose. - val groupByAliases = constructGroupByAlias(groupByExprs) + val groupByAliases = constructGroupByAlias(finalGroupByExpressions) val expand = constructExpand(selectedGroupByExprs, child, groupByAliases, gid) val groupingAttrs = expand.output.drop(child.output.length) val aggregations = constructAggregateExprs( -groupByExprs, aggregationExprs, groupByAliases, groupingAttrs, gid) +finalGroupByExpressions, aggregationExprs, groupByA
spark git commit: [SPARK-24811][SQL] Avro: add new function from_avro and to_avro
Repository: spark Updated Branches: refs/heads/master cc4d64bb1 -> 244bcff19 [SPARK-24811][SQL] Avro: add new function from_avro and to_avro ## What changes were proposed in this pull request? Add a new function from_avro for parsing a binary column of avro format and converting it into its corresponding catalyst value. Add a new function to_avro for converting a column into binary of avro format with the specified schema. This PR is in progress. Will add test cases. ## How was this patch tested? Author: Gengliang Wang Closes #21774 from gengliangwang/from_and_to_avro. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/244bcff1 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/244bcff1 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/244bcff1 Branch: refs/heads/master Commit: 244bcff19463d82ec72baf15bc0a5209f21f2ef3 Parents: cc4d64b Author: Gengliang Wang Authored: Fri Jul 20 09:19:29 2018 -0700 Committer: Xiao Li Committed: Fri Jul 20 09:19:29 2018 -0700 -- .../spark/sql/avro/AvroDataToCatalyst.scala | 68 +++ .../spark/sql/avro/CatalystDataToAvro.scala | 69 .../org/apache/spark/sql/avro/package.scala | 31 .../avro/AvroCatalystDataConversionSuite.scala | 175 +++ .../spark/sql/avro/AvroFunctionsSuite.scala | 83 + .../expressions/ExpressionEvalHelper.scala | 6 + 6 files changed, 432 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/244bcff1/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala -- diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala new file mode 100644 index 000..6671b3f --- /dev/null +++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.avro.Schema +import org.apache.avro.generic.GenericDatumReader +import org.apache.avro.io.{BinaryDecoder, DecoderFactory} + +import org.apache.spark.sql.avro.{AvroDeserializer, SchemaConverters} +import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, UnaryExpression} +import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode} +import org.apache.spark.sql.types.{AbstractDataType, BinaryType, DataType} + +case class AvroDataToCatalyst(child: Expression, jsonFormatSchema: String) + extends UnaryExpression with ExpectsInputTypes { + + override def inputTypes: Seq[AbstractDataType] = Seq(BinaryType) + + override lazy val dataType: DataType = SchemaConverters.toSqlType(avroSchema).dataType + + override def nullable: Boolean = true + + @transient private lazy val avroSchema = new Schema.Parser().parse(jsonFormatSchema) + + @transient private lazy val reader = new GenericDatumReader[Any](avroSchema) + + @transient private lazy val deserializer = new AvroDeserializer(avroSchema, dataType) + + @transient private var decoder: BinaryDecoder = _ + + @transient private var result: Any = _ + + override def nullSafeEval(input: Any): Any = { +val binary = input.asInstanceOf[Array[Byte]] +decoder = DecoderFactory.get().binaryDecoder(binary, 0, binary.length, decoder) +result = reader.read(result, decoder) +deserializer.deserialize(result) + } + + override def simpleString: String = { +s"from_avro(${child.sql}, ${dataType.simpleString})" + } + + override def sql: String = { +s"from_avro(${child.sql}, ${dataType.catalogString})" + } + + override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { +val expr = ctx.addReferenceObj("this", this) +defineCodeGen(ctx, ev, input => + s"(${CodeGenerator.boxedType(dataType)})$expr.nullSafeEval($input)") + } +} http://git-wip-us.apache.org/repos/
spark git commit: Revert "[SPARK-24811][SQL] Avro: add new function from_avro and to_avro"
Repository: spark Updated Branches: refs/heads/master 3cb1b5780 -> 9ad77b303 Revert "[SPARK-24811][SQL] Avro: add new function from_avro and to_avro" This reverts commit 244bcff19463d82ec72baf15bc0a5209f21f2ef3. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9ad77b30 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9ad77b30 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9ad77b30 Branch: refs/heads/master Commit: 9ad77b3037b476b726b773c38d1cd264d89d51e2 Parents: 3cb1b57 Author: Xiao Li Authored: Fri Jul 20 12:55:38 2018 -0700 Committer: Xiao Li Committed: Fri Jul 20 12:55:38 2018 -0700 -- .../spark/sql/avro/AvroDataToCatalyst.scala | 68 --- .../spark/sql/avro/CatalystDataToAvro.scala | 69 .../org/apache/spark/sql/avro/package.scala | 31 .../avro/AvroCatalystDataConversionSuite.scala | 175 --- .../spark/sql/avro/AvroFunctionsSuite.scala | 83 - .../expressions/ExpressionEvalHelper.scala | 6 - 6 files changed, 432 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/9ad77b30/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala -- diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala deleted file mode 100644 index 6671b3f..000 --- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - *http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql - -import org.apache.avro.Schema -import org.apache.avro.generic.GenericDatumReader -import org.apache.avro.io.{BinaryDecoder, DecoderFactory} - -import org.apache.spark.sql.avro.{AvroDeserializer, SchemaConverters} -import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, UnaryExpression} -import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode} -import org.apache.spark.sql.types.{AbstractDataType, BinaryType, DataType} - -case class AvroDataToCatalyst(child: Expression, jsonFormatSchema: String) - extends UnaryExpression with ExpectsInputTypes { - - override def inputTypes: Seq[AbstractDataType] = Seq(BinaryType) - - override lazy val dataType: DataType = SchemaConverters.toSqlType(avroSchema).dataType - - override def nullable: Boolean = true - - @transient private lazy val avroSchema = new Schema.Parser().parse(jsonFormatSchema) - - @transient private lazy val reader = new GenericDatumReader[Any](avroSchema) - - @transient private lazy val deserializer = new AvroDeserializer(avroSchema, dataType) - - @transient private var decoder: BinaryDecoder = _ - - @transient private var result: Any = _ - - override def nullSafeEval(input: Any): Any = { -val binary = input.asInstanceOf[Array[Byte]] -decoder = DecoderFactory.get().binaryDecoder(binary, 0, binary.length, decoder) -result = reader.read(result, decoder) -deserializer.deserialize(result) - } - - override def simpleString: String = { -s"from_avro(${child.sql}, ${dataType.simpleString})" - } - - override def sql: String = { -s"from_avro(${child.sql}, ${dataType.catalogString})" - } - - override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { -val expr = ctx.addReferenceObj("this", this) -defineCodeGen(ctx, ev, input => - s"(${CodeGenerator.boxedType(dataType)})$expr.nullSafeEval($input)") - } -} http://git-wip-us.apache.org/repos/asf/spark/blob/9ad77b30/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala -- diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala deleted file mode 10064
spark git commit: [SPARK-22880][SQL] Add cascadeTruncate option to JDBC datasource
Repository: spark Updated Branches: refs/heads/master 9ad77b303 -> 2333a34d3 [SPARK-22880][SQL] Add cascadeTruncate option to JDBC datasource This commit adds the `cascadeTruncate` option to the JDBC datasource API, for databases that support this functionality (PostgreSQL and Oracle at the moment). This allows for applying a cascading truncate that affects tables that have foreign key constraints on the table being truncated. ## What changes were proposed in this pull request? Add `cascadeTruncate` option to JDBC datasource API. Allow this to affect the `TRUNCATE` query for databases that support this option. ## How was this patch tested? Existing tests for `truncateQuery` were updated. Also, an additional test was added to ensure that the correct syntax was applied, and that enabling the config for databases that do not support this option does not result in invalid queries. Author: Daniel van der Ende Closes #20057 from danielvdende/SPARK-22880. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2333a34d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2333a34d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2333a34d Branch: refs/heads/master Commit: 2333a34d390f2fa19b939b8007be0deb31f31d3c Parents: 9ad77b3 Author: Daniel van der Ende Authored: Fri Jul 20 13:03:57 2018 -0700 Committer: Xiao Li Committed: Fri Jul 20 13:03:57 2018 -0700 -- docs/sql-programming-guide.md | 7 +++ .../datasources/jdbc/JDBCOptions.scala | 3 ++ .../execution/datasources/jdbc/JdbcUtils.scala | 7 ++- .../spark/sql/jdbc/AggregatedDialect.scala | 13 +- .../apache/spark/sql/jdbc/DerbyDialect.scala| 2 + .../apache/spark/sql/jdbc/JdbcDialects.scala| 20 - .../apache/spark/sql/jdbc/OracleDialect.scala | 16 +++ .../apache/spark/sql/jdbc/PostgresDialect.scala | 29 .../apache/spark/sql/jdbc/TeradataDialect.scala | 18 .../org/apache/spark/sql/jdbc/JDBCSuite.scala | 46 +--- 10 files changed, 140 insertions(+), 21 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/2333a34d/docs/sql-programming-guide.md -- diff --git a/docs/sql-programming-guide.md b/docs/sql-programming-guide.md index ad23dae..4bab58a 100644 --- a/docs/sql-programming-guide.md +++ b/docs/sql-programming-guide.md @@ -1407,6 +1407,13 @@ the following case-insensitive options: This is a JDBC writer related option. When SaveMode.Overwrite is enabled, this option causes Spark to truncate an existing table instead of dropping and recreating it. This can be more efficient, and prevents the table metadata (e.g., indices) from being removed. However, it will not work in some cases, such as when the new data has a different schema. It defaults to false. This option applies only to writing. + + +cascadeTruncate + +This is a JDBC writer related option. If enabled and supported by the JDBC database (PostgreSQL and Oracle at the moment), this options allows execution of a TRUNCATE TABLE t CASCADE (in the case of PostgreSQL a TRUNCATE TABLE ONLY t CASCADE is executed to prevent inadvertently truncating descendant tables). This will affect other tables, and thus should be used with care. This option applies only to writing. It defaults to the default cascading truncate behaviour of the JDBC database in question, specified in the isCascadeTruncate in each JDBCDialect. + + createTableOptions http://git-wip-us.apache.org/repos/asf/spark/blob/2333a34d/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala index eea966d..574aed4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala @@ -157,6 +157,8 @@ class JDBCOptions( // // if to truncate the table from the JDBC database val isTruncate = parameters.getOrElse(JDBC_TRUNCATE, "false").toBoolean + + val isCascadeTruncate: Option[Boolean] = parameters.get(JDBC_CASCADE_TRUNCATE).map(_.toBoolean) // the create table option , which can be table_options or partition_options. // E.g., "CREATE TABLE t (name string) ENGINE=InnoDB DEFAULT CHARSET=utf8" // TODO: to reuse the existing partition parameters for those partition specific options @
spark git commit: [SPARK-24876][SQL] Avro: simplify schema serialization
Repository: spark Updated Branches: refs/heads/master 2333a34d3 -> 00b864aa7 [SPARK-24876][SQL] Avro: simplify schema serialization ## What changes were proposed in this pull request? Previously in the refactoring of Avro Serializer and Deserializer, a new class SerializableSchema is created for serializing the Avro schema: https://github.com/apache/spark/pull/21762/files#diff-01fea32e6ec6bcf6f34d06282e08705aR37 On second thought, we can use `toString` method for serialization. After that, parse the JSON format schema on executor. This makes the code much simpler. ## How was this patch tested? Unit test Author: Gengliang Wang Closes #21829 from gengliangwang/removeSerializableSchema. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/00b864aa Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/00b864aa Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/00b864aa Branch: refs/heads/master Commit: 00b864aa7054a34f3d7a118d92eae0b3c28b86e5 Parents: 2333a34 Author: Gengliang Wang Authored: Fri Jul 20 14:57:59 2018 -0700 Committer: Xiao Li Committed: Fri Jul 20 14:57:59 2018 -0700 -- .../apache/spark/sql/avro/AvroFileFormat.scala | 2 +- .../sql/avro/AvroOutputWriterFactory.scala | 14 +++- .../spark/sql/avro/SerializableSchema.scala | 69 .../sql/avro/SerializableSchemaSuite.scala | 56 4 files changed, 12 insertions(+), 129 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/00b864aa/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala -- diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala index 1d0f40e..780e457 100755 --- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala +++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala @@ -146,7 +146,7 @@ private[avro] class AvroFileFormat extends FileFormat with DataSourceRegister { log.error(s"unsupported compression codec $unknown") } -new AvroOutputWriterFactory(dataSchema, new SerializableSchema(outputAvroSchema)) +new AvroOutputWriterFactory(dataSchema, outputAvroSchema.toString) } override def buildReader( http://git-wip-us.apache.org/repos/asf/spark/blob/00b864aa/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriterFactory.scala -- diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriterFactory.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriterFactory.scala index 18a6d93..116020e 100644 --- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriterFactory.scala +++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriterFactory.scala @@ -17,14 +17,22 @@ package org.apache.spark.sql.avro +import org.apache.avro.Schema import org.apache.hadoop.mapreduce.TaskAttemptContext import org.apache.spark.sql.execution.datasources.{OutputWriter, OutputWriterFactory} import org.apache.spark.sql.types.StructType +/** + * A factory that produces [[AvroOutputWriter]]. + * @param catalystSchema Catalyst schema of input data. + * @param avroSchemaAsJsonString Avro schema of output result, in JSON string format. + */ private[avro] class AvroOutputWriterFactory( -schema: StructType, -avroSchema: SerializableSchema) extends OutputWriterFactory { +catalystSchema: StructType, +avroSchemaAsJsonString: String) extends OutputWriterFactory { + + private lazy val avroSchema = new Schema.Parser().parse(avroSchemaAsJsonString) override def getFileExtension(context: TaskAttemptContext): String = ".avro" @@ -32,6 +40,6 @@ private[avro] class AvroOutputWriterFactory( path: String, dataSchema: StructType, context: TaskAttemptContext): OutputWriter = { -new AvroOutputWriter(path, context, schema, avroSchema.value) +new AvroOutputWriter(path, context, catalystSchema, avroSchema) } } http://git-wip-us.apache.org/repos/asf/spark/blob/00b864aa/external/avro/src/main/scala/org/apache/spark/sql/avro/SerializableSchema.scala -- diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/SerializableSchema.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/SerializableSchema.scala deleted file mode 100644 index ec0ddc7..000 --- a/external/avro/src/main/scala/org/apache/spark/sql/avro/SerializableSchema.scala +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache
spark git commit: [SPARK-24880][BUILD] Fix the group id for spark-kubernetes-integration-tests
Repository: spark Updated Branches: refs/heads/master 00b864aa7 -> f765bb782 [SPARK-24880][BUILD] Fix the group id for spark-kubernetes-integration-tests ## What changes were proposed in this pull request? The correct group id should be `org.apache.spark`. This is causing the nightly build failure: https://amplab.cs.berkeley.edu/jenkins/job/spark-master-maven-snapshots/2295/console ` [ERROR] Failed to execute goal org.apache.maven.plugins:maven-deploy-plugin:2.8.2:deploy (default-deploy) on project spark-kubernetes-integration-tests_2.11: Failed to deploy artifacts: Could not transfer artifact spark-kubernetes-integration-tests:spark-kubernetes-integration-tests_2.11:jar:2.4.0-20180720.101629-1 from/to apache.snapshots.https (https://repository.apache.org/content/repositories/snapshots): Access denied to: https://repository.apache.org/content/repositories/snapshots/spark-kubernetes-integration-tests/spark-kubernetes-integration-tests_2.11/2.4.0-SNAPSHOT/spark-kubernetes-integration-tests_2.11-2.4.0-20180720.101629-1.jar, ReasonPhrase: Forbidden. -> [Help 1] [ERROR] ` ## How was this patch tested? Jenkins. Author: zsxwing Closes #21831 from zsxwing/fix-k8s-test. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f765bb78 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f765bb78 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f765bb78 Branch: refs/heads/master Commit: f765bb7823a60440cb42819edd98b14f65e13b18 Parents: 00b864a Author: zsxwing Authored: Fri Jul 20 15:23:04 2018 -0700 Committer: Xiao Li Committed: Fri Jul 20 15:23:04 2018 -0700 -- resource-managers/kubernetes/integration-tests/pom.xml | 1 - 1 file changed, 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/f765bb78/resource-managers/kubernetes/integration-tests/pom.xml -- diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 29334cc..614705c 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -25,7 +25,6 @@ spark-kubernetes-integration-tests_2.11 - spark-kubernetes-integration-tests 1.3.0 1.4.0 - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [PYSPARK][TEST][MINOR] Fix UDFInitializationTests
Repository: spark Updated Branches: refs/heads/master 597bdeff2 -> 96f312076 [PYSPARK][TEST][MINOR] Fix UDFInitializationTests ## What changes were proposed in this pull request? Fix a typo in pyspark sql tests Author: William Sheu Closes #21833 from PenguinToast/fix-test-typo. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/96f31207 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/96f31207 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/96f31207 Branch: refs/heads/master Commit: 96f3120760ba0a83ef6347327ecfb130487e02dd Parents: 597bdef Author: William Sheu Authored: Fri Jul 20 19:48:32 2018 -0700 Committer: Xiao Li Committed: Fri Jul 20 19:48:32 2018 -0700 -- python/pyspark/sql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/96f31207/python/pyspark/sql/tests.py -- diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py index 565654e..2d6b9f0 100644 --- a/python/pyspark/sql/tests.py +++ b/python/pyspark/sql/tests.py @@ -3595,7 +3595,7 @@ class UDFInitializationTests(unittest.TestCase): SparkSession._instantiatedSession.stop() if SparkContext._active_spark_context is not None: -SparkContext._active_spark_contex.stop() +SparkContext._active_spark_context.stop() def test_udf_init_shouldnt_initalize_context(self): from pyspark.sql.functions import UserDefinedFunction - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24879][SQL] Fix NPE in Hive partition pruning filter pushdown
Repository: spark Updated Branches: refs/heads/master 96f312076 -> bbd6f0c25 [SPARK-24879][SQL] Fix NPE in Hive partition pruning filter pushdown ## What changes were proposed in this pull request? We get a NPE when we have a filter on a partition column of the form `col in (x, null)`. This is due to the filter converter in HiveShim not handling `null`s correctly. This patch fixes this bug while still pushing down as much of the partition pruning predicates as possible, by filtering out `null`s from any `in` predicate. Since Hive only supports very simple partition pruning filters, this change should preserve correctness. ## How was this patch tested? Unit tests, manual tests Author: William Sheu Closes #21832 from PenguinToast/partition-pruning-npe. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bbd6f0c2 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bbd6f0c2 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bbd6f0c2 Branch: refs/heads/master Commit: bbd6f0c25fe19dc6c946e63cac7b98d0f78b3463 Parents: 96f3120 Author: William Sheu Authored: Fri Jul 20 19:59:28 2018 -0700 Committer: Xiao Li Committed: Fri Jul 20 19:59:28 2018 -0700 -- .../apache/spark/sql/hive/client/HiveShim.scala | 19 ++- .../spark/sql/hive/client/FiltersSuite.scala | 14 ++ 2 files changed, 32 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/bbd6f0c2/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala -- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala index 933384e..bc9d4cd 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala @@ -598,6 +598,7 @@ private[client] class Shim_v0_13 extends Shim_v0_12 { object ExtractableLiteral { def unapply(expr: Expression): Option[String] = expr match { +case Literal(null, _) => None // `null`s can be cast as other types; we want to avoid NPEs. case Literal(value, _: IntegralType) => Some(value.toString) case Literal(value, _: StringType) => Some(quoteStringLiteral(value.toString)) case _ => None @@ -606,7 +607,23 @@ private[client] class Shim_v0_13 extends Shim_v0_12 { object ExtractableLiterals { def unapply(exprs: Seq[Expression]): Option[Seq[String]] = { -val extractables = exprs.map(ExtractableLiteral.unapply) +// SPARK-24879: The Hive metastore filter parser does not support "null", but we still want +// to push down as many predicates as we can while still maintaining correctness. +// In SQL, the `IN` expression evaluates as follows: +// > `1 in (2, NULL)` -> NULL +// > `1 in (1, NULL)` -> true +// > `1 in (2)` -> false +// Since Hive metastore filters are NULL-intolerant binary operations joined only by +// `AND` and `OR`, we can treat `NULL` as `false` and thus rewrite `1 in (2, NULL)` as +// `1 in (2)`. +// If the Hive metastore begins supporting NULL-tolerant predicates and Spark starts +// pushing down these predicates, then this optimization will become incorrect and need +// to be changed. +val extractables = exprs +.filter { + case Literal(null, _) => false + case _ => true +}.map(ExtractableLiteral.unapply) if (extractables.nonEmpty && extractables.forall(_.isDefined)) { Some(extractables.map(_.get)) } else { http://git-wip-us.apache.org/repos/asf/spark/blob/bbd6f0c2/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala -- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala index 1976569..2a4efd0 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala @@ -72,6 +72,20 @@ class FiltersSuite extends SparkFunSuite with Logging with PlanTest { (Literal("p2\" and q=\"q2") === a("stringcol", StringType)) :: Nil, """stringcol = 'p1" and q="q1' and 'p2" and q="q2' = stringcol""") + filterTest("SPARK-24879 null literals should be ignored for IN constructs", +(a("intcol", IntegerType) in (Literal(1), Literal(null))) :: Nil, +"(intcol = 1)") + + // Applying the predicate `x IN (NULL)` sho
spark git commit: [SPARK-24879][SQL] Fix NPE in Hive partition pruning filter pushdown
Repository: spark Updated Branches: refs/heads/branch-2.3 db1f3cc76 -> bd6bfacb2 [SPARK-24879][SQL] Fix NPE in Hive partition pruning filter pushdown ## What changes were proposed in this pull request? We get a NPE when we have a filter on a partition column of the form `col in (x, null)`. This is due to the filter converter in HiveShim not handling `null`s correctly. This patch fixes this bug while still pushing down as much of the partition pruning predicates as possible, by filtering out `null`s from any `in` predicate. Since Hive only supports very simple partition pruning filters, this change should preserve correctness. ## How was this patch tested? Unit tests, manual tests Author: William Sheu Closes #21832 from PenguinToast/partition-pruning-npe. (cherry picked from commit bbd6f0c25fe19dc6c946e63cac7b98d0f78b3463) Signed-off-by: Xiao Li Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bd6bfacb Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bd6bfacb Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bd6bfacb Branch: refs/heads/branch-2.3 Commit: bd6bfacb2ef473b858cc91e0692591deaea26118 Parents: db1f3cc Author: William Sheu Authored: Fri Jul 20 19:59:28 2018 -0700 Committer: Xiao Li Committed: Fri Jul 20 20:00:17 2018 -0700 -- .../apache/spark/sql/hive/client/HiveShim.scala | 19 ++- .../spark/sql/hive/client/FiltersSuite.scala | 14 ++ 2 files changed, 32 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/bd6bfacb/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala -- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala index 60fe31f..68acea7 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala @@ -599,6 +599,7 @@ private[client] class Shim_v0_13 extends Shim_v0_12 { object ExtractableLiteral { def unapply(expr: Expression): Option[String] = expr match { +case Literal(null, _) => None // `null`s can be cast as other types; we want to avoid NPEs. case Literal(value, _: IntegralType) => Some(value.toString) case Literal(value, _: StringType) => Some(quoteStringLiteral(value.toString)) case _ => None @@ -607,7 +608,23 @@ private[client] class Shim_v0_13 extends Shim_v0_12 { object ExtractableLiterals { def unapply(exprs: Seq[Expression]): Option[Seq[String]] = { -val extractables = exprs.map(ExtractableLiteral.unapply) +// SPARK-24879: The Hive metastore filter parser does not support "null", but we still want +// to push down as many predicates as we can while still maintaining correctness. +// In SQL, the `IN` expression evaluates as follows: +// > `1 in (2, NULL)` -> NULL +// > `1 in (1, NULL)` -> true +// > `1 in (2)` -> false +// Since Hive metastore filters are NULL-intolerant binary operations joined only by +// `AND` and `OR`, we can treat `NULL` as `false` and thus rewrite `1 in (2, NULL)` as +// `1 in (2)`. +// If the Hive metastore begins supporting NULL-tolerant predicates and Spark starts +// pushing down these predicates, then this optimization will become incorrect and need +// to be changed. +val extractables = exprs +.filter { + case Literal(null, _) => false + case _ => true +}.map(ExtractableLiteral.unapply) if (extractables.nonEmpty && extractables.forall(_.isDefined)) { Some(extractables.map(_.get)) } else { http://git-wip-us.apache.org/repos/asf/spark/blob/bd6bfacb/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala -- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala index 1976569..2a4efd0 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala @@ -72,6 +72,20 @@ class FiltersSuite extends SparkFunSuite with Logging with PlanTest { (Literal("p2\" and q=\"q2") === a("stringcol", StringType)) :: Nil, """stringcol = 'p1" and q="q1' and 'p2" and q="q2' = stringcol""") + filterTest("SPARK-24879 null literals should be ignored for IN constructs", +(a("intcol", IntegerType) in (Lit
spark git commit: [SPARK-24836][SQL] New option for Avro datasource - ignoreExtension
Repository: spark Updated Branches: refs/heads/master bbd6f0c25 -> 106880edc [SPARK-24836][SQL] New option for Avro datasource - ignoreExtension ## What changes were proposed in this pull request? I propose to add new option for AVRO datasource which should control ignoring of files without `.avro` extension in read. The option has name `ignoreExtension` with default value `true`. If both options `ignoreExtension` and `avro.mapred.ignore.inputs.without.extension` are set, `ignoreExtension` overrides the former one. Here is an example of usage: ``` spark .read .option("ignoreExtension", false) .avro("path to avro files") ``` ## How was this patch tested? I added a test which checks the option directly and a test for checking that new option overrides hadoop's config. Author: Maxim Gekk Closes #21798 from MaxGekk/avro-ignore-extension. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/106880ed Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/106880ed Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/106880ed Branch: refs/heads/master Commit: 106880edcd67bc20e8610a16f8ce6aa250268eeb Parents: bbd6f0c Author: Maxim Gekk Authored: Fri Jul 20 20:04:40 2018 -0700 Committer: Xiao Li Committed: Fri Jul 20 20:04:40 2018 -0700 -- .../apache/spark/sql/avro/AvroFileFormat.scala | 33 +--- .../org/apache/spark/sql/avro/AvroOptions.scala | 29 +-- .../org/apache/spark/sql/avro/AvroSuite.scala | 55 +++- 3 files changed, 91 insertions(+), 26 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/106880ed/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala -- diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala index 780e457..078efab 100755 --- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala +++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala @@ -58,21 +58,19 @@ private[avro] class AvroFileFormat extends FileFormat with DataSourceRegister { options: Map[String, String], files: Seq[FileStatus]): Option[StructType] = { val conf = spark.sparkContext.hadoopConfiguration -val parsedOptions = new AvroOptions(options) +val parsedOptions = new AvroOptions(options, conf) // Schema evolution is not supported yet. Here we only pick a single random sample file to // figure out the schema of the whole dataset. val sampleFile = - if (AvroFileFormat.ignoreFilesWithoutExtensions(conf)) { -files.find(_.getPath.getName.endsWith(".avro")).getOrElse { - throw new FileNotFoundException( -"No Avro files found. Hadoop option \"avro.mapred.ignore.inputs.without.extension\" " + - " is set to true. Do all input files have \".avro\" extension?" - ) + if (parsedOptions.ignoreExtension) { +files.headOption.getOrElse { + throw new FileNotFoundException("Files for schema inferring have been not found.") } } else { -files.headOption.getOrElse { - throw new FileNotFoundException("No Avro files found.") +files.find(_.getPath.getName.endsWith(".avro")).getOrElse { + throw new FileNotFoundException( +"No Avro files found. If files don't have .avro extension, set ignoreExtension to true") } } @@ -115,7 +113,7 @@ private[avro] class AvroFileFormat extends FileFormat with DataSourceRegister { job: Job, options: Map[String, String], dataSchema: StructType): OutputWriterFactory = { -val parsedOptions = new AvroOptions(options) +val parsedOptions = new AvroOptions(options, spark.sessionState.newHadoopConf()) val outputAvroSchema = SchemaConverters.toAvroType( dataSchema, nullable = false, parsedOptions.recordName, parsedOptions.recordNamespace) @@ -160,7 +158,7 @@ private[avro] class AvroFileFormat extends FileFormat with DataSourceRegister { val broadcastedConf = spark.sparkContext.broadcast(new AvroFileFormat.SerializableConfiguration(hadoopConf)) -val parsedOptions = new AvroOptions(options) +val parsedOptions = new AvroOptions(options, hadoopConf) (file: PartitionedFile) => { val log = LoggerFactory.getLogger(classOf[AvroFileFormat]) @@ -171,9 +169,7 @@ private[avro] class AvroFileFormat extends FileFormat with DataSourceRegister { // Doing input file filtering is improper because we may generate empty tasks that process no // input files but stress the scheduler. We should probably add a more general in
spark git commit: [SPARK-24811][SQL] Avro: add new function from_avro and to_avro
Repository: spark Updated Branches: refs/heads/master 81af88687 -> 8817c68f5 [SPARK-24811][SQL] Avro: add new function from_avro and to_avro ## What changes were proposed in this pull request? 1. Add a new function from_avro for parsing a binary column of avro format and converting it into its corresponding catalyst value. 2. Add a new function to_avro for converting a column into binary of avro format with the specified schema. I created #21774 for this, but it failed the build https://amplab.cs.berkeley.edu/jenkins/view/Spark%20QA%20Compile/job/spark-master-compile-maven-hadoop-2.6/7902/ Additional changes In this PR: 1. Add `scalacheck` dependency in pom.xml to resolve the failure. 2. Update the `log4j.properties` to make it consistent with other modules. ## How was this patch tested? Unit test Compile with different commands: ``` ./build/mvn --force -DzincPort=3643 -DskipTests -Phadoop-2.6 -Phive-thriftserver -Pkinesis-asl -Pspark-ganglia-lgpl -Pmesos -Pyarn compile test-compile ./build/mvn --force -DzincPort=3643 -DskipTests -Phadoop-2.7 -Phive-thriftserver -Pkinesis-asl -Pspark-ganglia-lgpl -Pmesos -Pyarn compile test-compile ./build/mvn --force -DzincPort=3643 -DskipTests -Phadoop-3.1 -Phive-thriftserver -Pkinesis-asl -Pspark-ganglia-lgpl -Pmesos -Pyarn compile test-compile ``` Author: Gengliang Wang Closes #21838 from gengliangwang/from_and_to_avro. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8817c68f Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8817c68f Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8817c68f Branch: refs/heads/master Commit: 8817c68f5099901753585716e00281736938bca0 Parents: 81af886 Author: Gengliang Wang Authored: Sun Jul 22 17:36:57 2018 -0700 Committer: Xiao Li Committed: Sun Jul 22 17:36:57 2018 -0700 -- external/avro/pom.xml | 5 + .../spark/sql/avro/AvroDataToCatalyst.scala | 68 +++ .../spark/sql/avro/CatalystDataToAvro.scala | 69 .../org/apache/spark/sql/avro/package.scala | 31 .../avro/src/test/resources/log4j.properties| 39 + .../avro/AvroCatalystDataConversionSuite.scala | 175 +++ .../spark/sql/avro/AvroFunctionsSuite.scala | 83 + .../expressions/ExpressionEvalHelper.scala | 6 + 8 files changed, 446 insertions(+), 30 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/8817c68f/external/avro/pom.xml -- diff --git a/external/avro/pom.xml b/external/avro/pom.xml index 42e865b..ad7df1f 100644 --- a/external/avro/pom.xml +++ b/external/avro/pom.xml @@ -62,6 +62,11 @@ test + org.scalacheck + scalacheck_${scala.binary.version} + test + + org.apache.spark spark-tags_${scala.binary.version} http://git-wip-us.apache.org/repos/asf/spark/blob/8817c68f/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala -- diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala new file mode 100644 index 000..6671b3f --- /dev/null +++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.avro.Schema +import org.apache.avro.generic.GenericDatumReader +import org.apache.avro.io.{BinaryDecoder, DecoderFactory} + +import org.apache.spark.sql.avro.{AvroDeserializer, SchemaConverters} +import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, UnaryExpression} +import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode} +import org.apache.spark.sql.types.{AbstractDataType, BinaryType, DataType} + +case class AvroDa
spark git commit: [SPARK-24802][SQL] Add a new config for Optimization Rule Exclusion
Repository: spark Updated Branches: refs/heads/master ab18b02e6 -> 434319e73 [SPARK-24802][SQL] Add a new config for Optimization Rule Exclusion ## What changes were proposed in this pull request? Since Spark has provided fairly clear interfaces for adding user-defined optimization rules, it would be nice to have an easy-to-use interface for excluding an optimization rule from the Spark query optimizer as well. This would make customizing Spark optimizer easier and sometimes could debugging issues too. - Add a new config spark.sql.optimizer.excludedRules, with the value being a list of rule names separated by comma. - Modify the current batches method to remove the excluded rules from the default batches. Log the rules that have been excluded. - Split the existing default batches into "post-analysis batches" and "optimization batches" so that only rules in the "optimization batches" can be excluded. ## How was this patch tested? Add a new test suite: OptimizerRuleExclusionSuite Author: maryannxue Closes #21764 from maryannxue/rule-exclusion. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/434319e7 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/434319e7 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/434319e7 Branch: refs/heads/master Commit: 434319e73f8cb6e080671bdde42a72228bd814ef Parents: ab18b02 Author: maryannxue Authored: Mon Jul 23 08:25:24 2018 -0700 Committer: Xiao Li Committed: Mon Jul 23 08:25:24 2018 -0700 -- .../sql/catalyst/optimizer/Optimizer.scala | 53 +- .../org/apache/spark/sql/internal/SQLConf.scala | 10 ++ .../optimizer/OptimizerRuleExclusionSuite.scala | 101 +++ 3 files changed, 163 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/434319e7/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index 2cc27d8..6faecd3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -46,7 +46,7 @@ abstract class Optimizer(sessionCatalog: SessionCatalog) protected def fixedPoint = FixedPoint(SQLConf.get.optimizerMaxIterations) - def batches: Seq[Batch] = { + def defaultBatches: Seq[Batch] = { val operatorOptimizationRuleSet = Seq( // Operator push down @@ -160,6 +160,22 @@ abstract class Optimizer(sessionCatalog: SessionCatalog) UpdateNullabilityInAttributeReferences) } + def nonExcludableRules: Seq[String] = +EliminateDistinct.ruleName :: + EliminateSubqueryAliases.ruleName :: + EliminateView.ruleName :: + ReplaceExpressions.ruleName :: + ComputeCurrentTime.ruleName :: + GetCurrentDatabase(sessionCatalog).ruleName :: + RewriteDistinctAggregates.ruleName :: + ReplaceDeduplicateWithAggregate.ruleName :: + ReplaceIntersectWithSemiJoin.ruleName :: + ReplaceExceptWithFilter.ruleName :: + ReplaceExceptWithAntiJoin.ruleName :: + ReplaceDistinctWithAggregate.ruleName :: + PullupCorrelatedPredicates.ruleName :: + RewritePredicateSubquery.ruleName :: Nil + /** * Optimize all the subqueries inside expression. */ @@ -175,6 +191,41 @@ abstract class Optimizer(sessionCatalog: SessionCatalog) * Override to provide additional rules for the operator optimization batch. */ def extendedOperatorOptimizationRules: Seq[Rule[LogicalPlan]] = Nil + + override def batches: Seq[Batch] = { +val excludedRulesConf = + SQLConf.get.optimizerExcludedRules.toSeq.flatMap(Utils.stringToSeq) +val excludedRules = excludedRulesConf.filter { ruleName => + val nonExcludable = nonExcludableRules.contains(ruleName) + if (nonExcludable) { +logWarning(s"Optimization rule '${ruleName}' was not excluded from the optimizer " + + s"because this rule is a non-excludable rule.") + } + !nonExcludable +} +if (excludedRules.isEmpty) { + defaultBatches +} else { + defaultBatches.flatMap { batch => +val filteredRules = batch.rules.filter { rule => + val exclude = excludedRules.contains(rule.ruleName) + if (exclude) { +logInfo(s"Optimization rule '${rule.ruleName}' is excluded from the optimizer.") + } + !exclude +} +if (batch.rules == filteredRules) { + Some(batch) +} else if (filteredRules.nonEmpty) { + Some(Batch(batch.nam
spark git commit: [SPARK-24887][SQL] Avro: use SerializableConfiguration in Spark utils to deduplicate code
Repository: spark Updated Branches: refs/heads/master 434319e73 -> 08e315f63 [SPARK-24887][SQL] Avro: use SerializableConfiguration in Spark utils to deduplicate code ## What changes were proposed in this pull request? To implement the method `buildReader` in `FileFormat`, it is required to serialize the hadoop configuration for executors. Previous spark-avro uses its own class `SerializableConfiguration` for the serialization. As now it is part of Spark, we can use SerializableConfiguration in Spark util to deduplicate the code. ## How was this patch tested? Unit test Author: Gengliang Wang Closes #21846 from gengliangwang/removeSerializableConfiguration. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/08e315f6 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/08e315f6 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/08e315f6 Branch: refs/heads/master Commit: 08e315f6330984b757f241079dfc9e1028e5cd0a Parents: 434319e Author: Gengliang Wang Authored: Mon Jul 23 08:31:48 2018 -0700 Committer: Xiao Li Committed: Mon Jul 23 08:31:48 2018 -0700 -- .../apache/spark/sql/avro/AvroFileFormat.scala | 44 + .../avro/SerializableConfigurationSuite.scala | 50 2 files changed, 2 insertions(+), 92 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/08e315f6/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala -- diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala index 078efab..b043252 100755 --- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala +++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala @@ -23,8 +23,6 @@ import java.util.zip.Deflater import scala.util.control.NonFatal -import com.esotericsoftware.kryo.{Kryo, KryoSerializable} -import com.esotericsoftware.kryo.io.{Input, Output} import org.apache.avro.Schema import org.apache.avro.file.{DataFileConstants, DataFileReader} import org.apache.avro.generic.{GenericDatumReader, GenericRecord} @@ -41,6 +39,7 @@ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.execution.datasources.{FileFormat, OutputWriterFactory, PartitionedFile} import org.apache.spark.sql.sources.{DataSourceRegister, Filter} import org.apache.spark.sql.types.StructType +import org.apache.spark.util.SerializableConfiguration private[avro] class AvroFileFormat extends FileFormat with DataSourceRegister { private val log = LoggerFactory.getLogger(getClass) @@ -157,7 +156,7 @@ private[avro] class AvroFileFormat extends FileFormat with DataSourceRegister { hadoopConf: Configuration): (PartitionedFile) => Iterator[InternalRow] = { val broadcastedConf = - spark.sparkContext.broadcast(new AvroFileFormat.SerializableConfiguration(hadoopConf)) + spark.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) val parsedOptions = new AvroOptions(options, hadoopConf) (file: PartitionedFile) => { @@ -233,43 +232,4 @@ private[avro] class AvroFileFormat extends FileFormat with DataSourceRegister { private[avro] object AvroFileFormat { val IgnoreFilesWithoutExtensionProperty = "avro.mapred.ignore.inputs.without.extension" - - class SerializableConfiguration(@transient var value: Configuration) - extends Serializable with KryoSerializable { -@transient private[avro] lazy val log = LoggerFactory.getLogger(getClass) - -private def writeObject(out: ObjectOutputStream): Unit = tryOrIOException { - out.defaultWriteObject() - value.write(out) -} - -private def readObject(in: ObjectInputStream): Unit = tryOrIOException { - value = new Configuration(false) - value.readFields(in) -} - -private def tryOrIOException[T](block: => T): T = { - try { -block - } catch { -case e: IOException => - log.error("Exception encountered", e) - throw e -case NonFatal(e) => - log.error("Exception encountered", e) - throw new IOException(e) - } -} - -def write(kryo: Kryo, out: Output): Unit = { - val dos = new DataOutputStream(out) - value.write(dos) - dos.flush() -} - -def read(kryo: Kryo, in: Input): Unit = { - value = new Configuration(false) - value.readFields(new DataInputStream(in)) -} - } } http://git-wip-us.apache.org/repos/asf/spark/blob/08e315f6/external/avro/src/test/scala/org/apache/spark/sql/avro/SerializableConfigurationSuite.scala -- dif
spark git commit: [SPARK-24850][SQL] fix str representation of CachedRDDBuilder
Repository: spark Updated Branches: refs/heads/master 08e315f63 -> 2edf17eff [SPARK-24850][SQL] fix str representation of CachedRDDBuilder ## What changes were proposed in this pull request? As of https://github.com/apache/spark/pull/21018, InMemoryRelation includes its cacheBuilder when logging query plans. This PR changes the string representation of the CachedRDDBuilder to not include the cached spark plan. ## How was this patch tested? spark-shell, query: ``` var df_cached = spark.read.format("csv").option("header", "true").load("test.csv").cache() 0 to 1 foreach { _ => df_cached = df_cached.join(spark.read.format("csv").option("header", "true").load("test.csv"), "A").cache() } df_cached.explain ``` as of master results in: ``` == Physical Plan == InMemoryTableScan [A#10, B#11, B#35, B#87] +- InMemoryRelation [A#10, B#11, B#35, B#87], CachedRDDBuilder(true,1,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Project [A#10, B#11, B#35, B#87] +- *(2) BroadcastHashJoin [A#10], [A#86], Inner, BuildRight :- *(2) Filter isnotnull(A#10) : +- InMemoryTableScan [A#10, B#11, B#35], [isnotnull(A#10)] : +- InMemoryRelation [A#10, B#11, B#35], CachedRDDBuilder(true,1,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Project [A#10, B#11, B#35] +- *(2) BroadcastHashJoin [A#10], [A#34], Inner, BuildRight :- *(2) Filter isnotnull(A#10) : +- InMemoryTableScan [A#10, B#11], [isnotnull(A#10)] : +- InMemoryRelation [A#10, B#11], CachedRDDBuilder(true,1,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) FileScan csv [A#10,B#11] Batched: false, Format: CSV, Location: InMemoryFileIndex[file:test.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct ,None) : +- *(1) FileScan csv [A#10,B#11] Batched: false, Format: CSV, Location: InMemoryFileIndex[file:test.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct +- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false])) +- *(1) Filter isnotnull(A#34) +- InMemoryTableScan [A#34, B#35], [isnotnull(A#34)] +- InMemoryRelation [A#34, B#35], CachedRDDBuilder(true,1,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) FileScan csv [A#10,B#11] Batched: false, Format: CSV, Location: InMemoryFileIndex[file:test.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct ,None) +- *(1) FileScan csv [A#10,B#11] Batched: false, Format: CSV, Location: InMemoryFileIndex[file:test.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct ,None) : +- *(2) Project [A#10, B#11, B#35] : +- *(2) BroadcastHashJoin [A#10], [A#34], Inner, BuildRight : :- *(2) Filter isnotnull(A#10) : : +- InMemoryTableScan [A#10, B#11], [isnotnull(A#10)] : : +- InMemoryRelation [A#10, B#11], CachedRDDBuilder(true,1,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) FileScan csv [A#10,B#11] Batched: false, Format: CSV, Location: InMemoryFileIndex[file:test.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct ,None) : : +- *(1) FileScan csv [A#10,B#11] Batched: false, Format: CSV, Location: InMemoryFileIndex[file:test.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct : +- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false])) : +- *(1) Filter isnotnull(A#34) : +- InMemoryTableScan [A#34, B#35], [isnotnull(A#34)] : +- InMemoryRelation [A#34, B#35], CachedRDDBuilder(true,1,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) FileScan csv [A#10,B#11] Batched: false, Format: CSV, Location: InMemoryFileIndex[file:test.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct ,None) : +- *(1) FileScan csv [A#10,B#11] Batched: false, Format: CSV, Location: InMemoryFileIndex[file:test.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct +- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false])) +- *(1) Filter isnotnull(A#86) +- InMemoryTableScan [A#86, B#87], [isnotnull(A#86)] +- InMemoryRelation [A#86, B#87], CachedRDDBuilder(true,1,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) FileScan csv [A#10,B#11] Batched: false, Format: CSV, Location: InMemoryFileIndex[file:test.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct ,None) +- *(1) FileScan csv [A#10,B#11] Batched: false, Format: CSV, Location: InMemoryFileIndex[file:test.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct ,None) +- *(2) Project [A#10, B#11, B#35, B#87] +- *(2) BroadcastHashJoin [A#10], [A#86], Inner, BuildRight :- *(2) Filter isnotnull(A#10) : +- InMemoryTableScan [A#10, B#11, B#35], [isnotnull(A#10)] : +- InMemoryRelation [A#10, B#11, B#35], CachedRDDBuilder(true,1,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Project [A#10, B#11, B#35] +- *(2) BroadcastHashJoin [A#10], [A#34], Inner, BuildRight :- *(2) Filter isnotnull(A#10) : +- InMemoryTableScan [A#10, B#11], [isnotnull(A#10)] : +- InMemoryRelation [A#10, B#11], CachedRDDBuilder(true,100
spark git commit: [SPARK-24339][SQL] Prunes the unused columns from child of ScriptTransformation
Repository: spark Updated Branches: refs/heads/master 61f0ca4f1 -> cfc3e1aaa [SPARK-24339][SQL] Prunes the unused columns from child of ScriptTransformation ## What changes were proposed in this pull request? Modify the strategy in ColumnPruning to add a Project between ScriptTransformation and its child, this strategy can reduce the scan time especially in the scenario of the table has many columns. ## How was this patch tested? Add UT in ColumnPruningSuite and ScriptTransformationSuite. Author: Yuanjian Li Closes #21839 from xuanyuanking/SPARK-24339. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cfc3e1aa Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cfc3e1aa Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cfc3e1aa Branch: refs/heads/master Commit: cfc3e1aaa44b58da660be3378effdc48e088b9d3 Parents: 61f0ca4 Author: Yuanjian Li Authored: Mon Jul 23 13:04:39 2018 -0700 Committer: Xiao Li Committed: Mon Jul 23 13:04:39 2018 -0700 -- .../sql/catalyst/optimizer/Optimizer.scala | 5 +++- .../catalyst/optimizer/ColumnPruningSuite.scala | 24 .../execution/ScriptTransformationSuite.scala | 19 3 files changed, 47 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/cfc3e1aa/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index 6faecd3..5ed7412 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -501,13 +501,16 @@ object ColumnPruning extends Rule[LogicalPlan] { case d @ DeserializeToObject(_, _, child) if (child.outputSet -- d.references).nonEmpty => d.copy(child = prunedChild(child, d.references)) -// Prunes the unused columns from child of Aggregate/Expand/Generate +// Prunes the unused columns from child of Aggregate/Expand/Generate/ScriptTransformation case a @ Aggregate(_, _, child) if (child.outputSet -- a.references).nonEmpty => a.copy(child = prunedChild(child, a.references)) case f @ FlatMapGroupsInPandas(_, _, _, child) if (child.outputSet -- f.references).nonEmpty => f.copy(child = prunedChild(child, f.references)) case e @ Expand(_, _, child) if (child.outputSet -- e.references).nonEmpty => e.copy(child = prunedChild(child, e.references)) +case s @ ScriptTransformation(_, _, _, child, _) +if (child.outputSet -- s.references).nonEmpty => + s.copy(child = prunedChild(child, s.references)) // prune unrequired references case p @ Project(_, g: Generate) if p.references != g.outputSet => http://git-wip-us.apache.org/repos/asf/spark/blob/cfc3e1aa/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala -- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala index 8b05ba3..f6db3c9 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala @@ -140,6 +140,30 @@ class ColumnPruningSuite extends PlanTest { comparePlans(optimized, expected) } + test("Column pruning for ScriptTransformation") { +val input = LocalRelation('a.int, 'b.string, 'c.double) +val query = + ScriptTransformation( +Seq('a, 'b), +"func", +Seq.empty, +input, +null).analyze +val optimized = Optimize.execute(query) + +val expected = + ScriptTransformation( +Seq('a, 'b), +"func", +Seq.empty, +Project( + Seq('a, 'b), + input), +null).analyze + +comparePlans(optimized, expected) + } + test("Column pruning on Filter") { val input = LocalRelation('a.int, 'b.string, 'c.double) val plan1 = Filter('a > 1, input).analyze http://git-wip-us.apache.org/repos/asf/spark/blob/cfc3e1aa/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala -- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala b/sql/hive/src/test/scala/org/ap
spark git commit: [SPARK-24870][SQL] Cache can't work normally if there are case letters in SQL
Repository: spark Updated Branches: refs/heads/master d2436a852 -> 13a67b070 [SPARK-24870][SQL] Cache can't work normally if there are case letters in SQL ## What changes were proposed in this pull request? Modified the canonicalized to not case-insensitive. Before the PR, cache can't work normally if there are case letters in SQL, for example: sql("CREATE TABLE IF NOT EXISTS src (key INT, value STRING) USING hive") sql("select key, sum(case when Key > 0 then 1 else 0 end) as positiveNum " + "from src group by key").cache().createOrReplaceTempView("src_cache") sql( s"""select a.key from (select key from src_cache where positiveNum = 1)a left join (select key from src_cache )b on a.key=b.key """).explain The physical plan of the sql is: ![image](https://user-images.githubusercontent.com/26834091/42979518-3decf0fa-8c05-11e8-9837-d5e4c334cb1f.png) The subquery "select key from src_cache where positiveNum = 1" on the left of join can use the cache data, but the subquery "select key from src_cache" on the right of join cannot use the cache data. ## How was this patch tested? new added test Author: 10129659 Closes #21823 from eatoncys/canonicalized. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/13a67b07 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/13a67b07 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/13a67b07 Branch: refs/heads/master Commit: 13a67b070d335bb257d13dacadea3450885c3d81 Parents: d2436a8 Author: 10129659 Authored: Mon Jul 23 23:05:08 2018 -0700 Committer: Xiao Li Committed: Mon Jul 23 23:05:08 2018 -0700 -- .../apache/spark/sql/catalyst/plans/QueryPlan.scala | 2 +- .../apache/spark/sql/execution/SameResultSuite.scala | 15 +++ 2 files changed, 16 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/13a67b07/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala index 4b4722b..b1ffdca 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala @@ -284,7 +284,7 @@ object QueryPlan extends PredicateHelper { if (ordinal == -1) { ar } else { - ar.withExprId(ExprId(ordinal)) + ar.withExprId(ExprId(ordinal)).canonicalized } }.canonicalized.asInstanceOf[T] } http://git-wip-us.apache.org/repos/asf/spark/blob/13a67b07/sql/core/src/test/scala/org/apache/spark/sql/execution/SameResultSuite.scala -- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SameResultSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SameResultSuite.scala index aaf51b5..d088e24 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SameResultSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SameResultSuite.scala @@ -18,8 +18,11 @@ package org.apache.spark.sql.execution import org.apache.spark.sql.{DataFrame, QueryTest} +import org.apache.spark.sql.catalyst.expressions.AttributeReference +import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, Project} import org.apache.spark.sql.functions._ import org.apache.spark.sql.test.SharedSQLContext +import org.apache.spark.sql.types.IntegerType /** * Tests for the sameResult function for [[SparkPlan]]s. @@ -58,4 +61,16 @@ class SameResultSuite extends QueryTest with SharedSQLContext { val df4 = spark.range(10).agg(sumDistinct($"id")) assert(df3.queryExecution.executedPlan.sameResult(df4.queryExecution.executedPlan)) } + + test("Canonicalized result is case-insensitive") { +val a = AttributeReference("A", IntegerType)() +val b = AttributeReference("B", IntegerType)() +val planUppercase = Project(Seq(a), LocalRelation(a, b)) + +val c = AttributeReference("a", IntegerType)() +val d = AttributeReference("b", IntegerType)() +val planLowercase = Project(Seq(c), LocalRelation(c, d)) + +assert(planUppercase.sameResult(planLowercase)) + } } - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-23325] Use InternalRow when reading with DataSourceV2.
Repository: spark Updated Branches: refs/heads/master 3d5c61e5f -> 9d27541a8 [SPARK-23325] Use InternalRow when reading with DataSourceV2. ## What changes were proposed in this pull request? This updates the DataSourceV2 API to use InternalRow instead of Row for the default case with no scan mix-ins. Support for readers that produce Row is added through SupportsDeprecatedScanRow, which matches the previous API. Readers that used Row now implement this class and should be migrated to InternalRow. Readers that previously implemented SupportsScanUnsafeRow have been migrated to use no SupportsScan mix-ins and produce InternalRow. ## How was this patch tested? This uses existing tests. Author: Ryan Blue Closes #21118 from rdblue/SPARK-23325-datasource-v2-internal-row. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9d27541a Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9d27541a Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9d27541a Branch: refs/heads/master Commit: 9d27541a856d95635386cbc98f2bb1f1f2f30c13 Parents: 3d5c61e Author: Ryan Blue Authored: Tue Jul 24 10:46:36 2018 -0700 Committer: Xiao Li Committed: Tue Jul 24 10:46:36 2018 -0700 -- .../sql/kafka010/KafkaContinuousReader.scala| 16 --- .../sql/kafka010/KafkaMicroBatchReader.scala| 21 - .../kafka010/KafkaMicroBatchSourceSuite.scala | 2 +- .../sql/sources/v2/reader/DataSourceReader.java | 6 +-- .../sources/v2/reader/InputPartitionReader.java | 7 +-- .../v2/reader/SupportsDeprecatedScanRow.java| 39 + .../v2/reader/SupportsScanColumnarBatch.java| 4 +- .../v2/reader/SupportsScanUnsafeRow.java| 46 .../datasources/v2/DataSourceRDD.scala | 1 - .../datasources/v2/DataSourceV2ScanExec.scala | 17 .../datasources/v2/DataSourceV2Strategy.scala | 13 +++--- .../continuous/ContinuousDataSourceRDD.scala| 26 +-- .../continuous/ContinuousQueuedDataReader.scala | 8 ++-- .../continuous/ContinuousRateStreamSource.scala | 4 +- .../spark/sql/execution/streaming/memory.scala | 16 +++ .../sources/ContinuousMemoryStream.scala| 7 +-- .../sources/RateStreamMicroBatchReader.scala| 4 +- .../execution/streaming/sources/socket.scala| 7 +-- .../sources/v2/JavaAdvancedDataSourceV2.java| 4 +- .../v2/JavaPartitionAwareDataSource.java| 4 +- .../v2/JavaSchemaRequiredDataSource.java| 5 ++- .../sql/sources/v2/JavaSimpleDataSourceV2.java | 5 ++- .../sources/v2/JavaUnsafeRowDataSourceV2.java | 9 ++-- .../sources/RateStreamProviderSuite.scala | 6 +-- .../sql/sources/v2/DataSourceV2Suite.scala | 30 +++-- .../sources/v2/SimpleWritableDataSource.scala | 7 +-- .../sql/streaming/StreamingQuerySuite.scala | 6 +-- .../ContinuousQueuedDataReaderSuite.scala | 6 +-- .../sources/StreamingDataSourceV2Suite.scala| 7 +-- 29 files changed, 168 insertions(+), 165 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/9d27541a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReader.scala -- diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReader.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReader.scala index badaa69..48b91df 100644 --- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReader.scala +++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReader.scala @@ -26,6 +26,7 @@ import org.apache.kafka.common.TopicPartition import org.apache.spark.TaskContext import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.UnsafeRow import org.apache.spark.sql.kafka010.KafkaSourceProvider.{INSTRUCTION_FOR_FAIL_ON_DATA_LOSS_FALSE, INSTRUCTION_FOR_FAIL_ON_DATA_LOSS_TRUE} import org.apache.spark.sql.sources.v2.reader._ @@ -53,7 +54,7 @@ class KafkaContinuousReader( metadataPath: String, initialOffsets: KafkaOffsetRangeLimit, failOnDataLoss: Boolean) - extends ContinuousReader with SupportsScanUnsafeRow with Logging { + extends ContinuousReader with Logging { private lazy val session = SparkSession.getActiveSession.get private lazy val sc = session.sparkContext @@ -86,7 +87,7 @@ class KafkaContinuousReader( KafkaSourceOffset(JsonUtils.partitionOffsets(json)) } - override def planUnsafeInputPartitions(): ju.List[InputPartition[UnsafeRow]] = { + override def planInputPartitions(): ju.List[InputPa
spark git commit: [SPARK-24812][SQL] Last Access Time in the table description is not valid
Repository: spark Updated Branches: refs/heads/master 9d27541a8 -> d4a277f0c [SPARK-24812][SQL] Last Access Time in the table description is not valid ## What changes were proposed in this pull request? Last Access Time will always displayed wrong date Thu Jan 01 05:30:00 IST 1970 when user run DESC FORMATTED table command In hive its displayed as "UNKNOWN" which makes more sense than displaying wrong date. seems to be a limitation as of now even from hive, better we can follow the hive behavior unless the limitation has been resolved from hive. spark client output ![spark_desc table](https://user-images.githubusercontent.com/12999161/42753448-ddeea66a-88a5-11e8-94aa-ef8d017f94c5.png) Hive client output ![hive_behaviour](https://user-images.githubusercontent.com/12999161/42753489-f4fd366e-88a5-11e8-83b0-0f3a53ce83dd.png) ## How was this patch tested? UT has been added which makes sure that the wrong date "Thu Jan 01 05:30:00 IST 1970 " shall not be added as value for the Last Access property Author: s71955 Closes #21775 from sujith71955/master_hive. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d4a277f0 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d4a277f0 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d4a277f0 Branch: refs/heads/master Commit: d4a277f0ce2d6e1832d87cae8faec38c5bc730f4 Parents: 9d27541 Author: s71955 Authored: Tue Jul 24 11:31:27 2018 -0700 Committer: Xiao Li Committed: Tue Jul 24 11:31:27 2018 -0700 -- docs/sql-programming-guide.md | 1 + .../apache/spark/sql/catalyst/catalog/interface.scala | 5 - .../apache/spark/sql/hive/execution/HiveDDLSuite.scala | 13 + 3 files changed, 18 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/d4a277f0/docs/sql-programming-guide.md -- diff --git a/docs/sql-programming-guide.md b/docs/sql-programming-guide.md index 4bab58a..e815e5b 100644 --- a/docs/sql-programming-guide.md +++ b/docs/sql-programming-guide.md @@ -1850,6 +1850,7 @@ working with timestamps in `pandas_udf`s to get the best performance, see ## Upgrading From Spark SQL 2.3 to 2.4 + - Since Spark 2.4, Spark will display table description column Last Access value as UNKNOWN when the value was Jan 01 1970. - Since Spark 2.4, Spark maximizes the usage of a vectorized ORC reader for ORC files by default. To do that, `spark.sql.orc.impl` and `spark.sql.orc.filterPushdown` change their default values to `native` and `true` respectively. - In PySpark, when Arrow optimization is enabled, previously `toPandas` just failed when Arrow optimization is unable to be used whereas `createDataFrame` from Pandas DataFrame allowed the fallback to non-optimization. Now, both `toPandas` and `createDataFrame` from Pandas DataFrame allow the fallback by default, which can be switched off by `spark.sql.execution.arrow.fallback.enabled`. - Since Spark 2.4, writing an empty dataframe to a directory launches at least one write task, even if physically the dataframe has no partition. This introduces a small behavior change that for self-describing file formats like Parquet and Orc, Spark creates a metadata-only file in the target directory when writing a 0-partition dataframe, so that schema inference can still work if users read that directory later. The new behavior is more reasonable and more consistent regarding writing empty dataframe. http://git-wip-us.apache.org/repos/asf/spark/blob/d4a277f0/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index c6105c5..a4ead53 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -114,7 +114,10 @@ case class CatalogTablePartition( map.put("Partition Parameters", s"{${parameters.map(p => p._1 + "=" + p._2).mkString(", ")}}") } map.put("Created Time", new Date(createTime).toString) -map.put("Last Access", new Date(lastAccessTime).toString) +val lastAccess = { + if (-1 == lastAccessTime) "UNKNOWN" else new Date(lastAccessTime).toString +} +map.put("Last Access", lastAccess) stats.foreach(s => map.put("Partition Statistics", s.simpleString)) map } http://git-wip-us.apache.org/repos/asf/spark/blob/d4a277f0/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala ---
spark git commit: [SPARK-24891][SQL] Fix HandleNullInputsForUDF rule
Repository: spark Updated Branches: refs/heads/master 15fff7903 -> c26b09216 [SPARK-24891][SQL] Fix HandleNullInputsForUDF rule ## What changes were proposed in this pull request? The HandleNullInputsForUDF would always add a new `If` node every time it is applied. That would cause a difference between the same plan being analyzed once and being analyzed twice (or more), thus raising issues like plan not matched in the cache manager. The solution is to mark the arguments as null-checked, which is to add a "KnownNotNull" node above those arguments, when adding the UDF under an `If` node, because clearly the UDF will not be called when any of those arguments is null. ## How was this patch tested? Add new tests under sql/UDFSuite and AnalysisSuite. Author: maryannxue Closes #21851 from maryannxue/spark-24891. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c26b0921 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c26b0921 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c26b0921 Branch: refs/heads/master Commit: c26b0921693814f0726507f16b836d82e2e8cfe0 Parents: 15fff79 Author: maryannxue Authored: Tue Jul 24 19:35:34 2018 -0700 Committer: Xiao Li Committed: Tue Jul 24 19:35:34 2018 -0700 -- .../spark/sql/catalyst/analysis/Analyzer.scala | 22 .../expressions/constraintExpressions.scala | 35 .../sql/catalyst/analysis/AnalysisSuite.scala | 16 +++-- .../scala/org/apache/spark/sql/UDFSuite.scala | 31 - 4 files changed, 94 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/c26b0921/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 866396c..4f474f4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.encoders.OuterScopes import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.SubExprUtils._ import org.apache.spark.sql.catalyst.expressions.aggregate._ -import org.apache.spark.sql.catalyst.expressions.objects.{LambdaVariable, MapObjects, NewInstance, UnresolvedMapObjects} +import org.apache.spark.sql.catalyst.expressions.objects._ import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules._ @@ -2145,14 +2145,24 @@ class Analyzer( val parameterTypes = ScalaReflection.getParameterTypes(func) assert(parameterTypes.length == inputs.length) + // TODO: skip null handling for not-nullable primitive inputs after we can completely + // trust the `nullable` information. + // (cls, expr) => cls.isPrimitive && expr.nullable + val needsNullCheck = (cls: Class[_], expr: Expression) => +cls.isPrimitive && !expr.isInstanceOf[KnowNotNull] val inputsNullCheck = parameterTypes.zip(inputs) -// TODO: skip null handling for not-nullable primitive inputs after we can completely -// trust the `nullable` information. -// .filter { case (cls, expr) => cls.isPrimitive && expr.nullable } -.filter { case (cls, _) => cls.isPrimitive } +.filter { case (cls, expr) => needsNullCheck(cls, expr) } .map { case (_, expr) => IsNull(expr) } .reduceLeftOption[Expression]((e1, e2) => Or(e1, e2)) - inputsNullCheck.map(If(_, Literal.create(null, udf.dataType), udf)).getOrElse(udf) + // Once we add an `If` check above the udf, it is safe to mark those checked inputs + // as not nullable (i.e., wrap them with `KnownNotNull`), because the null-returning + // branch of `If` will be called if any of these checked inputs is null. Thus we can + // prevent this rule from being applied repeatedly. + val newInputs = parameterTypes.zip(inputs).map{ case (cls, expr) => +if (needsNullCheck(cls, expr)) KnowNotNull(expr) else expr } + inputsNullCheck +.map(If(_, Literal.create(null, udf.dataType), udf.copy(children = newInputs))) +.getOrElse(udf) } } } http://git-wip-us.apache.org/repos/asf/spark/blob/c26b0921/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/constraintExpressions.scala
spark git commit: [SPARK-24891][SQL] Fix HandleNullInputsForUDF rule
Repository: spark Updated Branches: refs/heads/branch-2.3 740a23d7d -> 6a5999286 [SPARK-24891][SQL] Fix HandleNullInputsForUDF rule The HandleNullInputsForUDF would always add a new `If` node every time it is applied. That would cause a difference between the same plan being analyzed once and being analyzed twice (or more), thus raising issues like plan not matched in the cache manager. The solution is to mark the arguments as null-checked, which is to add a "KnownNotNull" node above those arguments, when adding the UDF under an `If` node, because clearly the UDF will not be called when any of those arguments is null. Add new tests under sql/UDFSuite and AnalysisSuite. Author: maryannxue Closes #21851 from maryannxue/spark-24891. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6a599928 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6a599928 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6a599928 Branch: refs/heads/branch-2.3 Commit: 6a59992866a971abf6052e479ba48c6abded4d04 Parents: 740a23d Author: maryannxue Authored: Tue Jul 24 19:35:34 2018 -0700 Committer: Xiao Li Committed: Tue Jul 24 19:39:23 2018 -0700 -- .../spark/sql/catalyst/analysis/Analyzer.scala | 22 .../expressions/constraintExpressions.scala | 35 .../sql/catalyst/analysis/AnalysisSuite.scala | 16 +++-- .../scala/org/apache/spark/sql/UDFSuite.scala | 31 - 4 files changed, 94 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/6a599928/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 2858bee..5963c14 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.encoders.OuterScopes import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.SubExprUtils._ import org.apache.spark.sql.catalyst.expressions.aggregate._ -import org.apache.spark.sql.catalyst.expressions.objects.{LambdaVariable, MapObjects, NewInstance, UnresolvedMapObjects} +import org.apache.spark.sql.catalyst.expressions.objects._ import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules._ @@ -2046,14 +2046,24 @@ class Analyzer( val parameterTypes = ScalaReflection.getParameterTypes(func) assert(parameterTypes.length == inputs.length) + // TODO: skip null handling for not-nullable primitive inputs after we can completely + // trust the `nullable` information. + // (cls, expr) => cls.isPrimitive && expr.nullable + val needsNullCheck = (cls: Class[_], expr: Expression) => +cls.isPrimitive && !expr.isInstanceOf[KnowNotNull] val inputsNullCheck = parameterTypes.zip(inputs) -// TODO: skip null handling for not-nullable primitive inputs after we can completely -// trust the `nullable` information. -// .filter { case (cls, expr) => cls.isPrimitive && expr.nullable } -.filter { case (cls, _) => cls.isPrimitive } +.filter { case (cls, expr) => needsNullCheck(cls, expr) } .map { case (_, expr) => IsNull(expr) } .reduceLeftOption[Expression]((e1, e2) => Or(e1, e2)) - inputsNullCheck.map(If(_, Literal.create(null, udf.dataType), udf)).getOrElse(udf) + // Once we add an `If` check above the udf, it is safe to mark those checked inputs + // as not nullable (i.e., wrap them with `KnownNotNull`), because the null-returning + // branch of `If` will be called if any of these checked inputs is null. Thus we can + // prevent this rule from being applied repeatedly. + val newInputs = parameterTypes.zip(inputs).map{ case (cls, expr) => +if (needsNullCheck(cls, expr)) KnowNotNull(expr) else expr } + inputsNullCheck +.map(If(_, Literal.create(null, udf.dataType), udf.copy(children = newInputs))) +.getOrElse(udf) } } } http://git-wip-us.apache.org/repos/asf/spark/blob/6a599928/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/constraintExpressions.scala -- diff --git a/sql/catalyst/src/main/scala/org/apa
spark git commit: [SPARK-24890][SQL] Short circuiting the `if` condition when `trueValue` and `falseValue` are the same
Repository: spark Updated Branches: refs/heads/master c26b09216 -> d4c341589 [SPARK-24890][SQL] Short circuiting the `if` condition when `trueValue` and `falseValue` are the same ## What changes were proposed in this pull request? When `trueValue` and `falseValue` are semantic equivalence, the condition expression in `if` can be removed to avoid extra computation in runtime. ## How was this patch tested? Test added. Author: DB Tsai Closes #21848 from dbtsai/short-circuit-if. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d4c34158 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d4c34158 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d4c34158 Branch: refs/heads/master Commit: d4c341589499099654ed4febf235f19897a21601 Parents: c26b092 Author: DB Tsai Authored: Tue Jul 24 20:21:11 2018 -0700 Committer: Xiao Li Committed: Tue Jul 24 20:21:11 2018 -0700 -- .../sql/catalyst/optimizer/expressions.scala| 7 -- .../optimizer/SimplifyConditionalSuite.scala| 24 +++- .../apache/spark/sql/test/SQLTestUtils.scala| 2 +- 3 files changed, 29 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/d4c34158/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala index cf17f59..4696699 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala @@ -390,6 +390,8 @@ object SimplifyConditionals extends Rule[LogicalPlan] with PredicateHelper { case If(TrueLiteral, trueValue, _) => trueValue case If(FalseLiteral, _, falseValue) => falseValue case If(Literal(null, _), _, falseValue) => falseValue + case If(cond, trueValue, falseValue) +if cond.deterministic && trueValue.semanticEquals(falseValue) => trueValue case e @ CaseWhen(branches, elseValue) if branches.exists(x => falseOrNullLiteral(x._1)) => // If there are branches that are always false, remove them. @@ -403,14 +405,14 @@ object SimplifyConditionals extends Rule[LogicalPlan] with PredicateHelper { e.copy(branches = newBranches) } - case e @ CaseWhen(branches, _) if branches.headOption.map(_._1) == Some(TrueLiteral) => + case CaseWhen(branches, _) if branches.headOption.map(_._1).contains(TrueLiteral) => // If the first branch is a true literal, remove the entire CaseWhen and use the value // from that. Note that CaseWhen.branches should never be empty, and as a result the // headOption (rather than head) added above is just an extra (and unnecessary) safeguard. branches.head._2 case CaseWhen(branches, _) if branches.exists(_._1 == TrueLiteral) => -// a branc with a TRue condition eliminates all following branches, +// a branch with a true condition eliminates all following branches, // these branches can be pruned away val (h, t) = branches.span(_._1 != TrueLiteral) CaseWhen( h :+ t.head, None) @@ -651,6 +653,7 @@ object SimplifyCaseConversionExpressions extends Rule[LogicalPlan] { } } + /** * Combine nested [[Concat]] expressions. */ http://git-wip-us.apache.org/repos/asf/spark/blob/d4c34158/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala -- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala index b597c8e..e210874 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala @@ -17,6 +17,8 @@ package org.apache.spark.sql.catalyst.optimizer +import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute +import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.dsl.plans._ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral} @@ -29,7 +31,8 @@ import org.apache.spark.sql.types.{IntegerType, NullType} class SimplifyConditionalSuite extends PlanTest with PredicateHelper { object Optimize extends RuleExe
spark git commit: [SPARK-23957][SQL] Sorts in subqueries are redundant and can be removed
Repository: spark Updated Branches: refs/heads/master d4c341589 -> afb062753 [SPARK-23957][SQL] Sorts in subqueries are redundant and can be removed ## What changes were proposed in this pull request? Thanks to henryr for the original idea at https://github.com/apache/spark/pull/21049 Description from the original PR : Subqueries (at least in SQL) have 'bag of tuples' semantics. Ordering them is therefore redundant (unless combined with a limit). This patch removes the top sort operators from the subquery plans. This closes https://github.com/apache/spark/pull/21049. ## How was this patch tested? Added test cases in SubquerySuite to cover in, exists and scalar subqueries. Please review http://spark.apache.org/contributing.html before opening a pull request. Author: Dilip Biswal Closes #21853 from dilipbiswal/SPARK-23957. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/afb06275 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/afb06275 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/afb06275 Branch: refs/heads/master Commit: afb0627536494c654ce5dd72db648f1ee7da641c Parents: d4c3415 Author: Dilip Biswal Authored: Tue Jul 24 20:46:27 2018 -0700 Committer: Xiao Li Committed: Tue Jul 24 20:46:27 2018 -0700 -- .../sql/catalyst/optimizer/Optimizer.scala | 12 +- .../org/apache/spark/sql/SubquerySuite.scala| 300 ++- 2 files changed, 310 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/afb06275/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index 5ed7412..adb1350 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -180,10 +180,20 @@ abstract class Optimizer(sessionCatalog: SessionCatalog) * Optimize all the subqueries inside expression. */ object OptimizeSubqueries extends Rule[LogicalPlan] { +private def removeTopLevelSort(plan: LogicalPlan): LogicalPlan = { + plan match { +case Sort(_, _, child) => child +case Project(fields, child) => Project(fields, removeTopLevelSort(child)) +case other => other + } +} def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions { case s: SubqueryExpression => val Subquery(newPlan) = Optimizer.this.execute(Subquery(s.plan)) -s.withNewPlan(newPlan) +// At this point we have an optimized subquery plan that we are going to attach +// to this subquery expression. Here we can safely remove any top level sort +// in the plan as tuples produced by a subquery are un-ordered. +s.withNewPlan(removeTopLevelSort(newPlan)) } } http://git-wip-us.apache.org/repos/asf/spark/blob/afb06275/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala -- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala index acef62d..cbffed9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala @@ -17,7 +17,10 @@ package org.apache.spark.sql -import org.apache.spark.sql.catalyst.plans.logical.Join +import scala.collection.mutable.ArrayBuffer + +import org.apache.spark.sql.catalyst.expressions.SubqueryExpression +import org.apache.spark.sql.catalyst.plans.logical.{Join, LogicalPlan, Sort} import org.apache.spark.sql.test.SharedSQLContext class SubquerySuite extends QueryTest with SharedSQLContext { @@ -970,4 +973,299 @@ class SubquerySuite extends QueryTest with SharedSQLContext { Row("3", "b") :: Row("4", "b") :: Nil) } } + + private def getNumSortsInQuery(query: String): Int = { +val plan = sql(query).queryExecution.optimizedPlan +getNumSorts(plan) + getSubqueryExpressions(plan).map{s => getNumSorts(s.plan)}.sum + } + + private def getSubqueryExpressions(plan: LogicalPlan): Seq[SubqueryExpression] = { +val subqueryExpressions = ArrayBuffer.empty[SubqueryExpression] +plan transformAllExpressions { + case s: SubqueryExpression => +subqueryExpressions ++= (getSubqueryExpressions(s.plan) :+ s) +s +} +subqueryExpressions + } + + private def getNumSorts(plan: LogicalPlan): Int = { +plan.collect { case s: Sort => s
spark git commit: [SPARK-18874][SQL][FOLLOW-UP] Improvement type mismatched message
Repository: spark Updated Branches: refs/heads/master 78e0a725e -> 7a5fd4a91 [SPARK-18874][SQL][FOLLOW-UP] Improvement type mismatched message ## What changes were proposed in this pull request? Improvement `IN` predicate type mismatched message: ```sql Mismatched columns: [(, t, 4, ., `, t, 4, a, `, :, d, o, u, b, l, e, ,, , t, 5, ., `, t, 5, a, `, :, d, e, c, i, m, a, l, (, 1, 8, ,, 0, ), ), (, t, 4, ., `, t, 4, c, `, :, s, t, r, i, n, g, ,, , t, 5, ., `, t, 5, c, `, :, b, i, g, i, n, t, )] ``` After this patch: ```sql Mismatched columns: [(t4.`t4a`:double, t5.`t5a`:decimal(18,0)), (t4.`t4c`:string, t5.`t5c`:bigint)] ``` ## How was this patch tested? unit tests Author: Yuming Wang Closes #21863 from wangyum/SPARK-18874. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7a5fd4a9 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7a5fd4a9 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7a5fd4a9 Branch: refs/heads/master Commit: 7a5fd4a91e19ee32b365eaf5678c627ad6c6d4c2 Parents: 78e0a72 Author: Yuming Wang Authored: Tue Jul 24 23:59:13 2018 -0700 Committer: Xiao Li Committed: Tue Jul 24 23:59:13 2018 -0700 -- .../sql/catalyst/expressions/predicates.scala | 2 +- .../negative-cases/subq-input-typecheck.sql | 16 - .../negative-cases/subq-input-typecheck.sql.out | 66 3 files changed, 70 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/7a5fd4a9/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala index 699601e..f4077f78 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala @@ -189,7 +189,7 @@ case class In(value: Expression, list: Seq[Expression]) extends Predicate { } else { val mismatchedColumns = valExprs.zip(childOutputs).flatMap { case (l, r) if l.dataType != r.dataType => -s"(${l.sql}:${l.dataType.catalogString}, ${r.sql}:${r.dataType.catalogString})" +Seq(s"(${l.sql}:${l.dataType.catalogString}, ${r.sql}:${r.dataType.catalogString})") case _ => None } TypeCheckResult.TypeCheckFailure( http://git-wip-us.apache.org/repos/asf/spark/blob/7a5fd4a9/sql/core/src/test/resources/sql-tests/inputs/subquery/negative-cases/subq-input-typecheck.sql -- diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/negative-cases/subq-input-typecheck.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/negative-cases/subq-input-typecheck.sql index b15f4da..95b115a 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/negative-cases/subq-input-typecheck.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/negative-cases/subq-input-typecheck.sql @@ -13,6 +13,14 @@ CREATE TEMPORARY VIEW t3 AS SELECT * FROM VALUES (3, 1, 2) AS t3(t3a, t3b, t3c); +CREATE TEMPORARY VIEW t4 AS SELECT * FROM VALUES + (CAST(1 AS DOUBLE), CAST(2 AS STRING), CAST(3 AS STRING)) +AS t1(t4a, t4b, t4c); + +CREATE TEMPORARY VIEW t5 AS SELECT * FROM VALUES + (CAST(1 AS DECIMAL(18, 0)), CAST(2 AS STRING), CAST(3 AS BIGINT)) +AS t1(t5a, t5b, t5c); + -- TC 01.01 SELECT ( SELECT max(t2b), min(t2b) @@ -44,4 +52,10 @@ WHERE (t1a, t1b) IN (SELECT t2a FROM t2 WHERE t1a = t2a); - +-- TC 01.05 +SELECT * FROM t4 +WHERE +(t4a, t4b, t4c) IN (SELECT t5a, + t5b, + t5c +FROM t5); http://git-wip-us.apache.org/repos/asf/spark/blob/7a5fd4a9/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/subq-input-typecheck.sql.out -- diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/subq-input-typecheck.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/subq-input-typecheck.sql.out index 70aeb93..dcd3005 100644 --- a/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/subq-input-typecheck.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/subq-input-typecheck.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 7 +-- Number of queries: 10 -- !query 0 @@ -33,6 +33,2
spark git commit: [SPARK-24768][FOLLOWUP][SQL] Avro migration followup: change artifactId to spark-avro
Repository: spark Updated Branches: refs/heads/master 7a5fd4a91 -> c44eb561e [SPARK-24768][FOLLOWUP][SQL] Avro migration followup: change artifactId to spark-avro ## What changes were proposed in this pull request? After rethinking on the artifactId, I think it should be `spark-avro` instead of `spark-sql-avro`, which is simpler, and consistent with the previous artifactId. I think we need to change it before Spark 2.4 release. Also a tiny change: use `spark.sessionState.newHadoopConf()` to get the hadoop configuration, thus the related hadoop configurations in SQLConf will come into effect. ## How was this patch tested? Unit test Author: Gengliang Wang Closes #21866 from gengliangwang/avro_followup. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c44eb561 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c44eb561 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c44eb561 Branch: refs/heads/master Commit: c44eb561ec371af0405710d2e9358f9797655145 Parents: 7a5fd4a Author: Gengliang Wang Authored: Wed Jul 25 08:42:45 2018 -0700 Committer: Xiao Li Committed: Wed Jul 25 08:42:45 2018 -0700 -- external/avro/pom.xml | 2 +- .../src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/c44eb561/external/avro/pom.xml -- diff --git a/external/avro/pom.xml b/external/avro/pom.xml index ad7df1f..8f118ba 100644 --- a/external/avro/pom.xml +++ b/external/avro/pom.xml @@ -25,7 +25,7 @@ ../../pom.xml - spark-sql-avro_2.11 + spark-avro_2.11 avro http://git-wip-us.apache.org/repos/asf/spark/blob/c44eb561/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala -- diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala index b043252..c6b3c13 100755 --- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala +++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala @@ -56,7 +56,7 @@ private[avro] class AvroFileFormat extends FileFormat with DataSourceRegister { spark: SparkSession, options: Map[String, String], files: Seq[FileStatus]): Option[StructType] = { -val conf = spark.sparkContext.hadoopConfiguration +val conf = spark.sessionState.newHadoopConf() val parsedOptions = new AvroOptions(options, conf) // Schema evolution is not supported yet. Here we only pick a single random sample file to - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[1/3] spark-website git commit: spark summit eu 2018
Repository: spark-website Updated Branches: refs/heads/asf-site f5d7dfafe -> d86cffd19 http://git-wip-us.apache.org/repos/asf/spark-website/blob/d86cffd1/site/releases/spark-release-1-1-1.html -- diff --git a/site/releases/spark-release-1-1-1.html b/site/releases/spark-release-1-1-1.html index f46e47a..bb5c345 100644 --- a/site/releases/spark-release-1-1-1.html +++ b/site/releases/spark-release-1-1-1.html @@ -162,6 +162,9 @@ Latest News + Spark+AI Summit (October 2-4th, 2018, London) agenda posted + (Jul 24, 2018) + Spark 2.2.2 released (Jul 02, 2018) @@ -171,9 +174,6 @@ Spark 2.3.1 released (Jun 08, 2018) - Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted - (Mar 01, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/d86cffd1/site/releases/spark-release-1-2-0.html -- diff --git a/site/releases/spark-release-1-2-0.html b/site/releases/spark-release-1-2-0.html index 4fe4f0a..e7b1ee2 100644 --- a/site/releases/spark-release-1-2-0.html +++ b/site/releases/spark-release-1-2-0.html @@ -162,6 +162,9 @@ Latest News + Spark+AI Summit (October 2-4th, 2018, London) agenda posted + (Jul 24, 2018) + Spark 2.2.2 released (Jul 02, 2018) @@ -171,9 +174,6 @@ Spark 2.3.1 released (Jun 08, 2018) - Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted - (Mar 01, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/d86cffd1/site/releases/spark-release-1-2-1.html -- diff --git a/site/releases/spark-release-1-2-1.html b/site/releases/spark-release-1-2-1.html index a334c62..068cdee 100644 --- a/site/releases/spark-release-1-2-1.html +++ b/site/releases/spark-release-1-2-1.html @@ -162,6 +162,9 @@ Latest News + Spark+AI Summit (October 2-4th, 2018, London) agenda posted + (Jul 24, 2018) + Spark 2.2.2 released (Jul 02, 2018) @@ -171,9 +174,6 @@ Spark 2.3.1 released (Jun 08, 2018) - Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted - (Mar 01, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/d86cffd1/site/releases/spark-release-1-2-2.html -- diff --git a/site/releases/spark-release-1-2-2.html b/site/releases/spark-release-1-2-2.html index 63876f7..2a5ec4d 100644 --- a/site/releases/spark-release-1-2-2.html +++ b/site/releases/spark-release-1-2-2.html @@ -162,6 +162,9 @@ Latest News + Spark+AI Summit (October 2-4th, 2018, London) agenda posted + (Jul 24, 2018) + Spark 2.2.2 released (Jul 02, 2018) @@ -171,9 +174,6 @@ Spark 2.3.1 released (Jun 08, 2018) - Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted - (Mar 01, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/d86cffd1/site/releases/spark-release-1-3-0.html -- diff --git a/site/releases/spark-release-1-3-0.html b/site/releases/spark-release-1-3-0.html index 567dc1d..480b28f 100644 --- a/site/releases/spark-release-1-3-0.html +++ b/site/releases/spark-release-1-3-0.html @@ -162,6 +162,9 @@ Latest News + Spark+AI Summit (October 2-4th, 2018, London) agenda posted + (Jul 24, 2018) + Spark 2.2.2 released (Jul 02, 2018) @@ -171,9 +174,6 @@ Spark 2.3.1 released (Jun 08, 2018) - Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted - (Mar 01, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/d86cffd1/site/releases/spark-release-1-3-1.html -- diff --git a/site/releases/spark-release-1-3-1.html b/site/releases/spark-release-1-3-1.html index 66f50db..88fb4cc 100644 --- a/site/releases/spark-release-1-3-1.html +++ b/site/releases/spark-release-1-3-1.html @@ -162,6 +162,9 @@ Latest News + Spark+AI Summit (October 2-4th, 2018, London) agenda posted + (Jul 24, 2018) + Spark 2.2.2 released (Jul 02, 2018) @@
[2/3] spark-website git commit: spark summit eu 2018
http://git-wip-us.apache.org/repos/asf/spark-website/blob/d86cffd1/site/news/spark-2-2-1-released.html -- diff --git a/site/news/spark-2-2-1-released.html b/site/news/spark-2-2-1-released.html index df7c2f0..b9d465f 100644 --- a/site/news/spark-2-2-1-released.html +++ b/site/news/spark-2-2-1-released.html @@ -162,6 +162,9 @@ Latest News + Spark+AI Summit (October 2-4th, 2018, London) agenda posted + (Jul 24, 2018) + Spark 2.2.2 released (Jul 02, 2018) @@ -171,9 +174,6 @@ Spark 2.3.1 released (Jun 08, 2018) - Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted - (Mar 01, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/d86cffd1/site/news/spark-2-2-2-released.html -- diff --git a/site/news/spark-2-2-2-released.html b/site/news/spark-2-2-2-released.html index 1c46315..2ab8b62 100644 --- a/site/news/spark-2-2-2-released.html +++ b/site/news/spark-2-2-2-released.html @@ -162,6 +162,9 @@ Latest News + Spark+AI Summit (October 2-4th, 2018, London) agenda posted + (Jul 24, 2018) + Spark 2.2.2 released (Jul 02, 2018) @@ -171,9 +174,6 @@ Spark 2.3.1 released (Jun 08, 2018) - Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted - (Mar 01, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/d86cffd1/site/news/spark-2-3-0-released.html -- diff --git a/site/news/spark-2-3-0-released.html b/site/news/spark-2-3-0-released.html index 1d99ab0..d6fcf03 100644 --- a/site/news/spark-2-3-0-released.html +++ b/site/news/spark-2-3-0-released.html @@ -162,6 +162,9 @@ Latest News + Spark+AI Summit (October 2-4th, 2018, London) agenda posted + (Jul 24, 2018) + Spark 2.2.2 released (Jul 02, 2018) @@ -171,9 +174,6 @@ Spark 2.3.1 released (Jun 08, 2018) - Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted - (Mar 01, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/d86cffd1/site/news/spark-2-3-1-released.html -- diff --git a/site/news/spark-2-3-1-released.html b/site/news/spark-2-3-1-released.html index 06b718a..16d475b 100644 --- a/site/news/spark-2-3-1-released.html +++ b/site/news/spark-2-3-1-released.html @@ -162,6 +162,9 @@ Latest News + Spark+AI Summit (October 2-4th, 2018, London) agenda posted + (Jul 24, 2018) + Spark 2.2.2 released (Jul 02, 2018) @@ -171,9 +174,6 @@ Spark 2.3.1 released (Jun 08, 2018) - Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted - (Mar 01, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/d86cffd1/site/news/spark-2.0.0-preview.html -- diff --git a/site/news/spark-2.0.0-preview.html b/site/news/spark-2.0.0-preview.html index da54cef..342ef2e 100644 --- a/site/news/spark-2.0.0-preview.html +++ b/site/news/spark-2.0.0-preview.html @@ -162,6 +162,9 @@ Latest News + Spark+AI Summit (October 2-4th, 2018, London) agenda posted + (Jul 24, 2018) + Spark 2.2.2 released (Jul 02, 2018) @@ -171,9 +174,6 @@ Spark 2.3.1 released (Jun 08, 2018) - Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted - (Mar 01, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/d86cffd1/site/news/spark-accepted-into-apache-incubator.html -- diff --git a/site/news/spark-accepted-into-apache-incubator.html b/site/news/spark-accepted-into-apache-incubator.html index 696c6d0..1f48ed2 100644 --- a/site/news/spark-accepted-into-apache-incubator.html +++ b/site/news/spark-accepted-into-apache-incubator.html @@ -162,6 +162,9 @@ Latest News + Spark+AI Summit (October 2-4th, 2018, London) agenda posted + (Jul 24, 2018) + Spark 2.2.2 released (Jul 02, 2018) @@ -171,9 +174,6 @@ Spark 2.3.1 released (Jun 08, 2018) - Spark+AI
[3/3] spark-website git commit: spark summit eu 2018
spark summit eu 2018 Project: http://git-wip-us.apache.org/repos/asf/spark-website/repo Commit: http://git-wip-us.apache.org/repos/asf/spark-website/commit/d86cffd1 Tree: http://git-wip-us.apache.org/repos/asf/spark-website/tree/d86cffd1 Diff: http://git-wip-us.apache.org/repos/asf/spark-website/diff/d86cffd1 Branch: refs/heads/asf-site Commit: d86cffd1918f9cb8f25d7beb0ae65f814f5232bd Parents: f5d7dfa Author: Xiao Li Authored: Wed Jul 25 09:06:55 2018 -0700 Committer: Xiao Li Committed: Wed Jul 25 09:06:55 2018 -0700 -- ...07-24-spark-summit-oct-2018-agenda-posted.md | 15 + site/committers.html| 6 +- site/community.html | 6 +- site/contributing.html | 6 +- site/developer-tools.html | 6 +- site/documentation.html | 6 +- site/downloads.html | 6 +- site/examples.html | 6 +- site/faq.html | 6 +- site/graphx/index.html | 6 +- site/history.html | 6 +- site/improvement-proposals.html | 6 +- site/index.html | 6 +- site/mailing-lists.html | 8 +- site/mllib/index.html | 6 +- site/news/amp-camp-2013-registration-ope.html | 6 +- .../news/announcing-the-first-spark-summit.html | 6 +- .../news/fourth-spark-screencast-published.html | 6 +- site/news/index.html| 15 +- site/news/nsdi-paper.html | 6 +- site/news/one-month-to-spark-summit-2015.html | 6 +- .../proposals-open-for-spark-summit-east.html | 6 +- ...registration-open-for-spark-summit-east.html | 6 +- .../news/run-spark-and-shark-on-amazon-emr.html | 6 +- site/news/spark-0-6-1-and-0-5-2-released.html | 6 +- site/news/spark-0-6-2-released.html | 6 +- site/news/spark-0-7-0-released.html | 6 +- site/news/spark-0-7-2-released.html | 6 +- site/news/spark-0-7-3-released.html | 6 +- site/news/spark-0-8-0-released.html | 6 +- site/news/spark-0-8-1-released.html | 6 +- site/news/spark-0-9-0-released.html | 6 +- site/news/spark-0-9-1-released.html | 6 +- site/news/spark-0-9-2-released.html | 6 +- site/news/spark-1-0-0-released.html | 6 +- site/news/spark-1-0-1-released.html | 6 +- site/news/spark-1-0-2-released.html | 6 +- site/news/spark-1-1-0-released.html | 6 +- site/news/spark-1-1-1-released.html | 6 +- site/news/spark-1-2-0-released.html | 6 +- site/news/spark-1-2-1-released.html | 6 +- site/news/spark-1-2-2-released.html | 6 +- site/news/spark-1-3-0-released.html | 6 +- site/news/spark-1-4-0-released.html | 6 +- site/news/spark-1-4-1-released.html | 6 +- site/news/spark-1-5-0-released.html | 6 +- site/news/spark-1-5-1-released.html | 6 +- site/news/spark-1-5-2-released.html | 6 +- site/news/spark-1-6-0-released.html | 6 +- site/news/spark-1-6-1-released.html | 6 +- site/news/spark-1-6-2-released.html | 6 +- site/news/spark-1-6-3-released.html | 6 +- site/news/spark-2-0-0-released.html | 6 +- site/news/spark-2-0-1-released.html | 6 +- site/news/spark-2-0-2-released.html | 6 +- site/news/spark-2-1-0-released.html | 6 +- site/news/spark-2-1-1-released.html | 6 +- site/news/spark-2-1-2-released.html | 6 +- site/news/spark-2-1-3-released.html | 6 +- site/news/spark-2-2-0-released.html | 6 +- site/news/spark-2-2-1-released.html | 6 +- site/news/spark-2-2-2-released.html | 6 +- site/news/spark-2-3-0-released.html | 6 +- site/news/spark-2-3-1-released.html | 6 +- site/news/spark-2.0.0-preview.html | 6 +- .../spark-accepted-into-apache-incubator.html | 6 +- site/news/spark-and-shark-in-the-news.html | 6 +- site/news/spark-becomes-tlp.html| 6 +- site/news/spark-featured-in-wired.html | 6 +- .../spark-mailing-lists-moving-to-apache.html | 6 +- site/news/spark-meetups.html| 6 +- site/news/spark-screencasts-published.html | 6 +- site/news/spark-summit-2013-is-a-wrap.html | 6 +- site/news/spark-summit-2014-videos-posted.html | 6 +- site/news/spark-summit-2015-videos-posted.html | 6 +- site/news/spark-summit-agenda-posted.html |
spark git commit: [SPARK-24849][SPARK-24911][SQL] Converting a value of StructType to a DDL string
Repository: spark Updated Branches: refs/heads/master 571a6f057 -> 2f77616e1 [SPARK-24849][SPARK-24911][SQL] Converting a value of StructType to a DDL string ## What changes were proposed in this pull request? In the PR, I propose to extend the `StructType`/`StructField` classes by new method `toDDL` which converts a value of the `StructType`/`StructField` type to a string formatted in DDL style. The resulted string can be used in a table creation. The `toDDL` method of `StructField` is reused in `SHOW CREATE TABLE`. In this way the PR fixes the bug of unquoted names of nested fields. ## How was this patch tested? I add a test for checking the new method and 2 round trip tests: `fromDDL` -> `toDDL` and `toDDL` -> `fromDDL` Author: Maxim Gekk Closes #21803 from MaxGekk/to-ddl. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2f77616e Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2f77616e Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2f77616e Branch: refs/heads/master Commit: 2f77616e1dc593fb9c376a8bd72416198cf3d6f5 Parents: 571a6f0 Author: Maxim Gekk Authored: Wed Jul 25 11:09:12 2018 -0700 Committer: Xiao Li Committed: Wed Jul 25 11:09:12 2018 -0700 -- .../spark/sql/catalyst/util/package.scala | 12 +++ .../apache/spark/sql/types/StructField.scala| 13 .../org/apache/spark/sql/types/StructType.scala | 10 +- .../spark/sql/types/StructTypeSuite.scala | 33 .../spark/sql/execution/command/tables.scala| 23 +++--- .../spark/sql/hive/ShowCreateTableSuite.scala | 15 + 6 files changed, 86 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/2f77616e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala index 4005087..0978e92 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala @@ -155,6 +155,18 @@ package object util { def toPrettySQL(e: Expression): String = usePrettyExpression(e).sql + + def escapeSingleQuotedString(str: String): String = { +val builder = StringBuilder.newBuilder + +str.foreach { + case '\'' => builder ++= s"\\\'" + case ch => builder += ch +} + +builder.toString() + } + /* FIX ME implicit class debugLogging(a: Any) { def debugLogging() { http://git-wip-us.apache.org/repos/asf/spark/blob/2f77616e/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructField.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructField.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructField.scala index 2c18fdc..902cae9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructField.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructField.scala @@ -21,6 +21,7 @@ import org.json4s.JsonAST.JValue import org.json4s.JsonDSL._ import org.apache.spark.annotation.InterfaceStability +import org.apache.spark.sql.catalyst.util.{escapeSingleQuotedString, quoteIdentifier} /** * A field inside a StructType. @@ -74,4 +75,16 @@ case class StructField( def getComment(): Option[String] = { if (metadata.contains("comment")) Option(metadata.getString("comment")) else None } + + /** + * Returns a string containing a schema in DDL format. For example, the following value: + * `StructField("eventId", IntegerType)` will be converted to `eventId` INT. + */ + def toDDL: String = { +val comment = getComment() + .map(escapeSingleQuotedString) + .map(" COMMENT '" + _ + "'") + +s"${quoteIdentifier(name)} ${dataType.sql}${comment.getOrElse("")}" + } } http://git-wip-us.apache.org/repos/asf/spark/blob/2f77616e/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala index b13e95f..c5ca169 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala @@ -27,7 +27,7 @@ import org.apache.spark.SparkException import org.apache.spark.annotation.InterfaceStability import org.apache.spark.sql.catalyst.expressions.{Attrib
spark git commit: [SPARK-24860][SQL] Support setting of partitionOverWriteMode in output options for writing DataFrame
Repository: spark Updated Branches: refs/heads/master 0c83f718e -> 17f469bc8 [SPARK-24860][SQL] Support setting of partitionOverWriteMode in output options for writing DataFrame ## What changes were proposed in this pull request? Besides spark setting spark.sql.sources.partitionOverwriteMode also allow setting partitionOverWriteMode per write ## How was this patch tested? Added unit test in InsertSuite Please review http://spark.apache.org/contributing.html before opening a pull request. Author: Koert Kuipers Closes #21818 from koertkuipers/feat-partition-overwrite-mode-per-write. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/17f469bc Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/17f469bc Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/17f469bc Branch: refs/heads/master Commit: 17f469bc808e076b45fffcedb0147991fa4c41f3 Parents: 0c83f71 Author: Koert Kuipers Authored: Wed Jul 25 13:06:03 2018 -0700 Committer: Xiao Li Committed: Wed Jul 25 13:06:03 2018 -0700 -- .../org/apache/spark/sql/internal/SQLConf.scala | 6 +- .../InsertIntoHadoopFsRelationCommand.scala | 9 +++-- .../apache/spark/sql/sources/InsertSuite.scala | 20 3 files changed, 32 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/17f469bc/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index d7c830d..53423e0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -1360,7 +1360,11 @@ object SQLConf { "overwriting. In dynamic mode, Spark doesn't delete partitions ahead, and only overwrite " + "those partitions that have data written into it at runtime. By default we use static " + "mode to keep the same behavior of Spark prior to 2.3. Note that this config doesn't " + -"affect Hive serde tables, as they are always overwritten with dynamic mode.") +"affect Hive serde tables, as they are always overwritten with dynamic mode. This can " + +"also be set as an output option for a data source using key partitionOverwriteMode " + +"(which takes precendence over this setting), e.g. " + +"dataframe.write.option(\"partitionOverwriteMode\", \"dynamic\").save(path)." + ) .stringConf .transform(_.toUpperCase(Locale.ROOT)) .checkValues(PartitionOverwriteMode.values.map(_.toString)) http://git-wip-us.apache.org/repos/asf/spark/blob/17f469bc/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala index dd7ef0d..8a2e00d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable, CatalogT import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.command._ import org.apache.spark.sql.internal.SQLConf.PartitionOverwriteMode @@ -91,8 +92,12 @@ case class InsertIntoHadoopFsRelationCommand( val pathExists = fs.exists(qualifiedOutputPath) -val enableDynamicOverwrite = - sparkSession.sessionState.conf.partitionOverwriteMode == PartitionOverwriteMode.DYNAMIC +val parameters = CaseInsensitiveMap(options) + +val partitionOverwriteMode = parameters.get("partitionOverwriteMode") + .map(mode => PartitionOverwriteMode.withName(mode.toUpperCase)) + .getOrElse(sparkSession.sessionState.conf.partitionOverwriteMode) +val enableDynamicOverwrite = partitionOverwriteMode == PartitionOverwriteMode.DYNAMIC // This config only makes sense when we are overwriting a partitioned dataset with dynamic // partition columns. val dynamicPartitionOverwrite = enableDynamicOv
spark git commit: [SPARK-24867][SQL] Add AnalysisBarrier to DataFrameWriter
Repository: spark Updated Branches: refs/heads/master 17f469bc8 -> d2e7deb59 [SPARK-24867][SQL] Add AnalysisBarrier to DataFrameWriter ## What changes were proposed in this pull request? ```Scala val udf1 = udf({(x: Int, y: Int) => x + y}) val df = spark.range(0, 3).toDF("a") .withColumn("b", udf1($"a", udf1($"a", lit(10 df.cache() df.write.saveAsTable("t") ``` Cache is not being used because the plans do not match with the cached plan. This is a regression caused by the changes we made in AnalysisBarrier, since not all the Analyzer rules are idempotent. ## How was this patch tested? Added a test. Also found a bug in the DSV1 write path. This is not a regression. Thus, opened a separate JIRA https://issues.apache.org/jira/browse/SPARK-24869 Author: Xiao Li Closes #21821 from gatorsmile/testMaster22. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d2e7deb5 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d2e7deb5 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d2e7deb5 Branch: refs/heads/master Commit: d2e7deb59f641e93778b763d5396f73d38f9a785 Parents: 17f469b Author: Xiao Li Authored: Wed Jul 25 17:22:37 2018 -0700 Committer: Xiao Li Committed: Wed Jul 25 17:22:37 2018 -0700 -- .../org/apache/spark/sql/DataFrameWriter.scala | 10 +++-- .../spark/sql/execution/command/ddl.scala | 7 ++-- .../scala/org/apache/spark/sql/UDFSuite.scala | 42 +++- 3 files changed, 51 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/d2e7deb5/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala index b9fa43f..39c0e10 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala @@ -254,7 +254,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) { val writer = ws.createWriter(jobId, df.logicalPlan.schema, mode, options) if (writer.isPresent) { runCommand(df.sparkSession, "save") { - WriteToDataSourceV2(writer.get(), df.logicalPlan) + WriteToDataSourceV2(writer.get(), df.planWithBarrier) } } @@ -275,7 +275,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) { sparkSession = df.sparkSession, className = source, partitionColumns = partitioningColumns.getOrElse(Nil), -options = extraOptions.toMap).planForWriting(mode, AnalysisBarrier(df.logicalPlan)) +options = extraOptions.toMap).planForWriting(mode, df.planWithBarrier) } } @@ -323,7 +323,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) { InsertIntoTable( table = UnresolvedRelation(tableIdent), partition = Map.empty[String, Option[String]], -query = df.logicalPlan, +query = df.planWithBarrier, overwrite = mode == SaveMode.Overwrite, ifPartitionNotExists = false) } @@ -459,7 +459,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) { partitionColumnNames = partitioningColumns.getOrElse(Nil), bucketSpec = getBucketSpec) -runCommand(df.sparkSession, "saveAsTable")(CreateTable(tableDesc, mode, Some(df.logicalPlan))) +runCommand(df.sparkSession, "saveAsTable") { + CreateTable(tableDesc, mode, Some(df.planWithBarrier)) +} } /** http://git-wip-us.apache.org/repos/asf/spark/blob/d2e7deb5/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala index 04bf8c6..c7f7e4d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala @@ -29,7 +29,7 @@ import org.apache.hadoop.mapred.{FileInputFormat, JobConf} import org.apache.spark.sql.{AnalysisException, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.{NoSuchTableException, Resolver} +import org.apache.spark.sql.catalyst.analysis.{EliminateBarriers, NoSuchTableException, Resolver} import org.apache.spark.sql.catalyst.catalog._ import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.expressions.{Attribute, Attribu
spark git commit: [SPARK-24867][SQL] Add AnalysisBarrier to DataFrameWriter
Repository: spark Updated Branches: refs/heads/branch-2.3 740606eb8 -> fa552c3c1 [SPARK-24867][SQL] Add AnalysisBarrier to DataFrameWriter ```Scala val udf1 = udf({(x: Int, y: Int) => x + y}) val df = spark.range(0, 3).toDF("a") .withColumn("b", udf1($"a", udf1($"a", lit(10 df.cache() df.write.saveAsTable("t") ``` Cache is not being used because the plans do not match with the cached plan. This is a regression caused by the changes we made in AnalysisBarrier, since not all the Analyzer rules are idempotent. Added a test. Also found a bug in the DSV1 write path. This is not a regression. Thus, opened a separate JIRA https://issues.apache.org/jira/browse/SPARK-24869 Author: Xiao Li Closes #21821 from gatorsmile/testMaster22. (cherry picked from commit d2e7deb59f641e93778b763d5396f73d38f9a785) Signed-off-by: Xiao Li Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fa552c3c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fa552c3c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fa552c3c Branch: refs/heads/branch-2.3 Commit: fa552c3c1102404fe98c72a5b83cffbc5ba41df3 Parents: 740606e Author: Xiao Li Authored: Wed Jul 25 17:22:37 2018 -0700 Committer: Xiao Li Committed: Wed Jul 25 17:24:32 2018 -0700 -- .../org/apache/spark/sql/DataFrameWriter.scala | 10 +++-- .../spark/sql/execution/command/ddl.scala | 7 ++-- .../scala/org/apache/spark/sql/UDFSuite.scala | 42 +++- 3 files changed, 51 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/fa552c3c/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala index ed7a910..6c9fb52 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala @@ -254,7 +254,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) { val writer = ws.createWriter(jobId, df.logicalPlan.schema, mode, options) if (writer.isPresent) { runCommand(df.sparkSession, "save") { - WriteToDataSourceV2(writer.get(), df.logicalPlan) + WriteToDataSourceV2(writer.get(), df.planWithBarrier) } } @@ -275,7 +275,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) { sparkSession = df.sparkSession, className = source, partitionColumns = partitioningColumns.getOrElse(Nil), -options = extraOptions.toMap).planForWriting(mode, AnalysisBarrier(df.logicalPlan)) +options = extraOptions.toMap).planForWriting(mode, df.planWithBarrier) } } @@ -323,7 +323,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) { InsertIntoTable( table = UnresolvedRelation(tableIdent), partition = Map.empty[String, Option[String]], -query = df.logicalPlan, +query = df.planWithBarrier, overwrite = mode == SaveMode.Overwrite, ifPartitionNotExists = false) } @@ -455,7 +455,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) { partitionColumnNames = partitioningColumns.getOrElse(Nil), bucketSpec = getBucketSpec) -runCommand(df.sparkSession, "saveAsTable")(CreateTable(tableDesc, mode, Some(df.logicalPlan))) +runCommand(df.sparkSession, "saveAsTable") { + CreateTable(tableDesc, mode, Some(df.planWithBarrier)) +} } /** http://git-wip-us.apache.org/repos/asf/spark/blob/fa552c3c/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala index 0f4831b..28313f2 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala @@ -29,7 +29,7 @@ import org.apache.hadoop.mapred.{FileInputFormat, JobConf} import org.apache.spark.sql.{AnalysisException, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.{NoSuchTableException, Resolver} +import org.apache.spark.sql.catalyst.analysis.{EliminateBarriers, NoSuchTableException, Resolver} import org.apache.spark.sql.catalyst.catalog._ import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.expressions.
spark git commit: [SPARK-24802][SQL][FOLLOW-UP] Add a new config for Optimization Rule Exclusion
Repository: spark Updated Branches: refs/heads/master 58353d7f4 -> 5ed7660d1 [SPARK-24802][SQL][FOLLOW-UP] Add a new config for Optimization Rule Exclusion ## What changes were proposed in this pull request? This is an extension to the original PR, in which rule exclusion did not work for classes derived from Optimizer, e.g., SparkOptimizer. To solve this issue, Optimizer and its derived classes will define/override `defaultBatches` and `nonExcludableRules` in order to define its default rule set as well as rules that cannot be excluded by the SQL config. In the meantime, Optimizer's `batches` method is dedicated to the rule exclusion logic and is defined "final". ## How was this patch tested? Added UT. Author: maryannxue Closes #21876 from maryannxue/rule-exclusion. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5ed7660d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5ed7660d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5ed7660d Branch: refs/heads/master Commit: 5ed7660d14022eb65396e28496c06e47c1dbab1d Parents: 58353d7 Author: maryannxue Authored: Thu Jul 26 11:06:23 2018 -0700 Committer: Xiao Li Committed: Thu Jul 26 11:06:23 2018 -0700 -- .../sql/catalyst/optimizer/Optimizer.scala | 24 - .../optimizer/OptimizerExtendableSuite.scala| 2 +- .../optimizer/OptimizerRuleExclusionSuite.scala | 53 +++- ...timizerStructuralIntegrityCheckerSuite.scala | 2 +- .../spark/sql/execution/SparkOptimizer.scala| 5 +- 5 files changed, 69 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/5ed7660d/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index adb1350..3c264eb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -46,6 +46,13 @@ abstract class Optimizer(sessionCatalog: SessionCatalog) protected def fixedPoint = FixedPoint(SQLConf.get.optimizerMaxIterations) + /** + * Defines the default rule batches in the Optimizer. + * + * Implementations of this class should override this method, and [[nonExcludableRules]] if + * necessary, instead of [[batches]]. The rule batches that eventually run in the Optimizer, + * i.e., returned by [[batches]], will be (defaultBatches - (excludedRules - nonExcludableRules)). + */ def defaultBatches: Seq[Batch] = { val operatorOptimizationRuleSet = Seq( @@ -160,6 +167,14 @@ abstract class Optimizer(sessionCatalog: SessionCatalog) UpdateNullabilityInAttributeReferences) } + /** + * Defines rules that cannot be excluded from the Optimizer even if they are specified in + * SQL config "excludedRules". + * + * Implementations of this class can override this method if necessary. The rule batches + * that eventually run in the Optimizer, i.e., returned by [[batches]], will be + * (defaultBatches - (excludedRules - nonExcludableRules)). + */ def nonExcludableRules: Seq[String] = EliminateDistinct.ruleName :: EliminateSubqueryAliases.ruleName :: @@ -202,7 +217,14 @@ abstract class Optimizer(sessionCatalog: SessionCatalog) */ def extendedOperatorOptimizationRules: Seq[Rule[LogicalPlan]] = Nil - override def batches: Seq[Batch] = { + /** + * Returns (defaultBatches - (excludedRules - nonExcludableRules)), the rule batches that + * eventually run in the Optimizer. + * + * Implementations of this class should override [[defaultBatches]], and [[nonExcludableRules]] + * if necessary, instead of this method. + */ + final override def batches: Seq[Batch] = { val excludedRulesConf = SQLConf.get.optimizerExcludedRules.toSeq.flatMap(Utils.stringToSeq) val excludedRules = excludedRulesConf.filter { ruleName => http://git-wip-us.apache.org/repos/asf/spark/blob/5ed7660d/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerExtendableSuite.scala -- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerExtendableSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerExtendableSuite.scala index 7112c03..36b083a 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerExtendableSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/cata
spark git commit: [SPARK-24795][CORE] Implement barrier execution mode
Repository: spark Updated Branches: refs/heads/master 5ed7660d1 -> e3486e1b9 [SPARK-24795][CORE] Implement barrier execution mode ## What changes were proposed in this pull request? Propose new APIs and modify job/task scheduling to support barrier execution mode, which requires all tasks in a same barrier stage start at the same time, and retry all tasks in case some tasks fail in the middle. The barrier execution mode is useful for some ML/DL workloads. The proposed API changes include: - `RDDBarrier` that marks an RDD as barrier (Spark must launch all the tasks together for the current stage). - `BarrierTaskContext` that support global sync of all tasks in a barrier stage, and provide extra `BarrierTaskInfo`s. In DAGScheduler, we retry all tasks of a barrier stage in case some tasks fail in the middle, this is achieved by unregistering map outputs for a shuffleId (for ShuffleMapStage) or clear the finished partitions in an active job (for ResultStage). ## How was this patch tested? Add `RDDBarrierSuite` to ensure we convert RDDs correctly; Add new test cases in `DAGSchedulerSuite` to ensure we do task scheduling correctly; Add new test cases in `SparkContextSuite` to ensure the barrier execution mode actually works (both under local mode and local cluster mode). Add new test cases in `TaskSchedulerImplSuite` to ensure we schedule tasks for barrier taskSet together. Author: Xingbo Jiang Closes #21758 from jiangxb1987/barrier-execution-mode. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e3486e1b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e3486e1b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e3486e1b Branch: refs/heads/master Commit: e3486e1b9556e00bc9c392a5b8440ab366780f9b Parents: 5ed7660 Author: Xingbo Jiang Authored: Thu Jul 26 12:09:01 2018 -0700 Committer: Xiao Li Committed: Thu Jul 26 12:09:01 2018 -0700 -- .../org/apache/spark/BarrierTaskContext.scala | 42 ++ .../apache/spark/BarrierTaskContextImpl.scala | 49 +++ .../org/apache/spark/BarrierTaskInfo.scala | 31 + .../org/apache/spark/MapOutputTracker.scala | 12 ++ .../org/apache/spark/rdd/MapPartitionsRDD.scala | 15 ++- .../main/scala/org/apache/spark/rdd/RDD.scala | 27 +++- .../scala/org/apache/spark/rdd/RDDBarrier.scala | 52 .../org/apache/spark/rdd/ShuffledRDD.scala | 2 + .../org/apache/spark/scheduler/ActiveJob.scala | 6 + .../apache/spark/scheduler/DAGScheduler.scala | 131 --- .../org/apache/spark/scheduler/ResultTask.scala | 9 +- .../apache/spark/scheduler/ShuffleMapTask.scala | 7 +- .../org/apache/spark/scheduler/Stage.scala | 8 +- .../scala/org/apache/spark/scheduler/Task.scala | 41 -- .../spark/scheduler/TaskDescription.scala | 7 +- .../spark/scheduler/TaskSchedulerImpl.scala | 66 -- .../apache/spark/scheduler/TaskSetManager.scala | 9 +- .../apache/spark/scheduler/WorkerOffer.scala| 8 +- .../cluster/CoarseGrainedSchedulerBackend.scala | 6 +- .../scheduler/local/LocalSchedulerBackend.scala | 3 +- .../org/apache/spark/SparkContextSuite.scala| 42 ++ .../apache/spark/executor/ExecutorSuite.scala | 1 + .../org/apache/spark/rdd/RDDBarrierSuite.scala | 43 ++ .../spark/scheduler/DAGSchedulerSuite.scala | 58 .../org/apache/spark/scheduler/FakeTask.scala | 24 +++- .../spark/scheduler/TaskDescriptionSuite.scala | 2 + .../scheduler/TaskSchedulerImplSuite.scala | 34 + .../MesosFineGrainedSchedulerBackendSuite.scala | 2 + 28 files changed, 673 insertions(+), 64 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/e3486e1b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala -- diff --git a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala new file mode 100644 index 000..4c35862 --- /dev/null +++ b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF
spark git commit: [SPARK-24307][CORE] Add conf to revert to old code.
Repository: spark Updated Branches: refs/heads/master e3486e1b9 -> 2c8274568 [SPARK-24307][CORE] Add conf to revert to old code. In case there are any issues in converting FileSegmentManagedBuffer to ChunkedByteBuffer, add a conf to go back to old code path. Followup to 7e847646d1f377f46dc3154dea37148d4e557a03 Author: Imran Rashid Closes #21867 from squito/SPARK-24307-p2. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2c827456 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2c827456 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2c827456 Branch: refs/heads/master Commit: 2c82745686f4456c4d5c84040a431dcb5b6cb60b Parents: e3486e1 Author: Imran Rashid Authored: Thu Jul 26 12:13:27 2018 -0700 Committer: Xiao Li Committed: Thu Jul 26 12:13:27 2018 -0700 -- .../scala/org/apache/spark/storage/BlockManager.scala| 11 ++- 1 file changed, 10 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/2c827456/core/src/main/scala/org/apache/spark/storage/BlockManager.scala -- diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala index 1db0327..5cd21e3 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala @@ -132,6 +132,8 @@ private[spark] class BlockManager( conf.getBoolean("spark.shuffle.service.enabled", false) private val chunkSize = conf.getSizeAsBytes("spark.storage.memoryMapLimitForTests", Int.MaxValue.toString).toInt + private val remoteReadNioBufferConversion = +conf.getBoolean("spark.network.remoteReadNioBufferConversion", false) val diskBlockManager = { // Only perform cleanup if an external service is not serving our shuffle files. @@ -731,7 +733,14 @@ private[spark] class BlockManager( } if (data != null) { -return Some(ChunkedByteBuffer.fromManagedBuffer(data, chunkSize)) +// SPARK-24307 undocumented "escape-hatch" in case there are any issues in converting to +// ChunkedByteBuffer, to go back to old code-path. Can be removed post Spark 2.4 if +// new path is stable. +if (remoteReadNioBufferConversion) { + return Some(new ChunkedByteBuffer(data.nioByteBuffer())) +} else { + return Some(ChunkedByteBuffer.fromManagedBuffer(data, chunkSize)) +} } logDebug(s"The value of block $blockId is null") } - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24919][BUILD] New linter rule for sparkContext.hadoopConfiguration
Repository: spark Updated Branches: refs/heads/master 2c8274568 -> fa09d9192 [SPARK-24919][BUILD] New linter rule for sparkContext.hadoopConfiguration ## What changes were proposed in this pull request? In most cases, we should use `spark.sessionState.newHadoopConf()` instead of `sparkContext.hadoopConfiguration`, so that the hadoop configurations specified in Spark session configuration will come into effect. Add a rule matching `spark.sparkContext.hadoopConfiguration` or `spark.sqlContext.sparkContext.hadoopConfiguration` to prevent the usage. ## How was this patch tested? Unit test Author: Gengliang Wang Closes #21873 from gengliangwang/linterRule. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fa09d919 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fa09d919 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fa09d919 Branch: refs/heads/master Commit: fa09d91925c07a58dea285d6cf85a751664f89ff Parents: 2c82745 Author: Gengliang Wang Authored: Thu Jul 26 16:50:59 2018 -0700 Committer: Xiao Li Committed: Thu Jul 26 16:50:59 2018 -0700 -- .../org/apache/spark/sql/avro/AvroSuite.scala | 26 +--- .../org/apache/spark/ml/image/HadoopUtils.scala | 4 +++ .../apache/spark/ml/clustering/LDASuite.scala | 2 +- scalastyle-config.xml | 13 ++ .../HadoopFileLinesReaderSuite.scala| 22 - .../spark/sql/hive/execution/HiveDDLSuite.scala | 2 +- .../sql/hive/execution/HiveQuerySuite.scala | 11 ++--- .../sql/hive/execution/SQLQuerySuite.scala | 2 +- 8 files changed, 45 insertions(+), 37 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/fa09d919/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala -- diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala index 865a145..a93309e 100644 --- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala +++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala @@ -638,12 +638,8 @@ class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils { intercept[FileNotFoundException] { withTempPath { dir => FileUtils.touch(new File(dir, "test")) -val hadoopConf = spark.sqlContext.sparkContext.hadoopConfiguration -try { - hadoopConf.set(AvroFileFormat.IgnoreFilesWithoutExtensionProperty, "true") +withSQLConf(AvroFileFormat.IgnoreFilesWithoutExtensionProperty -> "true") { spark.read.format("avro").load(dir.toString) -} finally { - hadoopConf.unset(AvroFileFormat.IgnoreFilesWithoutExtensionProperty) } } } @@ -717,15 +713,10 @@ class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils { Files.createFile(new File(tempSaveDir, "non-avro").toPath) - val hadoopConf = spark.sqlContext.sparkContext.hadoopConfiguration - val count = try { -hadoopConf.set(AvroFileFormat.IgnoreFilesWithoutExtensionProperty, "true") + withSQLConf(AvroFileFormat.IgnoreFilesWithoutExtensionProperty -> "true") { val newDf = spark.read.format("avro").load(tempSaveDir) -newDf.count() - } finally { -hadoopConf.unset(AvroFileFormat.IgnoreFilesWithoutExtensionProperty) +assert(newDf.count() == 8) } - assert(count == 8) } } @@ -888,20 +879,15 @@ class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils { Paths.get(new URL(episodesAvro).toURI), Paths.get(dir.getCanonicalPath, "episodes")) - val hadoopConf = spark.sqlContext.sparkContext.hadoopConfiguration - val count = try { -hadoopConf.set(AvroFileFormat.IgnoreFilesWithoutExtensionProperty, "true") + val hadoopConf = spark.sessionState.newHadoopConf() + withSQLConf(AvroFileFormat.IgnoreFilesWithoutExtensionProperty -> "true") { val newDf = spark .read .option("ignoreExtension", "true") .format("avro") .load(s"${dir.getCanonicalPath}/episodes") -newDf.count() - } finally { -hadoopConf.unset(AvroFileFormat.IgnoreFilesWithoutExtensionProperty) +assert(newDf.count() == 8) } - - assert(count == 8) } } } http://git-wip-us.apache.org/repos/asf/spark/blob/fa09d919/mllib/src/main/scala/org/apache/spark/ml/image/HadoopUtils.scala -- diff --git a/mllib/src/main/scala/org/apache/spark/ml/image/HadoopUtils.scala b/mllib/src/main/scala/org/apache/spark/ml/i
spark git commit: [SPARK-24288][SQL] Add a JDBC Option to enable preventing predicate pushdown
Repository: spark Updated Branches: refs/heads/master e6e9031d7 -> 21fcac164 [SPARK-24288][SQL] Add a JDBC Option to enable preventing predicate pushdown ## What changes were proposed in this pull request? Add a JDBC Option "pushDownPredicate" (default `true`) to allow/disallow predicate push-down in JDBC data source. ## How was this patch tested? Add a test in `JDBCSuite` Author: maryannxue Closes #21875 from maryannxue/spark-24288. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/21fcac16 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/21fcac16 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/21fcac16 Branch: refs/heads/master Commit: 21fcac1645bf01c453ddd4cb64c566895e66ea4f Parents: e6e9031 Author: maryannxue Authored: Thu Jul 26 23:47:32 2018 -0700 Committer: Xiao Li Committed: Thu Jul 26 23:47:32 2018 -0700 -- docs/sql-programming-guide.md | 7 +++ .../datasources/jdbc/JDBCOptions.scala | 4 ++ .../datasources/jdbc/JDBCRelation.scala | 6 +- .../org/apache/spark/sql/jdbc/JDBCSuite.scala | 66 ++-- 4 files changed, 63 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/21fcac16/docs/sql-programming-guide.md -- diff --git a/docs/sql-programming-guide.md b/docs/sql-programming-guide.md index e815e5b..4b013c6 100644 --- a/docs/sql-programming-guide.md +++ b/docs/sql-programming-guide.md @@ -1435,6 +1435,13 @@ the following case-insensitive options: The custom schema to use for reading data from JDBC connectors. For example, "id DECIMAL(38, 0), name STRING". You can also specify partial fields, and the others use the default type mapping. For example, "id DECIMAL(38, 0)". The column names should be identical to the corresponding column names of JDBC table. Users can specify the corresponding data types of Spark SQL instead of using the defaults. This option applies only to reading. + + +pushDownPredicate + + The option to enable or disable predicate push-down into the JDBC data source. The default value is true, in which case Spark will push down filters to the JDBC data source as much as possible. Otherwise, if set to false, no filter will be pushed down to the JDBC data source and thus all filters will be handled by Spark. Predicate push-down is usually turned off when the predicate filtering is performed faster by Spark than by the JDBC data source. + + http://git-wip-us.apache.org/repos/asf/spark/blob/21fcac16/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala index 574aed4..d80efce 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala @@ -183,6 +183,9 @@ class JDBCOptions( } // An option to execute custom SQL before fetching data from the remote DB val sessionInitStatement = parameters.get(JDBC_SESSION_INIT_STATEMENT) + + // An option to allow/disallow pushing down predicate into JDBC data source + val pushDownPredicate = parameters.getOrElse(JDBC_PUSHDOWN_PREDICATE, "true").toBoolean } class JdbcOptionsInWrite( @@ -234,4 +237,5 @@ object JDBCOptions { val JDBC_BATCH_INSERT_SIZE = newOption("batchsize") val JDBC_TXN_ISOLATION_LEVEL = newOption("isolationLevel") val JDBC_SESSION_INIT_STATEMENT = newOption("sessionInitStatement") + val JDBC_PUSHDOWN_PREDICATE = newOption("pushDownPredicate") } http://git-wip-us.apache.org/repos/asf/spark/blob/21fcac16/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala index 97e2d25..4f78f59 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala @@ -172,7 +172,11 @@ private[sql] case class JDBCRelation( // Check if JDBCRDD.compileFilter can accept input filters override def unhandledFilters(filters: Array[Filter]): Array[Filter] = { -filters.filter(JDBCRDD.compileFilter(_, JdbcDialects.get
spark git commit: [SPARK-24927][BUILD][BRANCH-2.3] The scope of snappy-java cannot be "provided"
Repository: spark Updated Branches: refs/heads/branch-2.3 fa552c3c1 -> d5f340f27 [SPARK-24927][BUILD][BRANCH-2.3] The scope of snappy-java cannot be "provided" ## What changes were proposed in this pull request? Please see [SPARK-24927][1] for more details. [1]: https://issues.apache.org/jira/browse/SPARK-24927 ## How was this patch tested? Manually tested. Author: Cheng Lian Closes #21879 from liancheng/spark-24927. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d5f340f2 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d5f340f2 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d5f340f2 Branch: refs/heads/branch-2.3 Commit: d5f340f27706bd9767f23ac9726f904028916814 Parents: fa552c3 Author: Cheng Lian Authored: Fri Jul 27 08:57:48 2018 -0700 Committer: Xiao Li Committed: Fri Jul 27 08:57:48 2018 -0700 -- pom.xml | 1 - 1 file changed, 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/d5f340f2/pom.xml -- diff --git a/pom.xml b/pom.xml index ea6f27c..179e9d2 100644 --- a/pom.xml +++ b/pom.xml @@ -536,7 +536,6 @@ org.xerial.snappy snappy-java ${snappy.version} -${hadoop.deps.scope} org.lz4 - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24927][BUILD][BRANCH-2.3] The scope of snappy-java cannot be "provided"
Repository: spark Updated Branches: refs/heads/branch-2.2 f339e2fd7 -> 73764737d [SPARK-24927][BUILD][BRANCH-2.3] The scope of snappy-java cannot be "provided" ## What changes were proposed in this pull request? Please see [SPARK-24927][1] for more details. [1]: https://issues.apache.org/jira/browse/SPARK-24927 ## How was this patch tested? Manually tested. Author: Cheng Lian Closes #21879 from liancheng/spark-24927. (cherry picked from commit d5f340f27706bd9767f23ac9726f904028916814) Signed-off-by: Xiao Li Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/73764737 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/73764737 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/73764737 Branch: refs/heads/branch-2.2 Commit: 73764737d2230003d57210a68a87c5977bef6782 Parents: f339e2f Author: Cheng Lian Authored: Fri Jul 27 08:57:48 2018 -0700 Committer: Xiao Li Committed: Fri Jul 27 08:58:23 2018 -0700 -- pom.xml | 1 - 1 file changed, 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/73764737/pom.xml -- diff --git a/pom.xml b/pom.xml index cf98296..6ab9989 100644 --- a/pom.xml +++ b/pom.xml @@ -510,7 +510,6 @@ org.xerial.snappy snappy-java ${snappy.version} -${hadoop.deps.scope} net.jpountz.lz4 - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24927][BUILD][BRANCH-2.3] The scope of snappy-java cannot be "provided"
Repository: spark Updated Branches: refs/heads/master ef6c8395c -> c9bec1d37 [SPARK-24927][BUILD][BRANCH-2.3] The scope of snappy-java cannot be "provided" ## What changes were proposed in this pull request? Please see [SPARK-24927][1] for more details. [1]: https://issues.apache.org/jira/browse/SPARK-24927 ## How was this patch tested? Manually tested. Author: Cheng Lian Closes #21879 from liancheng/spark-24927. (cherry picked from commit d5f340f27706bd9767f23ac9726f904028916814) Signed-off-by: Xiao Li Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c9bec1d3 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c9bec1d3 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c9bec1d3 Branch: refs/heads/master Commit: c9bec1d3717e5aeb6d3ec95d4c78111bfc33e0ca Parents: ef6c839 Author: Cheng Lian Authored: Fri Jul 27 08:57:48 2018 -0700 Committer: Xiao Li Committed: Fri Jul 27 08:58:42 2018 -0700 -- pom.xml | 1 - 1 file changed, 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/c9bec1d3/pom.xml -- diff --git a/pom.xml b/pom.xml index d75db0f..f320844 100644 --- a/pom.xml +++ b/pom.xml @@ -538,7 +538,6 @@ org.xerial.snappy snappy-java ${snappy.version} -${hadoop.deps.scope} org.lz4 - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-21274][SQL] Implement EXCEPT ALL clause.
Repository: spark Updated Branches: refs/heads/master 5828f41a5 -> 10f1f1965 [SPARK-21274][SQL] Implement EXCEPT ALL clause. ## What changes were proposed in this pull request? Implements EXCEPT ALL clause through query rewrites using existing operators in Spark. In this PR, an internal UDTF (replicate_rows) is added to aid in preserving duplicate rows. Please refer to [Link](https://drive.google.com/open?id=1nyW0T0b_ajUduQoPgZLAsyHK8s3_dko3ulQuxaLpUXE) for the design. **Note** This proposed UDTF is kept as a internal function that is purely used to aid with this particular rewrite to give us flexibility to change to a more generalized UDTF in future. Input Query ``` SQL SELECT c1 FROM ut1 EXCEPT ALL SELECT c1 FROM ut2 ``` Rewritten Query ```SQL SELECT c1 FROM ( SELECT replicate_rows(sum_val, c1) FROM ( SELECT c1, sum_val FROM ( SELECT c1, sum(vcol) AS sum_val FROM ( SELECT 1L as vcol, c1 FROM ut1 UNION ALL SELECT -1L as vcol, c1 FROM ut2 ) AS union_all GROUP BY union_all.c1 ) WHERE sum_val > 0 ) ) ``` ## How was this patch tested? Added test cases in SQLQueryTestSuite, DataFrameSuite and SetOperationSuite Author: Dilip Biswal Closes #21857 from dilipbiswal/dkb_except_all_final. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/10f1f196 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/10f1f196 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/10f1f196 Branch: refs/heads/master Commit: 10f1f196595df66cb82d1fb9e27cc7ef0a176766 Parents: 5828f41 Author: Dilip Biswal Authored: Fri Jul 27 13:47:33 2018 -0700 Committer: Xiao Li Committed: Fri Jul 27 13:47:33 2018 -0700 -- python/pyspark/sql/dataframe.py | 25 ++ .../spark/sql/catalyst/analysis/Analyzer.scala | 5 +- .../sql/catalyst/analysis/TypeCoercion.scala| 12 +- .../analysis/UnsupportedOperationChecker.scala | 2 +- .../sql/catalyst/expressions/generators.scala | 26 ++ .../sql/catalyst/optimizer/Optimizer.scala | 61 +++- .../optimizer/ReplaceExceptWithFilter.scala | 2 +- .../spark/sql/catalyst/parser/AstBuilder.scala | 2 +- .../plans/logical/basicLogicalOperators.scala | 7 +- .../catalyst/optimizer/SetOperationSuite.scala | 24 +- .../sql/catalyst/parser/ErrorParserSuite.scala | 3 - .../sql/catalyst/parser/PlanParserSuite.scala | 1 - .../scala/org/apache/spark/sql/Dataset.scala| 16 + .../spark/sql/execution/SparkStrategies.scala | 6 +- .../resources/sql-tests/inputs/except-all.sql | 146 + .../sql-tests/results/except-all.sql.out| 319 +++ .../org/apache/spark/sql/DataFrameSuite.scala | 70 +++- 17 files changed, 708 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/10f1f196/python/pyspark/sql/dataframe.py -- diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py index c40aea9..b2e0a5b 100644 --- a/python/pyspark/sql/dataframe.py +++ b/python/pyspark/sql/dataframe.py @@ -293,6 +293,31 @@ class DataFrame(object): else: print(self._jdf.queryExecution().simpleString()) +@since(2.4) +def exceptAll(self, other): +"""Return a new :class:`DataFrame` containing rows in this :class:`DataFrame` but +not in another :class:`DataFrame` while preserving duplicates. + +This is equivalent to `EXCEPT ALL` in SQL. + +>>> df1 = spark.createDataFrame( +... [("a", 1), ("a", 1), ("a", 1), ("a", 2), ("b", 3), ("c", 4)], ["C1", "C2"]) +>>> df2 = spark.createDataFrame([("a", 1), ("b", 3)], ["C1", "C2"]) + +>>> df1.exceptAll(df2).show() ++---+---+ +| C1| C2| ++---+---+ +| a| 1| +| a| 1| +| a| 2| +| c| 4| ++---+---+ + +Also as standard in SQL, this function resolves columns by position (not by name). +""" +return DataFrame(self._jdf.exceptAll(other._jdf), self.sql_ctx) + @since(1.3) def isLocal(self): """Returns ``True`` if the :func:`collect` and :func:`take` methods can be run locally http://git-wip-us.apache.org/repos/asf/spark/blob/10f1f196/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index d18509f..8abb1c7 100644 --- a/sql/catalyst/src/main
spark git commit: [MINOR] Improve documentation for HiveStringType's
Repository: spark Updated Branches: refs/heads/master 10f1f1965 -> 34ebcc6b5 [MINOR] Improve documentation for HiveStringType's The diff should be self-explanatory. Author: Reynold Xin Closes #21897 from rxin/hivestringtypedoc. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/34ebcc6b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/34ebcc6b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/34ebcc6b Branch: refs/heads/master Commit: 34ebcc6b5246c1a47e6d3b2dbb23e368de25219e Parents: 10f1f19 Author: Reynold Xin Authored: Fri Jul 27 15:34:06 2018 -0700 Committer: Xiao Li Committed: Fri Jul 27 15:34:06 2018 -0700 -- .../scala/org/apache/spark/sql/types/HiveStringType.scala| 8 ++-- 1 file changed, 6 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/34ebcc6b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala index e0bca93..4eb3226 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala @@ -56,14 +56,18 @@ object HiveStringType { } /** - * Hive char type. + * Hive char type. Similar to other HiveStringType's, these datatypes should only used for + * parsing, and should NOT be used anywhere else. Any instance of these data types should be + * replaced by a [[StringType]] before analysis. */ case class CharType(length: Int) extends HiveStringType { override def simpleString: String = s"char($length)" } /** - * Hive varchar type. + * Hive varchar type. Similar to other HiveStringType's, these datatypes should only used for + * parsing, and should NOT be used anywhere else. Any instance of these data types should be + * replaced by a [[StringType]] before analysis. */ case class VarcharType(length: Int) extends HiveStringType { override def simpleString: String = s"varchar($length)" - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [MINOR] Update docs for functions.scala to make it clear not all the built-in functions are defined there
Repository: spark Updated Branches: refs/heads/master 34ebcc6b5 -> 6424b146c [MINOR] Update docs for functions.scala to make it clear not all the built-in functions are defined there The title summarizes the change. Author: Reynold Xin Closes #21318 from rxin/functions. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6424b146 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6424b146 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6424b146 Branch: refs/heads/master Commit: 6424b146c91fdca734a3ec972067e8e1f88e8b9e Parents: 34ebcc6 Author: Reynold Xin Authored: Fri Jul 27 17:24:55 2018 -0700 Committer: Xiao Li Committed: Fri Jul 27 17:24:55 2018 -0700 -- .../main/scala/org/apache/spark/sql/functions.scala | 16 +++- 1 file changed, 15 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/6424b146/sql/core/src/main/scala/org/apache/spark/sql/functions.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index bcd0c94..2772958 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -39,7 +39,21 @@ import org.apache.spark.util.Utils /** - * Functions available for DataFrame operations. + * Commonly used functions available for DataFrame operations. Using functions defined here provides + * a little bit more compile-time safety to make sure the function exists. + * + * Spark also includes more built-in functions that are less common and are not defined here. + * You can still access them (and all the functions defined here) using the `functions.expr()` API + * and calling them through a SQL expression string. You can find the entire list of functions for + * the latest version of Spark at https://spark.apache.org/docs/latest/api/sql/index.html. + * + * As an example, `isnan` is a function that is defined here. You can use `isnan(col("myCol"))` + * to invoke the `isnan` function. This way the programming language's compiler ensures `isnan` + * exists and is of the proper form. You can also use `expr("isnan(myCol)")` function to invoke the + * same function. In this case, Spark itself will ensure `isnan` exists when it analyzes the query. + * + * `regr_count` is an example of a function that is built-in but not defined here, because it is + * less commonly used. To invoke it, use `expr("regr_count(yCol, xCol)")`. * * @groupname udf_funcs UDF functions * @groupname agg_funcs Aggregate functions - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24809][SQL] Serializing LongToUnsafeRowMap in executor may result in data error
Repository: spark Updated Branches: refs/heads/master 8fe5d2c39 -> 2c54aae1b [SPARK-24809][SQL] Serializing LongToUnsafeRowMap in executor may result in data error When join key is long or int in broadcast join, Spark will use `LongToUnsafeRowMap` to store key-values of the table witch will be broadcasted. But, when `LongToUnsafeRowMap` is broadcasted to executors, and it is too big to hold in memory, it will be stored in disk. At that time, because `write` uses a variable `cursor` to determine how many bytes in `page` of `LongToUnsafeRowMap` will be write out and the `cursor` was not restore when deserializing, executor will write out nothing from page into disk. ## What changes were proposed in this pull request? Restore cursor value when deserializing. Author: liulijia Closes #21772 from liutang123/SPARK-24809. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2c54aae1 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2c54aae1 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2c54aae1 Branch: refs/heads/master Commit: 2c54aae1bc2fa3da26917c89e6201fb2108d9fab Parents: 8fe5d2c Author: liulijia Authored: Sun Jul 29 13:13:00 2018 -0700 Committer: Xiao Li Committed: Sun Jul 29 13:13:00 2018 -0700 -- .../sql/execution/joins/HashedRelation.scala| 2 ++ .../execution/joins/HashedRelationSuite.scala | 29 2 files changed, 31 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/2c54aae1/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index 20ce01f..86eb47a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -772,6 +772,8 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap array = readLongArray(readBuffer, length) val pageLength = readLong().toInt page = readLongArray(readBuffer, pageLength) +// Restore cursor variable to make this map able to be serialized again on executors. +cursor = pageLength * 8 + Platform.LONG_ARRAY_OFFSET } override def readExternal(in: ObjectInput): Unit = { http://git-wip-us.apache.org/repos/asf/spark/blob/2c54aae1/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala -- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala index 037cc2e..d9b34dc 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala @@ -278,6 +278,35 @@ class HashedRelationSuite extends SparkFunSuite with SharedSQLContext { map.free() } + test("SPARK-24809: Serializing LongToUnsafeRowMap in executor may result in data error") { +val unsafeProj = UnsafeProjection.create(Array[DataType](LongType)) +val originalMap = new LongToUnsafeRowMap(mm, 1) + +val key1 = 1L +val value1 = 4852306286022334418L + +val key2 = 2L +val value2 = 8813607448788216010L + +originalMap.append(key1, unsafeProj(InternalRow(value1))) +originalMap.append(key2, unsafeProj(InternalRow(value2))) +originalMap.optimize() + +val ser = sparkContext.env.serializer.newInstance() +// Simulate serialize/deserialize twice on driver and executor +val firstTimeSerialized = ser.deserialize[LongToUnsafeRowMap](ser.serialize(originalMap)) +val secondTimeSerialized = + ser.deserialize[LongToUnsafeRowMap](ser.serialize(firstTimeSerialized)) + +val resultRow = new UnsafeRow(1) +assert(secondTimeSerialized.getValue(key1, resultRow).getLong(0) === value1) +assert(secondTimeSerialized.getValue(key2, resultRow).getLong(0) === value2) + +originalMap.free() +firstTimeSerialized.free() +secondTimeSerialized.free() + } + test("Spark-14521") { val ser = new KryoSerializer( (new SparkConf).set("spark.kryo.referenceTracking", "false")).newInstance() - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24809][SQL] Serializing LongToUnsafeRowMap in executor may result in data error
Repository: spark Updated Branches: refs/heads/branch-2.3 d5f340f27 -> 71eb7d468 [SPARK-24809][SQL] Serializing LongToUnsafeRowMap in executor may result in data error When join key is long or int in broadcast join, Spark will use `LongToUnsafeRowMap` to store key-values of the table witch will be broadcasted. But, when `LongToUnsafeRowMap` is broadcasted to executors, and it is too big to hold in memory, it will be stored in disk. At that time, because `write` uses a variable `cursor` to determine how many bytes in `page` of `LongToUnsafeRowMap` will be write out and the `cursor` was not restore when deserializing, executor will write out nothing from page into disk. ## What changes were proposed in this pull request? Restore cursor value when deserializing. Author: liulijia Closes #21772 from liutang123/SPARK-24809. (cherry picked from commit 2c54aae1bc2fa3da26917c89e6201fb2108d9fab) Signed-off-by: Xiao Li Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/71eb7d46 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/71eb7d46 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/71eb7d46 Branch: refs/heads/branch-2.3 Commit: 71eb7d4682a7e85e4de580ffe110da961d84817f Parents: d5f340f Author: liulijia Authored: Sun Jul 29 13:13:00 2018 -0700 Committer: Xiao Li Committed: Sun Jul 29 13:13:22 2018 -0700 -- .../sql/execution/joins/HashedRelation.scala| 2 ++ .../execution/joins/HashedRelationSuite.scala | 29 2 files changed, 31 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/71eb7d46/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index 20ce01f..86eb47a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -772,6 +772,8 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap array = readLongArray(readBuffer, length) val pageLength = readLong().toInt page = readLongArray(readBuffer, pageLength) +// Restore cursor variable to make this map able to be serialized again on executors. +cursor = pageLength * 8 + Platform.LONG_ARRAY_OFFSET } override def readExternal(in: ObjectInput): Unit = { http://git-wip-us.apache.org/repos/asf/spark/blob/71eb7d46/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala -- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala index 037cc2e..d9b34dc 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala @@ -278,6 +278,35 @@ class HashedRelationSuite extends SparkFunSuite with SharedSQLContext { map.free() } + test("SPARK-24809: Serializing LongToUnsafeRowMap in executor may result in data error") { +val unsafeProj = UnsafeProjection.create(Array[DataType](LongType)) +val originalMap = new LongToUnsafeRowMap(mm, 1) + +val key1 = 1L +val value1 = 4852306286022334418L + +val key2 = 2L +val value2 = 8813607448788216010L + +originalMap.append(key1, unsafeProj(InternalRow(value1))) +originalMap.append(key2, unsafeProj(InternalRow(value2))) +originalMap.optimize() + +val ser = sparkContext.env.serializer.newInstance() +// Simulate serialize/deserialize twice on driver and executor +val firstTimeSerialized = ser.deserialize[LongToUnsafeRowMap](ser.serialize(originalMap)) +val secondTimeSerialized = + ser.deserialize[LongToUnsafeRowMap](ser.serialize(firstTimeSerialized)) + +val resultRow = new UnsafeRow(1) +assert(secondTimeSerialized.getValue(key1, resultRow).getLong(0) === value1) +assert(secondTimeSerialized.getValue(key2, resultRow).getLong(0) === value2) + +originalMap.free() +firstTimeSerialized.free() +secondTimeSerialized.free() + } + test("Spark-14521") { val ser = new KryoSerializer( (new SparkConf).set("spark.kryo.referenceTracking", "false")).newInstance() - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.
spark git commit: [SPARK-24809][SQL] Serializing LongToUnsafeRowMap in executor may result in data error
Repository: spark Updated Branches: refs/heads/branch-2.2 73764737d -> f52d0c451 [SPARK-24809][SQL] Serializing LongToUnsafeRowMap in executor may result in data error When join key is long or int in broadcast join, Spark will use `LongToUnsafeRowMap` to store key-values of the table witch will be broadcasted. But, when `LongToUnsafeRowMap` is broadcasted to executors, and it is too big to hold in memory, it will be stored in disk. At that time, because `write` uses a variable `cursor` to determine how many bytes in `page` of `LongToUnsafeRowMap` will be write out and the `cursor` was not restore when deserializing, executor will write out nothing from page into disk. ## What changes were proposed in this pull request? Restore cursor value when deserializing. Author: liulijia Closes #21772 from liutang123/SPARK-24809. (cherry picked from commit 2c54aae1bc2fa3da26917c89e6201fb2108d9fab) Signed-off-by: Xiao Li Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f52d0c45 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f52d0c45 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f52d0c45 Branch: refs/heads/branch-2.2 Commit: f52d0c4515f3f0ceaea03c661fb7739c70c25236 Parents: 7376473 Author: liulijia Authored: Sun Jul 29 13:13:00 2018 -0700 Committer: Xiao Li Committed: Sun Jul 29 13:13:57 2018 -0700 -- .../sql/execution/joins/HashedRelation.scala| 2 ++ .../execution/joins/HashedRelationSuite.scala | 29 2 files changed, 31 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/f52d0c45/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index 07ee3d0..78190bf 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -741,6 +741,8 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap array = readLongArray(readBuffer, length) val pageLength = readLong().toInt page = readLongArray(readBuffer, pageLength) +// Restore cursor variable to make this map able to be serialized again on executors. +cursor = pageLength * 8 + Platform.LONG_ARRAY_OFFSET } override def readExternal(in: ObjectInput): Unit = { http://git-wip-us.apache.org/repos/asf/spark/blob/f52d0c45/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala -- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala index f0288c8..9c9e9dc 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala @@ -277,6 +277,35 @@ class HashedRelationSuite extends SparkFunSuite with SharedSQLContext { map.free() } + test("SPARK-24809: Serializing LongToUnsafeRowMap in executor may result in data error") { +val unsafeProj = UnsafeProjection.create(Array[DataType](LongType)) +val originalMap = new LongToUnsafeRowMap(mm, 1) + +val key1 = 1L +val value1 = 4852306286022334418L + +val key2 = 2L +val value2 = 8813607448788216010L + +originalMap.append(key1, unsafeProj(InternalRow(value1))) +originalMap.append(key2, unsafeProj(InternalRow(value2))) +originalMap.optimize() + +val ser = sparkContext.env.serializer.newInstance() +// Simulate serialize/deserialize twice on driver and executor +val firstTimeSerialized = ser.deserialize[LongToUnsafeRowMap](ser.serialize(originalMap)) +val secondTimeSerialized = + ser.deserialize[LongToUnsafeRowMap](ser.serialize(firstTimeSerialized)) + +val resultRow = new UnsafeRow(1) +assert(secondTimeSerialized.getValue(key1, resultRow).getLong(0) === value1) +assert(secondTimeSerialized.getValue(key2, resultRow).getLong(0) === value2) + +originalMap.free() +firstTimeSerialized.free() +secondTimeSerialized.free() + } + test("Spark-14521") { val ser = new KryoSerializer( (new SparkConf).set("spark.kryo.referenceTracking", "false")).newInstance() - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.
spark git commit: [SPARK-24809][SQL] Serializing LongToUnsafeRowMap in executor may result in data error
Repository: spark Updated Branches: refs/heads/branch-2.1 7d50fec3f -> a3eb07db3 [SPARK-24809][SQL] Serializing LongToUnsafeRowMap in executor may result in data error When join key is long or int in broadcast join, Spark will use `LongToUnsafeRowMap` to store key-values of the table witch will be broadcasted. But, when `LongToUnsafeRowMap` is broadcasted to executors, and it is too big to hold in memory, it will be stored in disk. At that time, because `write` uses a variable `cursor` to determine how many bytes in `page` of `LongToUnsafeRowMap` will be write out and the `cursor` was not restore when deserializing, executor will write out nothing from page into disk. ## What changes were proposed in this pull request? Restore cursor value when deserializing. Author: liulijia Closes #21772 from liutang123/SPARK-24809. (cherry picked from commit 2c54aae1bc2fa3da26917c89e6201fb2108d9fab) Signed-off-by: Xiao Li Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a3eb07db Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a3eb07db Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a3eb07db Branch: refs/heads/branch-2.1 Commit: a3eb07db3be80be663ca66f1e9a11fcef8ab6c20 Parents: 7d50fec Author: liulijia Authored: Sun Jul 29 13:13:00 2018 -0700 Committer: Xiao Li Committed: Sun Jul 29 13:14:57 2018 -0700 -- .../sql/execution/joins/HashedRelation.scala| 2 ++ .../execution/joins/HashedRelationSuite.scala | 29 2 files changed, 31 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/a3eb07db/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index f7e8ea6..206afcd 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -741,6 +741,8 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap array = readLongArray(readBuffer, length) val pageLength = readLong().toInt page = readLongArray(readBuffer, pageLength) +// Restore cursor variable to make this map able to be serialized again on executors. +cursor = pageLength * 8 + Platform.LONG_ARRAY_OFFSET } override def readExternal(in: ObjectInput): Unit = { http://git-wip-us.apache.org/repos/asf/spark/blob/a3eb07db/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala -- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala index f0288c8..9c9e9dc 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala @@ -277,6 +277,35 @@ class HashedRelationSuite extends SparkFunSuite with SharedSQLContext { map.free() } + test("SPARK-24809: Serializing LongToUnsafeRowMap in executor may result in data error") { +val unsafeProj = UnsafeProjection.create(Array[DataType](LongType)) +val originalMap = new LongToUnsafeRowMap(mm, 1) + +val key1 = 1L +val value1 = 4852306286022334418L + +val key2 = 2L +val value2 = 8813607448788216010L + +originalMap.append(key1, unsafeProj(InternalRow(value1))) +originalMap.append(key2, unsafeProj(InternalRow(value2))) +originalMap.optimize() + +val ser = sparkContext.env.serializer.newInstance() +// Simulate serialize/deserialize twice on driver and executor +val firstTimeSerialized = ser.deserialize[LongToUnsafeRowMap](ser.serialize(originalMap)) +val secondTimeSerialized = + ser.deserialize[LongToUnsafeRowMap](ser.serialize(firstTimeSerialized)) + +val resultRow = new UnsafeRow(1) +assert(secondTimeSerialized.getValue(key1, resultRow).getLong(0) === value1) +assert(secondTimeSerialized.getValue(key2, resultRow).getLong(0) === value2) + +originalMap.free() +firstTimeSerialized.free() +secondTimeSerialized.free() + } + test("Spark-14521") { val ser = new KryoSerializer( (new SparkConf).set("spark.kryo.referenceTracking", "false")).newInstance() - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.
spark git commit: [SPARK-21274][SQL] Implement INTERSECT ALL clause
Repository: spark Updated Branches: refs/heads/master 6690924c4 -> 65a4bc143 [SPARK-21274][SQL] Implement INTERSECT ALL clause ## What changes were proposed in this pull request? Implements INTERSECT ALL clause through query rewrites using existing operators in Spark. Please refer to [Link](https://drive.google.com/open?id=1nyW0T0b_ajUduQoPgZLAsyHK8s3_dko3ulQuxaLpUXE) for the design. Input Query ``` SQL SELECT c1 FROM ut1 INTERSECT ALL SELECT c1 FROM ut2 ``` Rewritten Query ```SQL SELECT c1 FROM ( SELECT replicate_row(min_count, c1) FROM ( SELECT c1, IF (vcol1_cnt > vcol2_cnt, vcol2_cnt, vcol1_cnt) AS min_count FROM ( SELECT c1, count(vcol1) as vcol1_cnt, count(vcol2) as vcol2_cnt FROM ( SELECT c1, true as vcol1, null as vcol2 FROM ut1 UNION ALL SELECT c1, null as vcol1, true as vcol2 FROM ut2 ) AS union_all GROUP BY c1 HAVING vcol1_cnt >= 1 AND vcol2_cnt >= 1 ) ) ) ``` ## How was this patch tested? Added test cases in SQLQueryTestSuite, DataFrameSuite, SetOperationSuite Author: Dilip Biswal Closes #21886 from dilipbiswal/dkb_intersect_all_final. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/65a4bc14 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/65a4bc14 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/65a4bc14 Branch: refs/heads/master Commit: 65a4bc143ab5dc2ced589dc107bbafa8a7290931 Parents: 6690924 Author: Dilip Biswal Authored: Sun Jul 29 22:11:01 2018 -0700 Committer: Xiao Li Committed: Sun Jul 29 22:11:01 2018 -0700 -- python/pyspark/sql/dataframe.py | 22 ++ .../spark/sql/catalyst/analysis/Analyzer.scala | 2 +- .../sql/catalyst/analysis/TypeCoercion.scala| 4 +- .../analysis/UnsupportedOperationChecker.scala | 2 +- .../sql/catalyst/optimizer/Optimizer.scala | 81 ++- .../spark/sql/catalyst/parser/AstBuilder.scala | 2 +- .../plans/logical/basicLogicalOperators.scala | 7 +- .../catalyst/optimizer/SetOperationSuite.scala | 32 ++- .../sql/catalyst/parser/PlanParserSuite.scala | 1 - .../scala/org/apache/spark/sql/Dataset.scala| 19 +- .../spark/sql/execution/SparkStrategies.scala | 8 +- .../sql-tests/inputs/intersect-all.sql | 123 ++ .../sql-tests/results/intersect-all.sql.out | 241 +++ .../org/apache/spark/sql/DataFrameSuite.scala | 54 + .../org/apache/spark/sql/test/SQLTestData.scala | 13 + 15 files changed, 599 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/65a4bc14/python/pyspark/sql/dataframe.py -- diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py index b2e0a5b..07fb260 100644 --- a/python/pyspark/sql/dataframe.py +++ b/python/pyspark/sql/dataframe.py @@ -1500,6 +1500,28 @@ class DataFrame(object): """ return DataFrame(self._jdf.intersect(other._jdf), self.sql_ctx) +@since(2.4) +def intersectAll(self, other): +""" Return a new :class:`DataFrame` containing rows in both this dataframe and other +dataframe while preserving duplicates. + +This is equivalent to `INTERSECT ALL` in SQL. +>>> df1 = spark.createDataFrame([("a", 1), ("a", 1), ("b", 3), ("c", 4)], ["C1", "C2"]) +>>> df2 = spark.createDataFrame([("a", 1), ("a", 1), ("b", 3)], ["C1", "C2"]) + +>>> df1.intersectAll(df2).sort("C1", "C2").show() ++---+---+ +| C1| C2| ++---+---+ +| a| 1| +| a| 1| +| b| 3| ++---+---+ + +Also as standard in SQL, this function resolves columns by position (not by name). +""" +return DataFrame(self._jdf.intersectAll(other._jdf), self.sql_ctx) + @since(1.3) def subtract(self, other): """ Return a new :class:`DataFrame` containing rows in this frame http://git-wip-us.apache.org/repos/asf/spark/blob/65a4bc14/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 8abb1c7..9965cd6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -914,7 +914,7 @@ cl
spark git commit: [SPARK-24771][BUILD] Upgrade Apache AVRO to 1.8.2
Repository: spark Updated Branches: refs/heads/master fca0b8528 -> b90bfe3c4 [SPARK-24771][BUILD] Upgrade Apache AVRO to 1.8.2 ## What changes were proposed in this pull request? Upgrade Apache Avro from 1.7.7 to 1.8.2. The major new features: 1. More logical types. From the spec of 1.8.2 https://avro.apache.org/docs/1.8.2/spec.html#Logical+Types we can see comparing to [1.7.7](https://avro.apache.org/docs/1.7.7/spec.html#Logical+Types), the new version support: - Date - Time (millisecond precision) - Time (microsecond precision) - Timestamp (millisecond precision) - Timestamp (microsecond precision) - Duration 2. Single-object encoding: https://avro.apache.org/docs/1.8.2/spec.html#single_object_encoding This PR aims to update Apache Spark to support these new features. ## How was this patch tested? Unit test Author: Gengliang Wang Closes #21761 from gengliangwang/upgrade_avro_1.8. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b90bfe3c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b90bfe3c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b90bfe3c Branch: refs/heads/master Commit: b90bfe3c42eb9b51e6131a8f8923bcddfccd75bb Parents: fca0b85 Author: Gengliang Wang Authored: Mon Jul 30 07:30:47 2018 -0700 Committer: Xiao Li Committed: Mon Jul 30 07:30:47 2018 -0700 -- dev/deps/spark-deps-hadoop-2.6 | 10 +- dev/deps/spark-deps-hadoop-2.7 | 10 +- dev/deps/spark-deps-hadoop-3.1 | 10 +- pom.xml| 2 +- sql/core/pom.xml | 13 - 5 files changed, 16 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/b90bfe3c/dev/deps/spark-deps-hadoop-2.6 -- diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index ff6d5c3..4ef61b2 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -18,9 +18,9 @@ arrow-format-0.8.0.jar arrow-memory-0.8.0.jar arrow-vector-0.8.0.jar automaton-1.11-8.jar -avro-1.7.7.jar -avro-ipc-1.7.7.jar -avro-mapred-1.7.7-hadoop2.jar +avro-1.8.2.jar +avro-ipc-1.8.2.jar +avro-mapred-1.8.2-hadoop2.jar base64-2.3.8.jar bcprov-jdk15on-1.58.jar bonecp-0.8.0.RELEASE.jar @@ -37,7 +37,7 @@ commons-cli-1.2.jar commons-codec-1.10.jar commons-collections-3.2.2.jar commons-compiler-3.0.8.jar -commons-compress-1.4.1.jar +commons-compress-1.8.1.jar commons-configuration-1.6.jar commons-crypto-1.0.0.jar commons-dbcp-1.4.jar @@ -196,7 +196,7 @@ validation-api-1.1.0.Final.jar xbean-asm6-shaded-4.8.jar xercesImpl-2.9.1.jar xmlenc-0.52.jar -xz-1.0.jar +xz-1.5.jar zjsonpatch-0.3.0.jar zookeeper-3.4.6.jar zstd-jni-1.3.2-2.jar http://git-wip-us.apache.org/repos/asf/spark/blob/b90bfe3c/dev/deps/spark-deps-hadoop-2.7 -- diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index 72a94f8..a74ce1f 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -18,9 +18,9 @@ arrow-format-0.8.0.jar arrow-memory-0.8.0.jar arrow-vector-0.8.0.jar automaton-1.11-8.jar -avro-1.7.7.jar -avro-ipc-1.7.7.jar -avro-mapred-1.7.7-hadoop2.jar +avro-1.8.2.jar +avro-ipc-1.8.2.jar +avro-mapred-1.8.2-hadoop2.jar base64-2.3.8.jar bcprov-jdk15on-1.58.jar bonecp-0.8.0.RELEASE.jar @@ -37,7 +37,7 @@ commons-cli-1.2.jar commons-codec-1.10.jar commons-collections-3.2.2.jar commons-compiler-3.0.8.jar -commons-compress-1.4.1.jar +commons-compress-1.8.1.jar commons-configuration-1.6.jar commons-crypto-1.0.0.jar commons-dbcp-1.4.jar @@ -197,7 +197,7 @@ validation-api-1.1.0.Final.jar xbean-asm6-shaded-4.8.jar xercesImpl-2.9.1.jar xmlenc-0.52.jar -xz-1.0.jar +xz-1.5.jar zjsonpatch-0.3.0.jar zookeeper-3.4.6.jar zstd-jni-1.3.2-2.jar http://git-wip-us.apache.org/repos/asf/spark/blob/b90bfe3c/dev/deps/spark-deps-hadoop-3.1 -- diff --git a/dev/deps/spark-deps-hadoop-3.1 b/dev/deps/spark-deps-hadoop-3.1 index 3409dc4..e0fcca0 100644 --- a/dev/deps/spark-deps-hadoop-3.1 +++ b/dev/deps/spark-deps-hadoop-3.1 @@ -16,9 +16,9 @@ arrow-format-0.8.0.jar arrow-memory-0.8.0.jar arrow-vector-0.8.0.jar automaton-1.11-8.jar -avro-1.7.7.jar -avro-ipc-1.7.7.jar -avro-mapred-1.7.7-hadoop2.jar +avro-1.8.2.jar +avro-ipc-1.8.2.jar +avro-mapred-1.8.2-hadoop2.jar base64-2.3.8.jar bcprov-jdk15on-1.58.jar bonecp-0.8.0.RELEASE.jar @@ -34,7 +34,7 @@ commons-cli-1.2.jar commons-codec-1.10.jar commons-collections-3.2.2.jar commons-compiler-3.0.8.jar -commons-compress-1.4.1.jar +commons-compress-1.8.1.jar commons-configuration2-2.1.1.jar commons-crypto-1.0.0.jar commons-daemon-1.0
spark git commit: [SPARK-22814][SQL] Support Date/Timestamp in a JDBC partition column
Repository: spark Updated Branches: refs/heads/master b90bfe3c4 -> 47d84e4d0 [SPARK-22814][SQL] Support Date/Timestamp in a JDBC partition column ## What changes were proposed in this pull request? This pr supported Date/Timestamp in a JDBC partition column (a numeric column is only supported in the master). This pr also modified code to verify a partition column type; ``` val jdbcTable = spark.read .option("partitionColumn", "text") .option("lowerBound", "aaa") .option("upperBound", "zzz") .option("numPartitions", 2) .jdbc("jdbc:postgresql:postgres", "t", options) // with this pr org.apache.spark.sql.AnalysisException: Partition column type should be numeric, date, or timestamp, but string found.; at org.apache.spark.sql.execution.datasources.jdbc.JDBCRelation$.verifyAndGetNormalizedPartitionColumn(JDBCRelation.scala:165) at org.apache.spark.sql.execution.datasources.jdbc.JDBCRelation$.columnPartition(JDBCRelation.scala:85) at org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider.createRelation(JdbcRelationProvider.scala:36) at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:317) // without this pr java.lang.NumberFormatException: For input string: "aaa" at java.lang.NumberFormatException.forInputString(NumberFormatException.java:65) at java.lang.Long.parseLong(Long.java:589) at java.lang.Long.parseLong(Long.java:631) at scala.collection.immutable.StringLike$class.toLong(StringLike.scala:277) ``` Closes #1 ## How was this patch tested? Added tests in `JDBCSuite`. Author: Takeshi Yamamuro Closes #21834 from maropu/SPARK-22814. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/47d84e4d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/47d84e4d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/47d84e4d Branch: refs/heads/master Commit: 47d84e4d0e56e14f9402770dceaf0b4302c00e98 Parents: b90bfe3 Author: Takeshi Yamamuro Authored: Mon Jul 30 07:42:00 2018 -0700 Committer: Xiao Li Committed: Mon Jul 30 07:42:00 2018 -0700 -- docs/sql-programming-guide.md | 4 +- .../spark/sql/jdbc/OracleIntegrationSuite.scala | 86 +-- .../spark/sql/catalyst/util/DateTimeUtils.scala | 10 +- .../datasources/PartitioningUtils.scala | 2 +- .../datasources/jdbc/JDBCOptions.scala | 4 +- .../datasources/jdbc/JDBCRelation.scala | 107 +++ .../datasources/jdbc/JdbcRelationProvider.scala | 21 +--- .../org/apache/spark/sql/jdbc/JDBCSuite.scala | 77 - 8 files changed, 258 insertions(+), 53 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/47d84e4d/docs/sql-programming-guide.md -- diff --git a/docs/sql-programming-guide.md b/docs/sql-programming-guide.md index 4b013c6..cff521c 100644 --- a/docs/sql-programming-guide.md +++ b/docs/sql-programming-guide.md @@ -1345,8 +1345,8 @@ the following case-insensitive options: These options must all be specified if any of them is specified. In addition, numPartitions must be specified. They describe how to partition the table when reading in parallel from multiple workers. - partitionColumn must be a numeric column from the table in question. Notice - that lowerBound and upperBound are just used to decide the + partitionColumn must be a numeric, date, or timestamp column from the table in question. + Notice that lowerBound and upperBound are just used to decide the partition stride, not for filtering the rows in table. So all rows in the table will be partitioned and returned. This option applies only to reading. http://git-wip-us.apache.org/repos/asf/spark/blob/47d84e4d/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala -- diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala index 8512496..09a2cd8 100644 --- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala +++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala @@ -17,12 +17,14 @@ package org.apache.spark.sql.jdbc +import java.math.BigDecimal import java.sql.{Connection, Date, Timestamp} import java.util.{Properties, TimeZone} -import java.math.BigDecimal -import org.apache.spark.sql.{DataFrame, QueryTest, Row, SaveMode} +import org.apache.spark.sql.{Row, SaveMode} import org.
spark git commit: [SPARK-24865][SQL] Remove AnalysisBarrier addendum
Repository: spark Updated Branches: refs/heads/master d6b7545b5 -> abbb4ab4d [SPARK-24865][SQL] Remove AnalysisBarrier addendum ## What changes were proposed in this pull request? I didn't want to pollute the diff in the previous PR and left some TODOs. This is a follow-up to address those TODOs. ## How was this patch tested? Should be covered by existing tests. Author: Reynold Xin Closes #21896 from rxin/SPARK-24865-addendum. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/abbb4ab4 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/abbb4ab4 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/abbb4ab4 Branch: refs/heads/master Commit: abbb4ab4d8b12ba2d94b16407c0d62ae207ee4fa Parents: d6b7545 Author: Reynold Xin Authored: Mon Jul 30 14:05:45 2018 -0700 Committer: Xiao Li Committed: Mon Jul 30 14:05:45 2018 -0700 -- .../spark/sql/catalyst/analysis/Analyzer.scala | 1 - .../scala/org/apache/spark/sql/Dataset.scala| 88 ++-- .../spark/sql/RelationalGroupedDataset.scala| 13 ++- .../continuous/ContinuousAggregationSuite.scala | 4 +- 4 files changed, 50 insertions(+), 56 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/abbb4ab4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 9965cd6..1488ede 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -799,7 +799,6 @@ class Analyzer( right case Some((oldRelation, newRelation)) => val attributeRewrites = AttributeMap(oldRelation.output.zip(newRelation.output)) - // TODO(rxin): Why do we need transformUp here? right transformUp { case r if r == oldRelation => newRelation } transformUp { http://git-wip-us.apache.org/repos/asf/spark/blob/abbb4ab4/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala index d36c8d1..3b0a6d8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala @@ -195,10 +195,6 @@ class Dataset[T] private[sql]( } } - // Wraps analyzed logical plans with an analysis barrier so we won't traverse/resolve it again. - // TODO(rxin): remove this later. - @transient private[sql] val planWithBarrier = logicalPlan - /** * Currently [[ExpressionEncoder]] is the only implementation of [[Encoder]], here we turn the * passed in encoder to [[ExpressionEncoder]] explicitly, and mark it implicit so that we can use @@ -427,7 +423,7 @@ class Dataset[T] private[sql]( */ @Experimental @InterfaceStability.Evolving - def as[U : Encoder]: Dataset[U] = Dataset[U](sparkSession, planWithBarrier) + def as[U : Encoder]: Dataset[U] = Dataset[U](sparkSession, logicalPlan) /** * Converts this strongly typed collection of data to generic `DataFrame` with columns renamed. @@ -681,7 +677,7 @@ class Dataset[T] private[sql]( require(parsedDelay.milliseconds >= 0 && parsedDelay.months >= 0, s"delay threshold ($delayThreshold) should not be negative.") EliminateEventTimeWatermark( - EventTimeWatermark(UnresolvedAttribute(eventTime), parsedDelay, planWithBarrier)) + EventTimeWatermark(UnresolvedAttribute(eventTime), parsedDelay, logicalPlan)) } /** @@ -854,7 +850,7 @@ class Dataset[T] private[sql]( * @since 2.0.0 */ def join(right: Dataset[_]): DataFrame = withPlan { -Join(planWithBarrier, right.planWithBarrier, joinType = Inner, None) +Join(logicalPlan, right.logicalPlan, joinType = Inner, None) } /** @@ -932,7 +928,7 @@ class Dataset[T] private[sql]( // Analyze the self join. The assumption is that the analyzer will disambiguate left vs right // by creating a new instance for one of the branch. val joined = sparkSession.sessionState.executePlan( - Join(planWithBarrier, right.planWithBarrier, joinType = JoinType(joinType), None)) + Join(logicalPlan, right.logicalPlan, joinType = JoinType(joinType), None)) .analyzed.asInstanceOf[Join] withPlan { @@ -993,7 +989,7 @@ class Dataset[T] private[sql]( // Trigger analysis so in the case of self-join, the analyzer will clone the plan.
spark git commit: [SPARK-24972][SQL] PivotFirst could not handle pivot columns of complex types
Repository: spark Updated Branches: refs/heads/master 8141d5592 -> b4fd75fb9 [SPARK-24972][SQL] PivotFirst could not handle pivot columns of complex types ## What changes were proposed in this pull request? When the pivot column is of a complex type, the eval() result will be an UnsafeRow, while the keys of the HashMap for column value matching is a GenericInternalRow. As a result, there will be no match and the result will always be empty. So for a pivot column of complex-types, we should: 1) If the complex-type is not comparable (orderable), throw an Exception. It cannot be a pivot column. 2) Otherwise, if it goes through the `PivotFirst` code path, `PivotFirst` should use a TreeMap instead of HashMap for such columns. This PR has also reverted the walk-around in Analyzer that had been introduced to avoid this `PivotFirst` issue. ## How was this patch tested? Added UT. Author: maryannxue Closes #21926 from maryannxue/pivot_followup. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b4fd75fb Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b4fd75fb Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b4fd75fb Branch: refs/heads/master Commit: b4fd75fb9b615cfe592ad269cf20d02b483a0d33 Parents: 8141d55 Author: maryannxue Authored: Mon Jul 30 23:43:53 2018 -0700 Committer: Xiao Li Committed: Mon Jul 30 23:43:53 2018 -0700 -- .../spark/sql/catalyst/analysis/Analyzer.scala | 12 +- .../expressions/aggregate/PivotFirst.scala | 11 +- .../test/resources/sql-tests/inputs/pivot.sql | 78 - .../resources/sql-tests/results/pivot.sql.out | 116 +-- 4 files changed, 199 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/b4fd75fb/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 1488ede..76dc867 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -529,6 +529,10 @@ class Analyzer( || (p.groupByExprsOpt.isDefined && !p.groupByExprsOpt.get.forall(_.resolved)) || !p.pivotColumn.resolved || !p.pivotValues.forall(_.resolved) => p case Pivot(groupByExprsOpt, pivotColumn, pivotValues, aggregates, child) => +if (!RowOrdering.isOrderable(pivotColumn.dataType)) { + throw new AnalysisException( +s"Invalid pivot column '${pivotColumn}'. Pivot columns must be comparable.") +} // Check all aggregate expressions. aggregates.foreach(checkValidAggregateExpression) // Check all pivot values are literal and match pivot column data type. @@ -574,10 +578,14 @@ class Analyzer( // Since evaluating |pivotValues| if statements for each input row can get slow this is an // alternate plan that instead uses two steps of aggregation. val namedAggExps: Seq[NamedExpression] = aggregates.map(a => Alias(a, a.sql)()) - val bigGroup = groupByExprs ++ pivotColumn.references + val namedPivotCol = pivotColumn match { +case n: NamedExpression => n +case _ => Alias(pivotColumn, "__pivot_col")() + } + val bigGroup = groupByExprs :+ namedPivotCol val firstAgg = Aggregate(bigGroup, bigGroup ++ namedAggExps, child) val pivotAggs = namedAggExps.map { a => -Alias(PivotFirst(pivotColumn, a.toAttribute, evalPivotValues) +Alias(PivotFirst(namedPivotCol.toAttribute, a.toAttribute, evalPivotValues) .toAggregateExpression() , "__pivot_" + a.sql)() } http://git-wip-us.apache.org/repos/asf/spark/blob/b4fd75fb/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PivotFirst.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PivotFirst.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PivotFirst.scala index 5237148..33bc5b5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PivotFirst.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PivotFirst.scala @@ -17,11 +17,11 @@ package org.apache.spark.sql.catalyst.expressions.aggregate -import scala.collection.immutable.HashMap +import scala.collection.immutable.{
spark git commit: [SPARK-24536] Validate that an evaluated limit clause cannot be null
Repository: spark Updated Branches: refs/heads/master b4fd75fb9 -> 4ac2126bc [SPARK-24536] Validate that an evaluated limit clause cannot be null ## What changes were proposed in this pull request? It proposes a version in which nullable expressions are not valid in the limit clause ## How was this patch tested? It was tested with unit and e2e tests. Please review http://spark.apache.org/contributing.html before opening a pull request. Author: Mauro Palsgraaf Closes #21807 from mauropalsgraaf/SPARK-24536. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4ac2126b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4ac2126b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4ac2126b Branch: refs/heads/master Commit: 4ac2126bc64bad1b4cbe1c697b4bcafacd67c96c Parents: b4fd75f Author: Mauro Palsgraaf Authored: Tue Jul 31 08:18:08 2018 -0700 Committer: Xiao Li Committed: Tue Jul 31 08:18:08 2018 -0700 -- .../sql/catalyst/analysis/CheckAnalysis.scala | 12 -- .../catalyst/analysis/AnalysisErrorSuite.scala | 6 +++ .../test/resources/sql-tests/inputs/limit.sql | 5 +++ .../resources/sql-tests/results/limit.sql.out | 45 ++-- 4 files changed, 50 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/4ac2126b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala index f9478a1..4addc83 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala @@ -68,10 +68,14 @@ trait CheckAnalysis extends PredicateHelper { case e if e.dataType != IntegerType => failAnalysis( s"The limit expression must be integer type, but got " + e.dataType.catalogString) - case e if e.eval().asInstanceOf[Int] < 0 => failAnalysis( -"The limit expression must be equal to or greater than 0, but got " + - e.eval().asInstanceOf[Int]) - case e => // OK + case e => +e.eval() match { + case null => failAnalysis( +s"The evaluated limit expression must not be null, but got ${limitExpr.sql}") + case v: Int if v < 0 => failAnalysis( +s"The limit expression must be equal to or greater than 0, but got $v") + case _ => // OK +} } } http://git-wip-us.apache.org/repos/asf/spark/blob/4ac2126b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala -- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala index f4cfed4..ae8d77b 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala @@ -400,6 +400,12 @@ class AnalysisErrorSuite extends AnalysisTest { ) errorTest( +"an evaluated limit class must not be null", +testRelation.limit(Literal(null, IntegerType)), +"The evaluated limit expression must not be null, but got " :: Nil + ) + + errorTest( "num_rows in limit clause must be equal to or greater than 0", listRelation.limit(-1), "The limit expression must be equal to or greater than 0, but got -1" :: Nil http://git-wip-us.apache.org/repos/asf/spark/blob/4ac2126b/sql/core/src/test/resources/sql-tests/inputs/limit.sql -- diff --git a/sql/core/src/test/resources/sql-tests/inputs/limit.sql b/sql/core/src/test/resources/sql-tests/inputs/limit.sql index f21912a..b4c73cf 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/limit.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/limit.sql @@ -13,6 +13,11 @@ SELECT * FROM testdata LIMIT CAST(1 AS int); SELECT * FROM testdata LIMIT -1; SELECT * FROM testData TABLESAMPLE (-1 ROWS); + +SELECT * FROM testdata LIMIT CAST(1 AS INT); +-- evaluated limit must not be null +SELECT * FROM testdata LIMIT CAST(NULL AS INT); + -- limit must be foldable SELECT * FROM testdata LIMIT key > 3; http://git-wip-us.apache.org/repos/asf/spark/blob/4ac2126b/sql/core/src/test/resources/sql-tests/results/limit.sql.out
spark git commit: [SPARK-24536] Validate that an evaluated limit clause cannot be null
Repository: spark Updated Branches: refs/heads/branch-2.3 25ea27b09 -> fc3df4517 [SPARK-24536] Validate that an evaluated limit clause cannot be null It proposes a version in which nullable expressions are not valid in the limit clause It was tested with unit and e2e tests. Please review http://spark.apache.org/contributing.html before opening a pull request. Author: Mauro Palsgraaf Closes #21807 from mauropalsgraaf/SPARK-24536. (cherry picked from commit 4ac2126bc64bad1b4cbe1c697b4bcafacd67c96c) Signed-off-by: Xiao Li Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fc3df451 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fc3df451 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fc3df451 Branch: refs/heads/branch-2.3 Commit: fc3df45177d176cc0fe43049b6f8df372f7ea0e0 Parents: 25ea27b Author: Mauro Palsgraaf Authored: Tue Jul 31 08:18:08 2018 -0700 Committer: Xiao Li Committed: Tue Jul 31 08:22:25 2018 -0700 -- .../sql/catalyst/analysis/CheckAnalysis.scala | 14 +++--- .../catalyst/analysis/AnalysisErrorSuite.scala | 6 +++ .../test/resources/sql-tests/inputs/limit.sql | 5 +++ .../resources/sql-tests/results/limit.sql.out | 45 ++-- 4 files changed, 51 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/fc3df451/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala index 0d189b4..beb11d7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala @@ -66,11 +66,15 @@ trait CheckAnalysis extends PredicateHelper { limitExpr.sql) case e if e.dataType != IntegerType => failAnalysis( s"The limit expression must be integer type, but got " + - e.dataType.simpleString) - case e if e.eval().asInstanceOf[Int] < 0 => failAnalysis( -"The limit expression must be equal to or greater than 0, but got " + - e.eval().asInstanceOf[Int]) - case e => // OK + e.dataType.catalogString) + case e => +e.eval() match { + case null => failAnalysis( +s"The evaluated limit expression must not be null, but got ${limitExpr.sql}") + case v: Int if v < 0 => failAnalysis( +s"The limit expression must be equal to or greater than 0, but got $v") + case _ => // OK +} } } http://git-wip-us.apache.org/repos/asf/spark/blob/fc3df451/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala -- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala index 5d2f8e7..70325b8 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala @@ -393,6 +393,12 @@ class AnalysisErrorSuite extends AnalysisTest { ) errorTest( +"an evaluated limit class must not be null", +testRelation.limit(Literal(null, IntegerType)), +"The evaluated limit expression must not be null, but got " :: Nil + ) + + errorTest( "num_rows in limit clause must be equal to or greater than 0", listRelation.limit(-1), "The limit expression must be equal to or greater than 0, but got -1" :: Nil http://git-wip-us.apache.org/repos/asf/spark/blob/fc3df451/sql/core/src/test/resources/sql-tests/inputs/limit.sql -- diff --git a/sql/core/src/test/resources/sql-tests/inputs/limit.sql b/sql/core/src/test/resources/sql-tests/inputs/limit.sql index f21912a..b4c73cf 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/limit.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/limit.sql @@ -13,6 +13,11 @@ SELECT * FROM testdata LIMIT CAST(1 AS int); SELECT * FROM testdata LIMIT -1; SELECT * FROM testData TABLESAMPLE (-1 ROWS); + +SELECT * FROM testdata LIMIT CAST(1 AS INT); +-- evaluated limit must not be null +SELECT * FROM testdata LIMIT CAST(NULL AS INT); + -- limit must be foldable SELECT * FROM testdata LIMIT key > 3; http://git-wip-us.apache.org/repos/asf/spark/blob/fc3df451/sql/core/src/test/resources/sql-tests/results/li