Repository: spark Updated Branches: refs/heads/master 8afe49141 -> 26ac60806
[SPARK-13487][SQL] User-facing RuntimeConfig interface ## What changes were proposed in this pull request? This patch creates the public API for runtime configuration and an implementation for it. The public runtime configuration includes configs for existing SQL, as well as Hadoop Configuration. This new interface is currently dead code. It will be added to SQLContext and a session entry point to Spark when we add that. ## How was this patch tested? a new unit test suite Author: Reynold Xin <r...@databricks.com> Closes #11378 from rxin/SPARK-13487. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/26ac6080 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/26ac6080 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/26ac6080 Branch: refs/heads/master Commit: 26ac60806cc23527ea8a75986c1eab83d312a15d Parents: 8afe491 Author: Reynold Xin <r...@databricks.com> Authored: Thu Feb 25 23:10:40 2016 -0800 Committer: Yin Huai <yh...@databricks.com> Committed: Thu Feb 25 23:10:40 2016 -0800 ---------------------------------------------------------------------- .../org/apache/spark/sql/RuntimeConfig.scala | 100 +++++++++++++++++++ .../spark/sql/internal/RuntimeConfigImpl.scala | 73 ++++++++++++++ .../org/apache/spark/sql/internal/SQLConf.scala | 3 +- .../spark/sql/internal/RuntimeConfigSuite.scala | 86 ++++++++++++++++ 4 files changed, 261 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/26ac6080/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala new file mode 100644 index 0000000..e90a042 --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +/** + * Runtime configuration interface for Spark. To access this, use `SparkSession.conf`. + * + * @since 2.0.0 + */ +abstract class RuntimeConfig { + + /** + * Sets the given Spark runtime configuration property. + * + * @since 2.0.0 + */ + def set(key: String, value: String): RuntimeConfig + + /** + * Sets the given Spark runtime configuration property. + * + * @since 2.0.0 + */ + def set(key: String, value: Boolean): RuntimeConfig + + /** + * Sets the given Spark runtime configuration property. + * + * @since 2.0.0 + */ + def set(key: String, value: Long): RuntimeConfig + + /** + * Returns the value of Spark runtime configuration property for the given key. + * + * @throws NoSuchElementException if the key is not set and does not have a default value + * @since 2.0.0 + */ + @throws[NoSuchElementException]("if the key is not set") + def get(key: String): String + + /** + * Returns the value of Spark runtime configuration property for the given key. + * + * @since 2.0.0 + */ + def getOption(key: String): Option[String] + + /** + * Resets the configuration property for the given key. + * + * @since 2.0.0 + */ + def unset(key: String): Unit + + /** + * Sets the given Hadoop configuration property. This is passed directly to Hadoop during I/O. + * + * @since 2.0.0 + */ + def setHadoop(key: String, value: String): RuntimeConfig + + /** + * Returns the value of the Hadoop configuration property. + * + * @throws NoSuchElementException if the key is not set + * @since 2.0.0 + */ + @throws[NoSuchElementException]("if the key is not set") + def getHadoop(key: String): String + + /** + * Returns the value of the Hadoop configuration property. + * + * @since 2.0.0 + */ + def getHadoopOption(key: String): Option[String] + + /** + * Resets the Hadoop configuration property for the given key. + * + * @since 2.0.0 + */ + def unsetHadoop(key: String): Unit +} http://git-wip-us.apache.org/repos/asf/spark/blob/26ac6080/sql/core/src/main/scala/org/apache/spark/sql/internal/RuntimeConfigImpl.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/RuntimeConfigImpl.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/RuntimeConfigImpl.scala new file mode 100644 index 0000000..058df1e --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/RuntimeConfigImpl.scala @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.internal + +import org.apache.spark.sql.RuntimeConfig + +/** + * Implementation for [[RuntimeConfig]]. + */ +class RuntimeConfigImpl extends RuntimeConfig { + + private val conf = new SQLConf + + private val hadoopConf = java.util.Collections.synchronizedMap( + new java.util.HashMap[String, String]()) + + override def set(key: String, value: String): RuntimeConfig = { + conf.setConfString(key, value) + this + } + + override def set(key: String, value: Boolean): RuntimeConfig = set(key, value.toString) + + override def set(key: String, value: Long): RuntimeConfig = set(key, value.toString) + + @throws[NoSuchElementException]("if the key is not set") + override def get(key: String): String = conf.getConfString(key) + + override def getOption(key: String): Option[String] = { + try Option(get(key)) catch { + case _: NoSuchElementException => None + } + } + + override def unset(key: String): Unit = conf.unsetConf(key) + + override def setHadoop(key: String, value: String): RuntimeConfig = { + hadoopConf.put(key, value) + this + } + + @throws[NoSuchElementException]("if the key is not set") + override def getHadoop(key: String): String = hadoopConf.synchronized { + if (hadoopConf.containsKey(key)) { + hadoopConf.get(key) + } else { + throw new NoSuchElementException(key) + } + } + + override def getHadoopOption(key: String): Option[String] = { + try Option(getHadoop(key)) catch { + case _: NoSuchElementException => None + } + } + + override def unsetHadoop(key: String): Unit = hadoopConf.remove(key) +} http://git-wip-us.apache.org/repos/asf/spark/blob/26ac6080/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index c1e3f38..9a50ef7 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.internal -import java.util.Properties +import java.util.{NoSuchElementException, Properties} import scala.collection.JavaConverters._ import scala.collection.immutable @@ -649,6 +649,7 @@ class SQLConf extends Serializable with CatalystConf with ParserConf with Loggin } /** Return the value of Spark SQL configuration property for the given key. */ + @throws[NoSuchElementException]("if key is not set") def getConfString(key: String): String = { Option(settings.get(key)). orElse { http://git-wip-us.apache.org/repos/asf/spark/blob/26ac6080/sql/core/src/test/scala/org/apache/spark/sql/internal/RuntimeConfigSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/RuntimeConfigSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/RuntimeConfigSuite.scala new file mode 100644 index 0000000..f809e01 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/RuntimeConfigSuite.scala @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.internal + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.RuntimeConfig + +class RuntimeConfigSuite extends SparkFunSuite { + + private def newConf(): RuntimeConfig = new RuntimeConfigImpl + + test("set and get") { + val conf = newConf() + conf + .set("k1", "v1") + .set("k2", 2) + .set("k3", value = false) + + assert(conf.get("k1") == "v1") + assert(conf.get("k2") == "2") + assert(conf.get("k3") == "false") + + intercept[NoSuchElementException] { + conf.get("notset") + } + } + + test("getOption") { + val conf = newConf().set("k1", "v1") + assert(conf.getOption("k1") == Some("v1")) + assert(conf.getOption("notset") == None) + } + + test("unset") { + val conf = newConf().set("k1", "v1") + assert(conf.get("k1") == "v1") + conf.unset("k1") + intercept[NoSuchElementException] { + conf.get("k1") + } + } + + test("set and get hadoop configuration") { + val conf = newConf() + conf + .setHadoop("k1", "v1") + .setHadoop("k2", "v2") + + assert(conf.getHadoop("k1") == "v1") + assert(conf.getHadoop("k2") == "v2") + + intercept[NoSuchElementException] { + conf.get("notset") + } + } + + test("getHadoopOption") { + val conf = newConf().setHadoop("k1", "v1") + assert(conf.getHadoopOption("k1") == Some("v1")) + assert(conf.getHadoopOption("notset") == None) + } + + test("unsetHadoop") { + val conf = newConf().setHadoop("k1", "v1") + assert(conf.getHadoop("k1") == "v1") + conf.unsetHadoop("k1") + intercept[NoSuchElementException] { + conf.getHadoop("k1") + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org