spark git commit: [SPARK-21170][CORE] Utils.tryWithSafeFinallyAndFailureCallbacks throws IllegalArgumentException: Self-suppression not permitted
Repository: spark Updated Branches: refs/heads/branch-2.2 85fddf406 -> 6fd39ea1c [SPARK-21170][CORE] Utils.tryWithSafeFinallyAndFailureCallbacks throws IllegalArgumentException: Self-suppression not permitted ## What changes were proposed in this pull request? Not adding the exception to the suppressed if it is the same instance as originalThrowable. ## How was this patch tested? Added new tests to verify this, these tests fail without source code changes and passes with the change. Author: Devaraj K Closes #18384 from devaraj-kavali/SPARK-21170. (cherry picked from commit 6beca9ce94f484de2f9ffb946bef8334781b3122) Signed-off-by: Sean Owen Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6fd39ea1 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6fd39ea1 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6fd39ea1 Branch: refs/heads/branch-2.2 Commit: 6fd39ea1c9dbf68763cb394a28d8a13c116341df Parents: 85fddf4 Author: Devaraj K Authored: Sat Jul 1 15:53:49 2017 +0100 Committer: Sean Owen Committed: Sat Jul 1 15:54:18 2017 +0100 -- .../scala/org/apache/spark/util/Utils.scala | 30 +++ .../org/apache/spark/util/UtilsSuite.scala | 88 +++- 2 files changed, 99 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/6fd39ea1/core/src/main/scala/org/apache/spark/util/Utils.scala -- diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 67497bb..999486c 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -1345,14 +1345,10 @@ private[spark] object Utils extends Logging { try { finallyBlock } catch { -case t: Throwable => - if (originalThrowable != null) { -originalThrowable.addSuppressed(t) -logWarning(s"Suppressing exception in finally: " + t.getMessage, t) -throw originalThrowable - } else { -throw t - } +case t: Throwable if (originalThrowable != null && originalThrowable != t) => + originalThrowable.addSuppressed(t) + logWarning(s"Suppressing exception in finally: ${t.getMessage}", t) + throw originalThrowable } } } @@ -1384,22 +1380,20 @@ private[spark] object Utils extends Logging { catchBlock } catch { case t: Throwable => -originalThrowable.addSuppressed(t) -logWarning(s"Suppressing exception in catch: " + t.getMessage, t) +if (originalThrowable != t) { + originalThrowable.addSuppressed(t) + logWarning(s"Suppressing exception in catch: ${t.getMessage}", t) +} } throw originalThrowable } finally { try { finallyBlock } catch { -case t: Throwable => - if (originalThrowable != null) { -originalThrowable.addSuppressed(t) -logWarning(s"Suppressing exception in finally: " + t.getMessage, t) -throw originalThrowable - } else { -throw t - } +case t: Throwable if (originalThrowable != null && originalThrowable != t) => + originalThrowable.addSuppressed(t) + logWarning(s"Suppressing exception in finally: ${t.getMessage}", t) + throw originalThrowable } } } http://git-wip-us.apache.org/repos/asf/spark/blob/6fd39ea1/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala -- diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala index 3339d5b..d130a1d 100644 --- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala @@ -38,7 +38,7 @@ import org.apache.commons.math3.stat.inference.ChiSquareTest import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path -import org.apache.spark.{SparkConf, SparkFunSuite} +import org.apache.spark.{SparkConf, SparkFunSuite, TaskContext} import org.apache.spark.internal.Logging import org.apache.spark.network.util.ByteUnit @@ -1025,4 +1025,90 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging { assert(redactedConf("spark.sensitive.property") === Utils.REDACTION_REPLACEMENT_TEXT) } + + test("tryWithSafeFinally") { +var e = new Error("Block0") +val finallyBlockError = new Error("Finally Block") +var isErrorOccurred = false +// if the try and finally bl
spark git commit: [SPARK-21170][CORE] Utils.tryWithSafeFinallyAndFailureCallbacks throws IllegalArgumentException: Self-suppression not permitted
Repository: spark Updated Branches: refs/heads/master e0b047eaf -> 6beca9ce9 [SPARK-21170][CORE] Utils.tryWithSafeFinallyAndFailureCallbacks throws IllegalArgumentException: Self-suppression not permitted ## What changes were proposed in this pull request? Not adding the exception to the suppressed if it is the same instance as originalThrowable. ## How was this patch tested? Added new tests to verify this, these tests fail without source code changes and passes with the change. Author: Devaraj K Closes #18384 from devaraj-kavali/SPARK-21170. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6beca9ce Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6beca9ce Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6beca9ce Branch: refs/heads/master Commit: 6beca9ce94f484de2f9ffb946bef8334781b3122 Parents: e0b047e Author: Devaraj K Authored: Sat Jul 1 15:53:49 2017 +0100 Committer: Sean Owen Committed: Sat Jul 1 15:53:49 2017 +0100 -- .../scala/org/apache/spark/util/Utils.scala | 30 +++ .../org/apache/spark/util/UtilsSuite.scala | 88 +++- 2 files changed, 99 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/6beca9ce/core/src/main/scala/org/apache/spark/util/Utils.scala -- diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index bbb7999..26f61e2 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -1348,14 +1348,10 @@ private[spark] object Utils extends Logging { try { finallyBlock } catch { -case t: Throwable => - if (originalThrowable != null) { -originalThrowable.addSuppressed(t) -logWarning(s"Suppressing exception in finally: " + t.getMessage, t) -throw originalThrowable - } else { -throw t - } +case t: Throwable if (originalThrowable != null && originalThrowable != t) => + originalThrowable.addSuppressed(t) + logWarning(s"Suppressing exception in finally: ${t.getMessage}", t) + throw originalThrowable } } } @@ -1387,22 +1383,20 @@ private[spark] object Utils extends Logging { catchBlock } catch { case t: Throwable => -originalThrowable.addSuppressed(t) -logWarning(s"Suppressing exception in catch: " + t.getMessage, t) +if (originalThrowable != t) { + originalThrowable.addSuppressed(t) + logWarning(s"Suppressing exception in catch: ${t.getMessage}", t) +} } throw originalThrowable } finally { try { finallyBlock } catch { -case t: Throwable => - if (originalThrowable != null) { -originalThrowable.addSuppressed(t) -logWarning(s"Suppressing exception in finally: " + t.getMessage, t) -throw originalThrowable - } else { -throw t - } +case t: Throwable if (originalThrowable != null && originalThrowable != t) => + originalThrowable.addSuppressed(t) + logWarning(s"Suppressing exception in finally: ${t.getMessage}", t) + throw originalThrowable } } } http://git-wip-us.apache.org/repos/asf/spark/blob/6beca9ce/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala -- diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala index f7bc8f8..4ce143f 100644 --- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala @@ -38,7 +38,7 @@ import org.apache.commons.math3.stat.inference.ChiSquareTest import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path -import org.apache.spark.{SparkConf, SparkFunSuite} +import org.apache.spark.{SparkConf, SparkFunSuite, TaskContext} import org.apache.spark.internal.Logging import org.apache.spark.network.util.ByteUnit @@ -1024,4 +1024,90 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging { assert(redactedConf("spark.sensitive.property") === Utils.REDACTION_REPLACEMENT_TEXT) } + + test("tryWithSafeFinally") { +var e = new Error("Block0") +val finallyBlockError = new Error("Finally Block") +var isErrorOccurred = false +// if the try and finally blocks throw different exception instances +try { + Utils.tryWithSafeFinally { throw e }(finallyB
spark git commit: [SPARK-18518][ML] HasSolver supports override
Repository: spark Updated Branches: refs/heads/master 37ef32e51 -> e0b047eaf [SPARK-18518][ML] HasSolver supports override ## What changes were proposed in this pull request? 1, make param support non-final with `finalFields` option 2, generate `HasSolver` with `finalFields = false` 3, override `solver` in LiR, GLR, and make MLPC inherit `HasSolver` ## How was this patch tested? existing tests Author: Ruifeng Zheng Author: Zheng RuiFeng Closes #16028 from zhengruifeng/param_non_final. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e0b047ea Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e0b047ea Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e0b047ea Branch: refs/heads/master Commit: e0b047eafed92eadf6842a9df964438095e12d41 Parents: 37ef32e Author: Ruifeng Zheng Authored: Sat Jul 1 15:37:41 2017 +0800 Committer: Yanbo Liang Committed: Sat Jul 1 15:37:41 2017 +0800 -- .../MultilayerPerceptronClassifier.scala| 19 .../ml/param/shared/SharedParamsCodeGen.scala | 11 +++-- .../spark/ml/param/shared/sharedParams.scala| 8 ++-- .../GeneralizedLinearRegression.scala | 21 - .../spark/ml/regression/LinearRegression.scala | 46 +++- python/pyspark/ml/classification.py | 18 +--- python/pyspark/ml/regression.py | 5 +++ 7 files changed, 82 insertions(+), 46 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/e0b047ea/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala -- diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala index ec39f96..ceba11e 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala @@ -27,13 +27,16 @@ import org.apache.spark.ml.ann.{FeedForwardTopology, FeedForwardTrainer} import org.apache.spark.ml.feature.LabeledPoint import org.apache.spark.ml.linalg.{Vector, Vectors} import org.apache.spark.ml.param._ -import org.apache.spark.ml.param.shared.{HasMaxIter, HasSeed, HasStepSize, HasTol} +import org.apache.spark.ml.param.shared._ import org.apache.spark.ml.util._ import org.apache.spark.sql.Dataset /** Params for Multilayer Perceptron. */ private[classification] trait MultilayerPerceptronParams extends PredictorParams - with HasSeed with HasMaxIter with HasTol with HasStepSize { + with HasSeed with HasMaxIter with HasTol with HasStepSize with HasSolver { + + import MultilayerPerceptronClassifier._ + /** * Layer sizes including input size and output size. * @@ -78,14 +81,10 @@ private[classification] trait MultilayerPerceptronParams extends PredictorParams * @group expertParam */ @Since("2.0.0") - final val solver: Param[String] = new Param[String](this, "solver", + final override val solver: Param[String] = new Param[String](this, "solver", "The solver algorithm for optimization. Supported options: " + - s"${MultilayerPerceptronClassifier.supportedSolvers.mkString(", ")}. (Default l-bfgs)", - ParamValidators.inArray[String](MultilayerPerceptronClassifier.supportedSolvers)) - - /** @group expertGetParam */ - @Since("2.0.0") - final def getSolver: String = $(solver) + s"${supportedSolvers.mkString(", ")}. (Default l-bfgs)", +ParamValidators.inArray[String](supportedSolvers)) /** * The initial weights of the model. @@ -101,7 +100,7 @@ private[classification] trait MultilayerPerceptronParams extends PredictorParams final def getInitialWeights: Vector = $(initialWeights) setDefault(maxIter -> 100, tol -> 1e-6, blockSize -> 128, -solver -> MultilayerPerceptronClassifier.LBFGS, stepSize -> 0.03) +solver -> LBFGS, stepSize -> 0.03) } /** Label to vector converter. */ http://git-wip-us.apache.org/repos/asf/spark/blob/e0b047ea/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala -- diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala b/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala index 013817a..23e0d45 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala @@ -80,8 +80,7 @@ private[shared] object SharedParamsCodeGen { " 0)", isValid = "ParamValidators.gt(0)"),