Github user HyukjinKwon commented on a diff in the pull request: https://github.com/apache/spark/pull/22237#discussion_r212849989 --- Diff: sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala --- @@ -469,4 +470,23 @@ class JsonFunctionsSuite extends QueryTest with SharedSQLContext { checkAnswer(sql("""select json[0] from jsonTable"""), Seq(Row(null))) } + + test("from_json invalid json - check modes") { + val df = Seq("""{"a" 1}""", """{"a": 2}""").toDS() + val schema = new StructType().add("a", IntegerType) + + checkAnswer( + df.select(from_json($"value", schema, Map("mode" -> "PERMISSIVE"))), + Row(Row(null)) :: Row(Row(2)) :: Nil) + + val exceptionOne = intercept[SparkException] { + df.select(from_json($"value", schema, Map("mode" -> "FAILFAST"))).collect() + }.getMessage + assert(exceptionOne.contains( + "Malformed records are detected in record parsing. Parse Mode: FAILFAST.")) + + checkAnswer( + df.select(from_json($"value", schema, Map("mode" -> "DROPMALFORMED"))), + Row(null) :: Row(Row(2)) :: Nil) --- End diff -- How does it work for DROPMALFORMED mode? This doesn't actually drop the record like JSON datasource.
--- --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org