Github user HyukjinKwon commented on a diff in the pull request:

    https://github.com/apache/spark/pull/22237#discussion_r213003614
  
    --- Diff: 
sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala ---
    @@ -469,4 +470,23 @@ class JsonFunctionsSuite extends QueryTest with 
SharedSQLContext {
     
         checkAnswer(sql("""select json[0] from jsonTable"""), Seq(Row(null)))
       }
    +
    +  test("from_json invalid json - check modes") {
    +    val df = Seq("""{"a" 1}""", """{"a": 2}""").toDS()
    +    val schema = new StructType().add("a", IntegerType)
    +
    +    checkAnswer(
    +      df.select(from_json($"value", schema, Map("mode" -> "PERMISSIVE"))),
    +      Row(Row(null)) :: Row(Row(2)) :: Nil)
    +
    +    val exceptionOne = intercept[SparkException] {
    +      df.select(from_json($"value", schema, Map("mode" -> 
"FAILFAST"))).collect()
    +    }.getMessage
    +    assert(exceptionOne.contains(
    +      "Malformed records are detected in record parsing. Parse Mode: 
FAILFAST."))
    +
    +    checkAnswer(
    +      df.select(from_json($"value", schema, Map("mode" -> 
"DROPMALFORMED"))),
    +      Row(null) :: Row(Row(2)) :: Nil)
    --- End diff --
    
    Nope, only possibility I raised was to make it generator expression. I 
haven't proposed a parse mode for this reason so far.


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

Reply via email to