[ https://issues.apache.org/jira/browse/SPARK-39997?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17576229#comment-17576229 ]
Apache Spark commented on SPARK-39997: -------------------------------------- User 'zinking' has created a pull request for this issue: https://github.com/apache/spark/pull/37428 > ParquetSchemaConverter fails match schema by id > ----------------------------------------------- > > Key: SPARK-39997 > URL: https://issues.apache.org/jira/browse/SPARK-39997 > Project: Spark > Issue Type: Bug > Components: SQL > Affects Versions: 3.3.0 > Reporter: Zhen Wang > Priority: Major > > {code:scala} > test("SPARK-38094: absence of field ids: reading nested schema struct field > renamed") { > withTempDir { dir => > // now with nested schema/complex type > val innerTypeRenamed = new StructType().add("c1", IntegerType, true, > withId(6)); > val readSchema = > new StructType() > .add("c", ArrayType(innerTypeRenamed), true, withId(3)) > .add("e", IntegerType, true, withId(5)) > val innerType = new StructType().add("c0", IntegerType, true, withId(6)) > val writeSchema = > new StructType() > .add("c", ArrayType(innerType), true, withId(3)) > .add("randomName", StringType, true) > val writeData = Seq(Row(Seq(Row(100)), "text"), Row(Seq(Row(100)), > "more")) > spark.createDataFrame(writeData.asJava, writeSchema) > .write.mode("overwrite").parquet(dir.getCanonicalPath) > withAllParquetReaders { > > checkAnswer(spark.read.schema(readSchema).parquet(dir.getCanonicalPath), > // a, b, c, d all couldn't be found > Row(Seq(Row(100)), null) :: Row(Seq(Row(100)), null) :: Nil) > } > } > } > {code} -- This message was sent by Atlassian Jira (v8.20.10#820010) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org