Github user HyukjinKwon commented on a diff in the pull request: https://github.com/apache/spark/pull/21837#discussion_r204601282 --- Diff: external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala --- @@ -896,4 +896,33 @@ class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils { assert(count == 8) } } + + test("SPARK-24881: write with compression - avro options") { + withTempPath { dir => + val uncompressDir = s"$dir/uncompress" + val deflateDir = s"$dir/deflate" + val snappyDir = s"$dir/snappy" + + val df = spark.read.format("avro").load(testAvro) + df.write + .option("compression", "uncompressed") + .format("avro") + .save(uncompressDir) + df.write + .options(Map("compression" -> "deflate", "compressionLevel" -> "9")) + .format("avro") + .save(deflateDir) + df.write + .option("compression", "snappy") + .format("avro") + .save(snappyDir) + + val uncompressSize = FileUtils.sizeOfDirectory(new File(uncompressDir)) + val deflateSize = FileUtils.sizeOfDirectory(new File(deflateDir)) --- End diff -- Thank you, @MaxGekk. Can we then check the type of compression at least `avro.codec deflate`?
--- --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org