Github user HyukjinKwon commented on a diff in the pull request:
https://github.com/apache/spark/pull/20959#discussion_r180942904
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
---
@@ -1279,4 +1278,68 @@ class CSVSuite extends QueryTest with
SharedSQLContext with SQLTestUtils {
Row("0,2013-111-11 12:13:14") :: Row(null) :: Nil
)
}
+
+ test("SPARK-23846: schema inferring touches less data if samplingRatio <
1.0") {
+ // Set default values for the DataSource parameters to make sure
+ // that whole test file is mapped to only one partition. This will
guarantee
+ // reliable sampling of the input file.
+ withSQLConf(
+ "spark.sql.files.maxPartitionBytes" -> (128 * 1024 * 1024).toString,
+ "spark.sql.files.openCostInBytes" -> (4 * 1024 * 1024).toString
+ )(withTempPath { path =>
+ val rdd = spark.sqlContext.range(0, 100).map {row =>
+ val predefinedSample = Set[Long](2, 8, 15, 27, 30, 34, 35, 37, 44,
46,
+ 57, 62, 68, 72)
+ val value = row.getLong(0)
+ if (predefinedSample.contains(value)) {
+ value.toString
+ } else {
+ (value.toDouble + 0.1).toString
+ }
+ }.repartition(1)
+ rdd.write.text(path.getAbsolutePath)
+
+ val ds = spark.read
+ .option("inferSchema", true)
+ .option("samplingRatio", 0.1)
+ .csv(path.getCanonicalPath)
+ assert(ds.schema == new StructType().add("_c0", IntegerType))
+ })
+ }
+
+ test("SPARK-23846: usage of samplingRatio while parsing a dataset of
strings") {
+ val dstr = spark.sparkContext.parallelize(0 until 100, 1).map { i =>
+ val predefinedSample = Set[Int](2, 8, 15, 27, 30, 34, 35, 37, 44, 46,
+ 57, 62, 68, 72)
+ if (predefinedSample.contains(i)) {
+ i.toString + "\n"
+ } else {
+ (i.toDouble + 0.1) + "\n"
--- End diff --
Hm .. does it need `\n`?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
