Github user HyukjinKwon commented on a diff in the pull request: https://github.com/apache/spark/pull/21533#discussion_r194812872 --- Diff: core/src/test/scala/org/apache/spark/SparkContextSuite.scala --- @@ -116,49 +116,52 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu test("basic case for addFile and listFiles") { val dir = Utils.createTempDir() + // file and absolute path for normal path val file1 = File.createTempFile("someprefix1", "somesuffix1", dir) val absolutePath1 = file1.getAbsolutePath + // file and absolute path for relative path val file2 = File.createTempFile("someprefix2", "somesuffix2", dir) val relativePath = file2.getParent + "/../" + file2.getParentFile.getName + "/" + file2.getName val absolutePath2 = file2.getAbsolutePath + // file and absolute path for path with local scheme + val file3 = File.createTempFile("someprefix3", "somesuffix3", dir) + val localPath = "local://" + file3.getParent + "/../" + file3.getParentFile.getName + + "/" + file3.getName + val absolutePath3 = file3.getAbsolutePath + try { Files.write("somewords1", file1, StandardCharsets.UTF_8) Files.write("somewords2", file2, StandardCharsets.UTF_8) - val length1 = file1.length() - val length2 = file2.length() + Files.write("somewords3", file3, StandardCharsets.UTF_8) - sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local")) - sc.addFile(file1.getAbsolutePath) - sc.addFile(relativePath) - sc.parallelize(Array(1), 1).map(x => { - val gotten1 = new File(SparkFiles.get(file1.getName)) - val gotten2 = new File(SparkFiles.get(file2.getName)) - if (!gotten1.exists()) { + def checkGottenFile(file: File, absolutePath: String): Unit = { + val length = file.length() + val gotten = new File(SparkFiles.get(file.getName)) + if (!gotten.exists()) { throw new SparkException("file doesn't exist : " + absolutePath1) } - if (!gotten2.exists()) { - throw new SparkException("file doesn't exist : " + absolutePath2) - } - if (length1 != gotten1.length()) { + if (file.length() != gotten.length()) { throw new SparkException( - s"file has different length $length1 than added file ${gotten1.length()} : " + + s"file has different length $length than added file ${gotten.length()} : " + absolutePath1) } - if (length2 != gotten2.length()) { - throw new SparkException( - s"file has different length $length2 than added file ${gotten2.length()} : " + - absolutePath2) - } - if (absolutePath1 == gotten1.getAbsolutePath) { + if (absolutePath == gotten.getAbsolutePath) { throw new SparkException("file should have been copied :" + absolutePath1) } - if (absolutePath2 == gotten2.getAbsolutePath) { - throw new SparkException("file should have been copied : " + absolutePath2) - } + } + + sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local")) + sc.addFile(file1.getAbsolutePath) + sc.addFile(relativePath) + sc.addFile(localPath) + sc.parallelize(Array(1), 1).map(x => { --- End diff -- nit: ``` map { x => ... } ```
--- --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org