Repository: spark Updated Branches: refs/heads/master 3708d13f1 -> f9307d8fc
[SPARK-13176][CORE] Use native file linking instead of external process ln Since Spark requires at least JRE 1.7, it is safe to use built-in java.nio.Files. Author: Jakob Odersky <ja...@odersky.com> Closes #11098 from jodersky/SPARK-13176. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f9307d8f Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f9307d8f Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f9307d8f Branch: refs/heads/master Commit: f9307d8fc5223b4c5be07e3dc691a327f3bbfa7f Parents: 3708d13 Author: Jakob Odersky <ja...@odersky.com> Authored: Tue Feb 9 08:43:46 2016 +0000 Committer: Sean Owen <so...@cloudera.com> Committed: Tue Feb 9 08:43:46 2016 +0000 ---------------------------------------------------------------------- .../scala/org/apache/spark/util/Utils.scala | 27 ++++++-------------- 1 file changed, 8 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/f9307d8f/core/src/main/scala/org/apache/spark/util/Utils.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 9ecbffb..e0c9bf0 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -22,6 +22,7 @@ import java.lang.management.ManagementFactory import java.net._ import java.nio.ByteBuffer import java.nio.channels.Channels +import java.nio.file.Files import java.util.{Locale, Properties, Random, UUID} import java.util.concurrent._ import javax.net.ssl.HttpsURLConnection @@ -34,7 +35,7 @@ import scala.reflect.ClassTag import scala.util.Try import scala.util.control.{ControlThrowable, NonFatal} -import com.google.common.io.{ByteStreams, Files} +import com.google.common.io.{ByteStreams, Files => GFiles} import com.google.common.net.InetAddresses import org.apache.commons.lang3.SystemUtils import org.apache.hadoop.conf.Configuration @@ -516,7 +517,7 @@ private[spark] object Utils extends Logging { // The file does not exist in the target directory. Copy or move it there. if (removeSourceFile) { - Files.move(sourceFile, destFile) + Files.move(sourceFile.toPath, destFile.toPath) } else { logInfo(s"Copying ${sourceFile.getAbsolutePath} to ${destFile.getAbsolutePath}") copyRecursive(sourceFile, destFile) @@ -534,7 +535,7 @@ private[spark] object Utils extends Logging { case (f1, f2) => filesEqualRecursive(f1, f2) } } else if (file1.isFile && file2.isFile) { - Files.equal(file1, file2) + GFiles.equal(file1, file2) } else { false } @@ -548,7 +549,7 @@ private[spark] object Utils extends Logging { val subfiles = source.listFiles() subfiles.foreach(f => copyRecursive(f, new File(dest, f.getName))) } else { - Files.copy(source, dest) + Files.copy(source.toPath, dest.toPath) } } @@ -1596,30 +1597,18 @@ private[spark] object Utils extends Logging { } /** - * Creates a symlink. Note jdk1.7 has Files.createSymbolicLink but not used here - * for jdk1.6 support. Supports windows by doing copy, everything else uses "ln -sf". + * Creates a symlink. * @param src absolute path to the source * @param dst relative path for the destination */ - def symlink(src: File, dst: File) { + def symlink(src: File, dst: File): Unit = { if (!src.isAbsolute()) { throw new IOException("Source must be absolute") } if (dst.isAbsolute()) { throw new IOException("Destination must be relative") } - var cmdSuffix = "" - val linkCmd = if (isWindows) { - // refer to http://technet.microsoft.com/en-us/library/cc771254.aspx - cmdSuffix = " /s /e /k /h /y /i" - "cmd /c xcopy " - } else { - cmdSuffix = "" - "ln -sf " - } - import scala.sys.process._ - (linkCmd + src.getAbsolutePath() + " " + dst.getPath() + cmdSuffix) lines_! - ProcessLogger(line => logInfo(line)) + Files.createSymbolicLink(dst.toPath, src.toPath) } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org