skonto commented on a change in pull request #25870: [SPARK-27936][K8S] Support
python deps
URL: https://github.com/apache/spark/pull/25870#discussion_r344355922
##########
File path:
resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala
##########
@@ -131,4 +133,41 @@ object Utils extends Logging {
s"under spark home test dir ${sparkHomeDir.toAbsolutePath}!")
}
}
+
+ def getTestFileAbsolutePath(fileName: String, sparkHomeDir: Path): String = {
+ val filePathsFound = Files
+ .walk(sparkHomeDir)
+ .filter(Files.isRegularFile(_))
+ .filter((f: Path) => {f.toFile.getName == fileName})
+ // we should not have more than one here under current test build dir
+ // we only need one though
+ val filePath = filePathsFound
+ .iterator()
+ .asScala
+ .map(_.toAbsolutePath.toString)
+ .toArray
+ .headOption
+ filePath match {
+ case Some(file) => file
+ case _ => throw new SparkException(s"No valid $fileName file was found "
+
+ s"under spark home test dir ${sparkHomeDir.toAbsolutePath}!")
+ }
+ }
+
+ def createZipFile(inFile: String, outFile: String): Unit = {
+ try {
+ val fileToZip = new File(inFile)
+ val fis = new FileInputStream(fileToZip)
+ val fos = new FileOutputStream(outFile)
+ val zipOut = new ZipOutputStream(fos)
+ val zipEntry = new ZipEntry(fileToZip.getName)
+ zipOut.putNextEntry(zipEntry)
+ IOUtils.copy(fis, zipOut)
+ IOUtils.closeQuietly(fis)
+ IOUtils.closeQuietly(zipOut)
+ } catch {
+ case e: Exception => log.error(s"Failed to zip file: $inFile"); throw e
+ }
+ }
}
+
Review comment:
@dongjoon-hyun did I ok :) Usually I add an \n at the end as an old-school
practice for sanity reasons people do it:
https://unix.stackexchange.com/questions/18743/whats-the-point-in-adding-a-new-line-to-the-end-of-a-file
https://thoughtbot.com/blog/no-newline-at-end-of-file
I kept this practice from my old C++ days... but anyway sometimes I confuse
myself too :)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
