This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 5358bd168 [CELEBORN-1255] Fix MR UT
5358bd168 is described below
commit 5358bd16842c82d30ba8ce35d1f6ccc48f318650
Author: [cxzl25 <[email protected]>
AuthorDate: Tue Jan 30 09:58:29 2024 +0800
[CELEBORN-1255] Fix MR UT
### What changes were proposed in this pull request?
To fix MR ut caused by incorrect classpath.
### Why are the changes needed?
The error is the same as below:
### Does this PR introduce _any_ user-facing change?
NO.
### How was this patch tested?
GA.
Closes #2266 from FMX/B1255.
Lead-authored-by: [cxzl25 <[email protected]>]
Co-authored-by: cxzl25 <[email protected]>
Co-authored-by: mingji <[email protected]>
Signed-off-by: zky.zhoukeyong <[email protected]>
---
.../test/scala/org/apache/celeborn/tests/mr/WordCountTest.scala | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git
a/tests/mr-it/src/test/scala/org/apache/celeborn/tests/mr/WordCountTest.scala
b/tests/mr-it/src/test/scala/org/apache/celeborn/tests/mr/WordCountTest.scala
index d7c0ca41f..18c7fd676 100644
---
a/tests/mr-it/src/test/scala/org/apache/celeborn/tests/mr/WordCountTest.scala
+++
b/tests/mr-it/src/test/scala/org/apache/celeborn/tests/mr/WordCountTest.scala
@@ -77,6 +77,7 @@ class WordCountTest extends AnyFunSuite with Logging with
MiniClusterFeature
classOf[Service])
yarnCluster = new MiniYARNCluster("MiniClusterWordCount", 1, 1, 1)
+ logInfo(s"Test working dir ${yarnCluster.getTestWorkDir.getAbsolutePath}")
yarnCluster.init(hadoopConf)
yarnCluster.start()
}
@@ -142,12 +143,17 @@ class WordCountTest extends AnyFunSuite with Logging with
MiniClusterFeature
(Utils.getCodeSourceLocation(getClass).split("/").dropRight(1) ++
Array(
"mapreduce_lib")).mkString("/")
val excludeJarList =
- Seq("hadoop-client-api", "hadoop-client-runtime",
"hadoop-client-minicluster")
+ Seq(
+ "hadoop-client-api",
+ "hadoop-client-runtime",
+ "hadoop-client-minicluster",
+ "celeborn-client-mr-shaded")
Files.list(Paths.get(mapreduceLibPath)).iterator().asScala.foreach(path => {
if (!excludeJarList.exists(path.toFile.getPath.contains(_))) {
job.addFileToClassPath(new Path(path.toString))
}
})
+ logInfo(s"Job class path
${job.getFileClassPaths.map(_.toString).mkString(",")}")
exitCode = job.waitForCompletion(true)
if (exitCode) {
