sanha closed pull request #146: [NEMO-244] Organize integration tests and resources URL: https://github.com/apache/incubator-nemo/pull/146
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/common/src/main/java/org/apache/nemo/common/test/ExampleTestArgs.java b/common/src/main/java/org/apache/nemo/common/test/ExampleTestArgs.java new file mode 100644 index 000000000..71586173b --- /dev/null +++ b/common/src/main/java/org/apache/nemo/common/test/ExampleTestArgs.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.nemo.common.test; + +/** + * Test Arguments for Examples. + */ +public final class ExampleTestArgs { + public static final int TIMEOUT = 240000; + private static String fileBasePath; + + private ExampleTestArgs() { + + } + + public static String getFileBasePath() { + if (fileBasePath == null) { + fileBasePath = System.getProperty("user.dir") + "/../resources/"; + } + return fileBasePath; + } +} diff --git a/compiler/test/src/main/java/org/apache/nemo/compiler/CompilerTestUtil.java b/compiler/test/src/main/java/org/apache/nemo/compiler/CompilerTestUtil.java index 6f348173f..2a4d35987 100644 --- a/compiler/test/src/main/java/org/apache/nemo/compiler/CompilerTestUtil.java +++ b/compiler/test/src/main/java/org/apache/nemo/compiler/CompilerTestUtil.java @@ -93,8 +93,8 @@ private static String findRoot(final String curDir) { } public static DAG<IRVertex, IREdge> compileWordCountDAG() throws Exception { - final String input = ROOT_DIR + "/examples/resources/test_input_wordcount"; - final String output = ROOT_DIR + "/examples/resources/test_output"; + final String input = ROOT_DIR + "/examples/resources/inputs/test_input_wordcount"; + final String output = ROOT_DIR + "/examples/resources/inputs/test_output"; final String main = "org.apache.nemo.examples.beam.WordCount"; final ArgBuilder mrArgBuilder = new ArgBuilder() @@ -105,7 +105,7 @@ private static String findRoot(final String curDir) { } public static DAG<IRVertex, IREdge> compileALSDAG() throws Exception { - final String input = ROOT_DIR + "/examples/resources/test_input_als"; + final String input = ROOT_DIR + "/examples/resources/inputs/test_input_als"; final String numFeatures = "10"; final String numIteration = "3"; final String main = "org.apache.nemo.examples.beam.AlternatingLeastSquare"; @@ -118,7 +118,7 @@ private static String findRoot(final String curDir) { } public static DAG<IRVertex, IREdge> compileALSInefficientDAG() throws Exception { - final String input = ROOT_DIR + "/examples/resources/test_input_als"; + final String input = ROOT_DIR + "/examples/resources/inputs/test_input_als"; final String numFeatures = "10"; final String numIteration = "3"; final String main = "org.apache.nemo.examples.beam.AlternatingLeastSquareInefficient"; @@ -131,7 +131,7 @@ private static String findRoot(final String curDir) { } public static DAG<IRVertex, IREdge> compileMLRDAG() throws Exception { - final String input = ROOT_DIR + "/examples/resources/test_input_mlr"; + final String input = ROOT_DIR + "/examples/resources/inputs/test_input_mlr"; final String numFeatures = "100"; final String numClasses = "5"; final String numIteration = "3"; diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/AlternatingLeastSquareITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/AlternatingLeastSquareITCase.java index 3def35091..095104a89 100644 --- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/AlternatingLeastSquareITCase.java +++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/AlternatingLeastSquareITCase.java @@ -20,6 +20,7 @@ import org.apache.nemo.client.JobLauncher; import org.apache.nemo.common.test.ArgBuilder; +import org.apache.nemo.common.test.ExampleTestArgs; import org.apache.nemo.common.test.ExampleTestUtil; import org.apache.nemo.compiler.optimizer.policy.DefaultPolicy; import org.apache.nemo.examples.beam.policy.DefaultPolicyParallelismFive; @@ -37,16 +38,13 @@ @RunWith(PowerMockRunner.class) @PrepareForTest(JobLauncher.class) public final class AlternatingLeastSquareITCase { - private static final int TIMEOUT = 240 * 1000; private static ArgBuilder builder; - private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/"; - - private static final String input = fileBasePath + "test_input_als"; + private static final String input = ExampleTestArgs.getFileBasePath() + "inputs/test_input_als"; private static final String outputFileName = "test_output_als"; - private static final String output = fileBasePath + outputFileName; - private static final String expectedOutputFileName = "expected_output_als"; - private static final String noPoisonResources = fileBasePath + "beam_test_executor_resources.json"; - private static final String poisonedResource = fileBasePath + "beam_test_poisoned_executor_resources.json"; + private static final String output = ExampleTestArgs.getFileBasePath() + outputFileName; + private static final String expectedOutputFileName = "outputs/expected_output_als"; + private static final String noPoisonResources = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json"; + private static final String poisonedResource = ExampleTestArgs.getFileBasePath() + "executors/beam_test_poisoned_executor_resources.json"; private static final String numFeatures = "10"; private static final String numIteration = "3"; private static final String lambda = "0.05"; @@ -61,13 +59,13 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { try { - ExampleTestUtil.ensureALSOutputValidity(fileBasePath, outputFileName, expectedOutputFileName); + ExampleTestUtil.ensureALSOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFileName); } finally { - ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName); + ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName); } } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void testDefault() throws Exception { JobLauncher.main(builder .addResourceJson(noPoisonResources) diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/BeamSimpleSumSQLITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/BeamSimpleSumSQLITCase.java index af3a3ffec..ccb1ca09d 100644 --- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/BeamSimpleSumSQLITCase.java +++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/BeamSimpleSumSQLITCase.java @@ -20,6 +20,7 @@ import org.apache.nemo.client.JobLauncher; import org.apache.nemo.common.test.ArgBuilder; +import org.apache.nemo.common.test.ExampleTestArgs; import org.apache.nemo.common.test.ExampleTestUtil; import org.apache.nemo.examples.beam.policy.DefaultPolicyParallelismFive; import org.junit.After; @@ -35,14 +36,12 @@ @RunWith(PowerMockRunner.class) @PrepareForTest(JobLauncher.class) public final class BeamSimpleSumSQLITCase { - private static final int TIMEOUT = 180000; private static ArgBuilder builder; - private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/"; private static final String outputFileName = "test_output_simplesql"; - private static final String expectedOutputFileName = "expected_output_simplesql"; - private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json"; - private static final String outputFilePath = fileBasePath + outputFileName; + private static final String expectedOutputFileName = "outputs/expected_output_simplesql"; + private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json"; + private static final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName; @Before public void setUp() throws Exception { @@ -55,13 +54,13 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { try { - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFileName); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFileName); } finally { - ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName); + ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName); } } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void test() throws Exception { JobLauncher.main(builder .addJobId(BeamSimpleSumSQLITCase.class.getSimpleName()) diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/BroadcastITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/BroadcastITCase.java index e8e8e6c58..425eaa152 100644 --- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/BroadcastITCase.java +++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/BroadcastITCase.java @@ -20,6 +20,7 @@ import org.apache.nemo.client.JobLauncher; import org.apache.nemo.common.test.ArgBuilder; +import org.apache.nemo.common.test.ExampleTestArgs; import org.apache.nemo.common.test.ExampleTestUtil; import org.apache.nemo.examples.beam.policy.DefaultPolicyParallelismFive; import org.apache.nemo.examples.beam.policy.TransientResourcePolicyParallelismFive; @@ -36,16 +37,14 @@ @RunWith(PowerMockRunner.class) @PrepareForTest(JobLauncher.class) public final class BroadcastITCase { - private static final int TIMEOUT = 180000; private static ArgBuilder builder; - private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/"; - private static final String inputFileName = "test_input_wordcount"; + private static final String inputFileName = "inputs/test_input_wordcount"; private static final String outputFileName = "test_output_broadcast"; - private static final String expectedOutputFileName = "expected_output_broadcast"; - private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json"; - private static final String inputFilePath = fileBasePath + inputFileName; - private static final String outputFilePath = fileBasePath + outputFileName; + private static final String expectedOutputFileName = "outputs/expected_output_broadcast"; + private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json"; + private static final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName; + private static final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName; @Before public void setUp() throws Exception { @@ -58,13 +57,13 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { try { - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFileName); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFileName); } finally { - ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName); + ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName); } } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void test() throws Exception { JobLauncher.main(builder .addJobId(BroadcastITCase.class.getSimpleName()) @@ -72,7 +71,7 @@ public void test() throws Exception { .build()); } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void testTransientResource() throws Exception { JobLauncher.main(builder .addJobId(BroadcastITCase.class.getSimpleName() + "_transient") diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/MultinomialLogisticRegressionITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/MultinomialLogisticRegressionITCase.java index 60860cec2..3d38e5ff4 100644 --- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/MultinomialLogisticRegressionITCase.java +++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/MultinomialLogisticRegressionITCase.java @@ -20,6 +20,7 @@ import org.apache.nemo.client.JobLauncher; import org.apache.nemo.common.test.ArgBuilder; +import org.apache.nemo.common.test.ExampleTestArgs; import org.apache.nemo.compiler.optimizer.policy.DefaultPolicy; import org.apache.nemo.examples.beam.policy.DefaultPolicyParallelismFive; import org.junit.Before; @@ -34,19 +35,17 @@ @RunWith(PowerMockRunner.class) @PrepareForTest(JobLauncher.class) public final class MultinomialLogisticRegressionITCase { - private static final int TIMEOUT = 240000; private static ArgBuilder builder = new ArgBuilder(); - private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/"; - private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json"; + private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json"; @Before public void setUp() throws Exception { builder = new ArgBuilder(); } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void test() throws Exception { - final String input = fileBasePath + "test_input_mlr"; + final String input = ExampleTestArgs.getFileBasePath() + "inputs/test_input_mlr"; final String numFeatures = "100"; final String numClasses = "5"; final String numIteration = "3"; diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/NetworkTraceAnalysisITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/NetworkTraceAnalysisITCase.java index 85f714eb6..40057311a 100644 --- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/NetworkTraceAnalysisITCase.java +++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/NetworkTraceAnalysisITCase.java @@ -20,6 +20,7 @@ import org.apache.nemo.client.JobLauncher; import org.apache.nemo.common.test.ArgBuilder; +import org.apache.nemo.common.test.ExampleTestArgs; import org.apache.nemo.common.test.ExampleTestUtil; import org.apache.nemo.examples.beam.policy.DefaultPolicyParallelismFive; import org.apache.nemo.examples.beam.policy.TransientResourcePolicyParallelismFive; @@ -34,18 +35,16 @@ @RunWith(PowerMockRunner.class) @PrepareForTest(JobLauncher.class) public final class NetworkTraceAnalysisITCase { - private static final int TIMEOUT = 120000; private static ArgBuilder builder; - private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/"; - private static final String inputFileName0 = "test_input_network0"; - private static final String inputFileName1 = "test_input_network1"; + private static final String inputFileName0 = "inputs/test_input_network0"; + private static final String inputFileName1 = "inputs/test_input_network1"; private static final String outputFileName = "test_output_network"; - private static final String expectedOutputFileName = "expected_output_network"; - private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json"; - private static final String inputFilePath0 = fileBasePath + inputFileName0; - private static final String inputFilePath1 = fileBasePath + inputFileName1; - private static final String outputFilePath = fileBasePath + outputFileName; + private static final String expectedOutputFileName = "outputs/expected_output_network"; + private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json"; + private static final String inputFilePath0 = ExampleTestArgs.getFileBasePath() + inputFileName0; + private static final String inputFilePath1 = ExampleTestArgs.getFileBasePath() + inputFileName1; + private static final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName; @Before public void setUp() throws Exception { @@ -58,13 +57,13 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { try { - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFileName); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFileName); } finally { - ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName); + ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName); } } - @Test(timeout = TIMEOUT) + @Test(timeout = ExampleTestArgs.TIMEOUT) public void test() throws Exception { JobLauncher.main(builder .addJobId(NetworkTraceAnalysisITCase.class.getSimpleName()) @@ -72,7 +71,7 @@ public void test() throws Exception { .build()); } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void testLargeShuffle() throws Exception { JobLauncher.main(builder .addJobId(NetworkTraceAnalysisITCase.class.getSimpleName() + "_largeshuffle") @@ -80,7 +79,7 @@ public void testLargeShuffle() throws Exception { .build()); } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void testTransientResource() throws Exception { JobLauncher.main(builder .addJobId(NetworkTraceAnalysisITCase.class.getSimpleName() + "_transient") diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/PartitionWordsByLengthITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/PartitionWordsByLengthITCase.java index aacd20484..093910510 100644 --- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/PartitionWordsByLengthITCase.java +++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/PartitionWordsByLengthITCase.java @@ -20,6 +20,7 @@ import org.apache.nemo.client.JobLauncher; import org.apache.nemo.common.test.ArgBuilder; +import org.apache.nemo.common.test.ExampleTestArgs; import org.apache.nemo.common.test.ExampleTestUtil; import org.apache.nemo.examples.beam.policy.DefaultPolicyParallelismFive; import org.apache.nemo.examples.beam.policy.LargeShufflePolicyParallelismFive; @@ -36,16 +37,14 @@ @RunWith(PowerMockRunner.class) @PrepareForTest(JobLauncher.class) public final class PartitionWordsByLengthITCase { - private static final int TIMEOUT = 120000; private static ArgBuilder builder; - private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/"; - private static final String inputFileName = "test_input_tag"; + private static final String inputFileName = "inputs/test_input_tag"; private static final String outputFileName = "test_output_tag"; - private static final String expectedOutputFileName = "expected_output_tag"; - private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json"; - private static final String inputFilePath = fileBasePath + inputFileName; - private static final String outputFilePath = fileBasePath + outputFileName; + private static final String expectedOutputFileName = "outputs/expected_output_tag"; + private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json"; + private static final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName; + private static final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName; @Before public void setUp() throws Exception { @@ -57,16 +56,16 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { try { - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName + "_short", expectedOutputFileName + "_short"); - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName + "_long", expectedOutputFileName + "_long"); - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName + "_very_long", expectedOutputFileName + "_very_long"); - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName + "_very_very_long", expectedOutputFileName + "_very_very_long"); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName + "_short", expectedOutputFileName + "_short"); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName + "_long", expectedOutputFileName + "_long"); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName + "_very_long", expectedOutputFileName + "_very_long"); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName + "_very_very_long", expectedOutputFileName + "_very_very_long"); } finally { - ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName); + ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName); } } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void testLargeShuffle() throws Exception { JobLauncher.main(builder .addResourceJson(executorResourceFileName) @@ -75,7 +74,7 @@ public void testLargeShuffle() throws Exception { .build()); } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void test() throws Exception { JobLauncher.main(builder .addResourceJson(executorResourceFileName) diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/PerKeyMedianITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/PerKeyMedianITCase.java index 41fc6c602..39d8d28f8 100644 --- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/PerKeyMedianITCase.java +++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/PerKeyMedianITCase.java @@ -20,6 +20,7 @@ import org.apache.nemo.client.JobLauncher; import org.apache.nemo.common.test.ArgBuilder; +import org.apache.nemo.common.test.ExampleTestArgs; import org.apache.nemo.common.test.ExampleTestUtil; import org.apache.nemo.examples.beam.policy.DataSkewPolicyParallelismFive; import org.junit.After; @@ -35,16 +36,14 @@ @RunWith(PowerMockRunner.class) @PrepareForTest(JobLauncher.class) public final class PerKeyMedianITCase { - private static final int TIMEOUT = 120 * 1000; private static ArgBuilder builder; - private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/"; - private static final String inputFileName = "test_input_median"; + private static final String inputFileName = "inputs/test_input_median"; private static final String outputFileName = "test_output_median"; - private static final String expectedOutputFileName = "expected_output_median"; - private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json"; - private static final String inputFilePath = fileBasePath + inputFileName; - private static final String outputFilePath = fileBasePath + outputFileName; + private static final String expectedOutputFileName = "outputs/expected_output_median"; + private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json"; + private static final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName; + private static final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName; @Before public void setUp() throws Exception { @@ -57,9 +56,9 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { try { - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFileName); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFileName); } finally { - ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName); + ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName); } } @@ -67,7 +66,7 @@ public void tearDown() throws Exception { * Testing data skew dynamic optimization. * @throws Exception exception on the way. */ - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void testDataSkew() throws Exception { JobLauncher.main(builder .addJobId(PerKeyMedianITCase.class.getSimpleName()) diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/PerPercentileAverageITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/PerPercentileAverageITCase.java index 76e238cdb..b43a939cd 100644 --- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/PerPercentileAverageITCase.java +++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/PerPercentileAverageITCase.java @@ -20,6 +20,7 @@ import org.apache.nemo.client.JobLauncher; import org.apache.nemo.common.test.ArgBuilder; +import org.apache.nemo.common.test.ExampleTestArgs; import org.apache.nemo.common.test.ExampleTestUtil; import org.apache.nemo.examples.beam.policy.DefaultPolicyParallelismFive; import org.junit.After; @@ -35,16 +36,14 @@ @RunWith(PowerMockRunner.class) @PrepareForTest(JobLauncher.class) public final class PerPercentileAverageITCase { - private static final int TIMEOUT = 120000; private static ArgBuilder builder; - private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/"; - private static final String inputFileName = "test_input_partition"; + private static final String inputFileName = "inputs/test_input_partition"; private static final String outputFileName = "test_output_partition"; - private static final String expectedOutputFileName = "expected_output_partition"; - private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json"; - private static final String inputFilePath = fileBasePath + inputFileName; - private static final String outputFilePath = fileBasePath + outputFileName; + private static final String expectedOutputFileName = "outputs/expected_output_partition"; + private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json"; + private static final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName; + private static final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName; @Before public void setUp() throws Exception { @@ -58,16 +57,16 @@ public void setUp() throws Exception { public void tearDown() throws Exception { try { for (int i = 0; i < 10; i++) { - ExampleTestUtil.ensureOutputValidity(fileBasePath, + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName + "_" + i, expectedOutputFileName + "_" + i); } } finally { - ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName); + ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName); } } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void test() throws Exception { JobLauncher.main(builder .addJobId(PerPercentileAverage.class.getSimpleName()) diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/WindowedWordCountITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/WindowedWordCountITCase.java index 7ef0e0281..97dafe506 100644 --- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/WindowedWordCountITCase.java +++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/WindowedWordCountITCase.java @@ -20,6 +20,7 @@ import org.apache.nemo.client.JobLauncher; import org.apache.nemo.common.test.ArgBuilder; +import org.apache.nemo.common.test.ExampleTestArgs; import org.apache.nemo.common.test.ExampleTestUtil; import org.apache.nemo.compiler.optimizer.policy.DefaultPolicy; import org.apache.nemo.examples.beam.policy.*; @@ -38,20 +39,17 @@ @RunWith(PowerMockRunner.class) @PrepareForTest(JobLauncher.class) public final class WindowedWordCountITCase { - - private static final int TIMEOUT = 120000; private static ArgBuilder builder; - private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/"; - private static final String inputFileName = "test_input_windowed_wordcount"; + private static final String inputFileName = "inputs/test_input_windowed_wordcount"; private static final String outputFileName = "test_output_windowed_wordcount"; - private static final String expectedOutputFileName = "expected_output_windowed_wordcount"; - private static final String expectedSlidingWindowOutputFileName = "expected_output_sliding_windowed_wordcount"; - private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json"; - private static final String inputFilePath = fileBasePath + inputFileName; - private static final String outputFilePath = fileBasePath + outputFileName; + private static final String expectedOutputFileName = "outputs/expected_output_windowed_wordcount"; + private static final String expectedSlidingWindowOutputFileName = "outputs/expected_output_sliding_windowed_wordcount"; + private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json"; + private static final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName; + private static final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName; - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void testBatchFixedWindow() throws Exception { builder = new ArgBuilder() .addUserMain(WindowedWordCount.class.getCanonicalName()) @@ -64,14 +62,14 @@ public void testBatchFixedWindow() throws Exception { .build()); try { - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFileName); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFileName); } finally { - ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName); + ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName); } } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void testBatchSlidingWindow() throws Exception { builder = new ArgBuilder() .addUserMain(WindowedWordCount.class.getCanonicalName()) @@ -84,13 +82,13 @@ public void testBatchSlidingWindow() throws Exception { .build()); try { - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedSlidingWindowOutputFileName); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedSlidingWindowOutputFileName); } finally { - ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName); + ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName); } } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void testStreamingSchedulerAndPipeFixedWindow() throws Exception { builder = new ArgBuilder() .addScheduler("org.apache.nemo.runtime.master.scheduler.StreamingScheduler") @@ -104,14 +102,14 @@ public void testStreamingSchedulerAndPipeFixedWindow() throws Exception { .build()); try { - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFileName); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFileName); } finally { - ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName); + ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName); } } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void testStreamingSchedulerAndPipeSlidingWindow() throws Exception { builder = new ArgBuilder() .addScheduler("org.apache.nemo.runtime.master.scheduler.StreamingScheduler") @@ -125,9 +123,9 @@ public void testStreamingSchedulerAndPipeSlidingWindow() throws Exception { .build()); try { - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedSlidingWindowOutputFileName); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedSlidingWindowOutputFileName); } finally { - ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName); + ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName); } } @@ -147,7 +145,7 @@ public void testUnboundedSlidingWindow() throws Exception { .build()); try { - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedSlidingWindowOutputFileName); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedSlidingWindowOutputFileName); } finally { } } diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/WordCountITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/WordCountITCase.java index 052bb323f..e31a8c5a4 100644 --- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/WordCountITCase.java +++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/WordCountITCase.java @@ -20,6 +20,7 @@ import org.apache.nemo.client.JobLauncher; import org.apache.nemo.common.test.ArgBuilder; +import org.apache.nemo.common.test.ExampleTestArgs; import org.apache.nemo.common.test.ExampleTestUtil; import org.apache.nemo.compiler.optimizer.policy.ConditionalLargeShufflePolicy; import org.apache.nemo.examples.beam.policy.*; @@ -36,17 +37,15 @@ @RunWith(PowerMockRunner.class) @PrepareForTest(JobLauncher.class) public final class WordCountITCase { - private static final int TIMEOUT = 120000; private static ArgBuilder builder; - private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/"; - private static final String inputFileName = "test_input_wordcount"; + private static final String inputFileName = "inputs/test_input_wordcount"; private static final String outputFileName = "test_output_wordcount"; - private static final String expectedOutputFileName = "expected_output_wordcount"; - private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json"; - private static final String oneExecutorResourceFileName = fileBasePath + "beam_test_one_executor_resources.json"; - private static final String inputFilePath = fileBasePath + inputFileName; - private static final String outputFilePath = fileBasePath + outputFileName; + private static final String expectedOutputFileName = "outputs/expected_output_wordcount"; + private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json"; + private static final String oneExecutorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_one_executor_resources.json"; + private static final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName; + private static final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName; @Before public void setUp() throws Exception { @@ -58,13 +57,13 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { try { - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFileName); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFileName); } finally { - ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName); + ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName); } } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void test() throws Exception { JobLauncher.main(builder .addResourceJson(executorResourceFileName) @@ -73,7 +72,7 @@ public void test() throws Exception { .build()); } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void testLargeShuffle() throws Exception { JobLauncher.main(builder .addResourceJson(executorResourceFileName) @@ -82,7 +81,7 @@ public void testLargeShuffle() throws Exception { .build()); } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void testLargeShuffleInOneExecutor() throws Exception { JobLauncher.main(builder .addResourceJson(oneExecutorResourceFileName) @@ -91,7 +90,7 @@ public void testLargeShuffleInOneExecutor() throws Exception { .build()); } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void testConditionalLargeShuffle() throws Exception { JobLauncher.main(builder .addResourceJson(executorResourceFileName) @@ -100,7 +99,7 @@ public void testConditionalLargeShuffle() throws Exception { .build()); } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void testTransientResource() throws Exception { JobLauncher.main(builder .addResourceJson(executorResourceFileName) @@ -109,7 +108,7 @@ public void testTransientResource() throws Exception { .build()); } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void testClonedScheduling() throws Exception { JobLauncher.main(builder .addResourceJson(executorResourceFileName) @@ -119,7 +118,7 @@ public void testClonedScheduling() throws Exception { .build()); } - @Test (timeout = TIMEOUT) + @Test (timeout = ExampleTestArgs.TIMEOUT) public void testSpeculativeExecution() throws Exception { JobLauncher.main(builder .addResourceJson(executorResourceFileName) diff --git a/examples/resources/beam_test_executor_resources.json b/examples/resources/executors/beam_test_executor_resources.json similarity index 100% rename from examples/resources/beam_test_executor_resources.json rename to examples/resources/executors/beam_test_executor_resources.json diff --git a/examples/resources/beam_test_one_executor_resources.json b/examples/resources/executors/beam_test_one_executor_resources.json similarity index 100% rename from examples/resources/beam_test_one_executor_resources.json rename to examples/resources/executors/beam_test_one_executor_resources.json diff --git a/examples/resources/beam_test_poisoned_executor_resources.json b/examples/resources/executors/beam_test_poisoned_executor_resources.json similarity index 100% rename from examples/resources/beam_test_poisoned_executor_resources.json rename to examples/resources/executors/beam_test_poisoned_executor_resources.json diff --git a/examples/resources/spark_test_executor_resources.json b/examples/resources/executors/spark_test_executor_resources.json similarity index 100% rename from examples/resources/spark_test_executor_resources.json rename to examples/resources/executors/spark_test_executor_resources.json diff --git a/examples/resources/test_input_als b/examples/resources/inputs/test_input_als similarity index 100% rename from examples/resources/test_input_als rename to examples/resources/inputs/test_input_als diff --git a/examples/resources/test_input_employees.json b/examples/resources/inputs/test_input_employees.json similarity index 100% rename from examples/resources/test_input_employees.json rename to examples/resources/inputs/test_input_employees.json diff --git a/examples/resources/test_input_median b/examples/resources/inputs/test_input_median similarity index 100% rename from examples/resources/test_input_median rename to examples/resources/inputs/test_input_median diff --git a/examples/resources/test_input_mlr b/examples/resources/inputs/test_input_mlr similarity index 100% rename from examples/resources/test_input_mlr rename to examples/resources/inputs/test_input_mlr diff --git a/examples/resources/test_input_network0 b/examples/resources/inputs/test_input_network0 similarity index 100% rename from examples/resources/test_input_network0 rename to examples/resources/inputs/test_input_network0 diff --git a/examples/resources/test_input_network1 b/examples/resources/inputs/test_input_network1 similarity index 100% rename from examples/resources/test_input_network1 rename to examples/resources/inputs/test_input_network1 diff --git a/examples/resources/test_input_partition b/examples/resources/inputs/test_input_partition similarity index 100% rename from examples/resources/test_input_partition rename to examples/resources/inputs/test_input_partition diff --git a/examples/resources/test_input_people.json b/examples/resources/inputs/test_input_people.json similarity index 100% rename from examples/resources/test_input_people.json rename to examples/resources/inputs/test_input_people.json diff --git a/examples/resources/test_input_people.txt b/examples/resources/inputs/test_input_people.txt similarity index 100% rename from examples/resources/test_input_people.txt rename to examples/resources/inputs/test_input_people.txt diff --git a/examples/resources/test_input_tag b/examples/resources/inputs/test_input_tag similarity index 100% rename from examples/resources/test_input_tag rename to examples/resources/inputs/test_input_tag diff --git a/examples/resources/test_input_windowed_wordcount b/examples/resources/inputs/test_input_windowed_wordcount similarity index 100% rename from examples/resources/test_input_windowed_wordcount rename to examples/resources/inputs/test_input_windowed_wordcount diff --git a/examples/resources/test_input_wordcount b/examples/resources/inputs/test_input_wordcount similarity index 100% rename from examples/resources/test_input_wordcount rename to examples/resources/inputs/test_input_wordcount diff --git a/examples/resources/test_input_wordcount_spark b/examples/resources/inputs/test_input_wordcount_spark similarity index 100% rename from examples/resources/test_input_wordcount_spark rename to examples/resources/inputs/test_input_wordcount_spark diff --git a/examples/resources/expected_output_als b/examples/resources/outputs/expected_output_als similarity index 100% rename from examples/resources/expected_output_als rename to examples/resources/outputs/expected_output_als diff --git a/examples/resources/expected_output_broadcast b/examples/resources/outputs/expected_output_broadcast similarity index 100% rename from examples/resources/expected_output_broadcast rename to examples/resources/outputs/expected_output_broadcast diff --git a/examples/resources/expected_output_median b/examples/resources/outputs/expected_output_median similarity index 100% rename from examples/resources/expected_output_median rename to examples/resources/outputs/expected_output_median diff --git a/examples/resources/expected_output_network b/examples/resources/outputs/expected_output_network similarity index 100% rename from examples/resources/expected_output_network rename to examples/resources/outputs/expected_output_network diff --git a/examples/resources/expected_output_partition_0 b/examples/resources/outputs/expected_output_partition_0 similarity index 100% rename from examples/resources/expected_output_partition_0 rename to examples/resources/outputs/expected_output_partition_0 diff --git a/examples/resources/expected_output_partition_1 b/examples/resources/outputs/expected_output_partition_1 similarity index 100% rename from examples/resources/expected_output_partition_1 rename to examples/resources/outputs/expected_output_partition_1 diff --git a/examples/resources/expected_output_partition_2 b/examples/resources/outputs/expected_output_partition_2 similarity index 100% rename from examples/resources/expected_output_partition_2 rename to examples/resources/outputs/expected_output_partition_2 diff --git a/examples/resources/expected_output_partition_3 b/examples/resources/outputs/expected_output_partition_3 similarity index 100% rename from examples/resources/expected_output_partition_3 rename to examples/resources/outputs/expected_output_partition_3 diff --git a/examples/resources/expected_output_partition_4 b/examples/resources/outputs/expected_output_partition_4 similarity index 100% rename from examples/resources/expected_output_partition_4 rename to examples/resources/outputs/expected_output_partition_4 diff --git a/examples/resources/expected_output_partition_5 b/examples/resources/outputs/expected_output_partition_5 similarity index 100% rename from examples/resources/expected_output_partition_5 rename to examples/resources/outputs/expected_output_partition_5 diff --git a/examples/resources/expected_output_partition_6 b/examples/resources/outputs/expected_output_partition_6 similarity index 100% rename from examples/resources/expected_output_partition_6 rename to examples/resources/outputs/expected_output_partition_6 diff --git a/examples/resources/expected_output_partition_7 b/examples/resources/outputs/expected_output_partition_7 similarity index 100% rename from examples/resources/expected_output_partition_7 rename to examples/resources/outputs/expected_output_partition_7 diff --git a/examples/resources/expected_output_partition_8 b/examples/resources/outputs/expected_output_partition_8 similarity index 100% rename from examples/resources/expected_output_partition_8 rename to examples/resources/outputs/expected_output_partition_8 diff --git a/examples/resources/expected_output_partition_9 b/examples/resources/outputs/expected_output_partition_9 similarity index 100% rename from examples/resources/expected_output_partition_9 rename to examples/resources/outputs/expected_output_partition_9 diff --git a/examples/resources/expected_output_reversed_wordcount_spark b/examples/resources/outputs/expected_output_reversed_wordcount_spark similarity index 100% rename from examples/resources/expected_output_reversed_wordcount_spark rename to examples/resources/outputs/expected_output_reversed_wordcount_spark diff --git a/examples/resources/expected_output_simplesql b/examples/resources/outputs/expected_output_simplesql similarity index 100% rename from examples/resources/expected_output_simplesql rename to examples/resources/outputs/expected_output_simplesql diff --git a/examples/resources/expected_output_sliding_windowed_wordcount b/examples/resources/outputs/expected_output_sliding_windowed_wordcount similarity index 100% rename from examples/resources/expected_output_sliding_windowed_wordcount rename to examples/resources/outputs/expected_output_sliding_windowed_wordcount diff --git a/examples/resources/expected_output_tag_long b/examples/resources/outputs/expected_output_tag_long similarity index 100% rename from examples/resources/expected_output_tag_long rename to examples/resources/outputs/expected_output_tag_long diff --git a/examples/resources/expected_output_tag_short b/examples/resources/outputs/expected_output_tag_short similarity index 100% rename from examples/resources/expected_output_tag_short rename to examples/resources/outputs/expected_output_tag_short diff --git a/examples/resources/expected_output_tag_very_long b/examples/resources/outputs/expected_output_tag_very_long similarity index 100% rename from examples/resources/expected_output_tag_very_long rename to examples/resources/outputs/expected_output_tag_very_long diff --git a/examples/resources/expected_output_tag_very_very_long b/examples/resources/outputs/expected_output_tag_very_very_long similarity index 100% rename from examples/resources/expected_output_tag_very_very_long rename to examples/resources/outputs/expected_output_tag_very_very_long diff --git a/examples/resources/expected_output_windowed_wordcount b/examples/resources/outputs/expected_output_windowed_wordcount similarity index 100% rename from examples/resources/expected_output_windowed_wordcount rename to examples/resources/outputs/expected_output_windowed_wordcount diff --git a/examples/resources/expected_output_word_and_line_count b/examples/resources/outputs/expected_output_word_and_line_count similarity index 100% rename from examples/resources/expected_output_word_and_line_count rename to examples/resources/outputs/expected_output_word_and_line_count diff --git a/examples/resources/expected_output_wordcount b/examples/resources/outputs/expected_output_wordcount similarity index 100% rename from examples/resources/expected_output_wordcount rename to examples/resources/outputs/expected_output_wordcount diff --git a/examples/resources/expected_output_wordcount_spark b/examples/resources/outputs/expected_output_wordcount_spark similarity index 100% rename from examples/resources/expected_output_wordcount_spark rename to examples/resources/outputs/expected_output_wordcount_spark diff --git a/examples/spark/src/test/java/org/apache/nemo/examples/spark/MRJava.java b/examples/spark/src/test/java/org/apache/nemo/examples/spark/MRJava.java index d89ef8b0d..39ecae5f7 100644 --- a/examples/spark/src/test/java/org/apache/nemo/examples/spark/MRJava.java +++ b/examples/spark/src/test/java/org/apache/nemo/examples/spark/MRJava.java @@ -20,6 +20,7 @@ import org.apache.nemo.client.JobLauncher; import org.apache.nemo.common.test.ArgBuilder; +import org.apache.nemo.common.test.ExampleTestArgs; import org.apache.nemo.common.test.ExampleTestUtil; import org.apache.nemo.compiler.optimizer.policy.DefaultPolicy; import org.junit.Before; @@ -36,10 +37,8 @@ @PrepareForTest(JobLauncher.class) @PowerMockIgnore("javax.management.*") public final class MRJava { - private static final int TIMEOUT = 180000; private static ArgBuilder builder; - private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/"; - private static final String executorResourceFileName = fileBasePath + "spark_test_executor_resources.json"; + private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "/executors/spark_test_executor_resources.json"; @Before public void setUp() { @@ -47,13 +46,13 @@ public void setUp() { .addResourceJson(executorResourceFileName); } - @Test(timeout = TIMEOUT) + @Test(timeout = ExampleTestArgs.TIMEOUT) public void testSparkWordCount() throws Exception { - final String inputFileName = "test_input_wordcount_spark"; + final String inputFileName = "/inputs/test_input_wordcount_spark"; final String outputFileName = "test_output_wordcount_spark"; - final String expectedOutputFilename = "expected_output_wordcount_spark"; - final String inputFilePath = fileBasePath + inputFileName; - final String outputFilePath = fileBasePath + outputFileName; + final String expectedOutputFilename = "/outputs/expected_output_wordcount_spark"; + final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName; + final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName; JobLauncher.main(builder .addJobId(JavaWordCount.class.getSimpleName() + "_test") @@ -63,19 +62,19 @@ public void testSparkWordCount() throws Exception { .build()); try { - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFilename); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFilename); } finally { - ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName); + ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName); } } - @Test(timeout = TIMEOUT) + @Test(timeout = ExampleTestArgs.TIMEOUT) public void testSparkWordAndLineCount() throws Exception { - final String inputFileName = "test_input_wordcount_spark"; + final String inputFileName = "/inputs/test_input_wordcount_spark"; final String outputFileName = "test_output_word_and_line_count"; - final String expectedOutputFilename = "expected_output_word_and_line_count"; - final String inputFilePath = fileBasePath + inputFileName; - final String outputFilePath = fileBasePath + outputFileName; + final String expectedOutputFilename = "/outputs/expected_output_word_and_line_count"; + final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName; + final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName; JobLauncher.main(builder .addJobId(JavaWordAndLineCount.class.getSimpleName() + "_test") @@ -85,9 +84,9 @@ public void testSparkWordAndLineCount() throws Exception { .build()); try { - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFilename); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFilename); } finally { - ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName); + ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName); } } } diff --git a/examples/spark/src/test/java/org/apache/nemo/examples/spark/SparkJava.java b/examples/spark/src/test/java/org/apache/nemo/examples/spark/SparkJava.java index b09da383c..1ebbefcd3 100644 --- a/examples/spark/src/test/java/org/apache/nemo/examples/spark/SparkJava.java +++ b/examples/spark/src/test/java/org/apache/nemo/examples/spark/SparkJava.java @@ -20,6 +20,7 @@ import org.apache.nemo.client.JobLauncher; import org.apache.nemo.common.test.ArgBuilder; +import org.apache.nemo.common.test.ExampleTestArgs; import org.apache.nemo.common.test.ExampleTestUtil; import org.apache.nemo.compiler.optimizer.policy.DefaultPolicy; import org.apache.nemo.examples.spark.sql.JavaUserDefinedTypedAggregation; @@ -38,10 +39,8 @@ @PrepareForTest(JobLauncher.class) @PowerMockIgnore("javax.management.*") public final class SparkJava { - private static final int TIMEOUT = 180000; private static ArgBuilder builder; - private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/"; - private static final String executorResourceFileName = fileBasePath + "spark_test_executor_resources.json"; + private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "inputs/spark_test_executor_resources.json"; @Before public void setUp() { @@ -49,7 +48,7 @@ public void setUp() { .addResourceJson(executorResourceFileName); } - @Test(timeout = TIMEOUT) + @Test(timeout = ExampleTestArgs.TIMEOUT) public void testSparkPi() throws Exception { final String numParallelism = "3"; @@ -61,10 +60,10 @@ public void testSparkPi() throws Exception { .build()); } - @Test(timeout = TIMEOUT) + @Test(timeout = ExampleTestArgs.TIMEOUT) public void testSparkSQLUserDefinedTypedAggregation() throws Exception { - final String inputFileName = "test_input_employees.json"; - final String inputFilePath = fileBasePath + inputFileName; + final String inputFileName = "inputs/test_input_employees.json"; + final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName; JobLauncher.main(builder .addJobId(JavaUserDefinedTypedAggregation.class.getSimpleName() + "_test") @@ -74,10 +73,10 @@ public void testSparkSQLUserDefinedTypedAggregation() throws Exception { .build()); } - @Test(timeout = TIMEOUT) + @Test(timeout = ExampleTestArgs.TIMEOUT) public void testSparkSQLUserDefinedUntypedAggregation() throws Exception { - final String inputFileName = "test_input_employees.json"; - final String inputFilePath = fileBasePath + inputFileName; + final String inputFileName = "inputs/test_input_employees.json"; + final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName; JobLauncher.main(builder .addJobId(JavaUserDefinedUntypedAggregation.class.getSimpleName() + "_test") @@ -87,12 +86,12 @@ public void testSparkSQLUserDefinedUntypedAggregation() throws Exception { .build()); } - @Test(timeout = TIMEOUT) + @Test(timeout = ExampleTestArgs.TIMEOUT) public void testSparkSQLExample() throws Exception { - final String peopleJson = "test_input_people.json"; - final String peopleTxt = "test_input_people.txt"; - final String inputFileJson = fileBasePath + peopleJson; - final String inputFileTxt = fileBasePath + peopleTxt; + final String peopleJson = "inputs/test_input_people.json"; + final String peopleTxt = "inputs/test_input_people.txt"; + final String inputFileJson = ExampleTestArgs.getFileBasePath() + peopleJson; + final String inputFileTxt = ExampleTestArgs.getFileBasePath() + peopleTxt; // TODO#12: Frontend support for Scala Spark. // JobLauncher.main(builder diff --git a/examples/spark/src/test/java/org/apache/nemo/examples/spark/SparkScala.java b/examples/spark/src/test/java/org/apache/nemo/examples/spark/SparkScala.java index 3dcf490ad..0ef2f1576 100644 --- a/examples/spark/src/test/java/org/apache/nemo/examples/spark/SparkScala.java +++ b/examples/spark/src/test/java/org/apache/nemo/examples/spark/SparkScala.java @@ -20,6 +20,7 @@ import org.apache.nemo.client.JobLauncher; import org.apache.nemo.common.test.ArgBuilder; +import org.apache.nemo.common.test.ExampleTestArgs; import org.apache.nemo.common.test.ExampleTestUtil; import org.apache.nemo.compiler.optimizer.policy.DefaultPolicy; import org.junit.Before; @@ -36,10 +37,8 @@ @PrepareForTest(JobLauncher.class) @PowerMockIgnore("javax.management.*") public final class SparkScala { - private static final int TIMEOUT = 120000; private static ArgBuilder builder; - private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/"; - private static final String executorResourceFileName = fileBasePath + "spark_test_executor_resources.json"; + private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/spark_test_executor_resources.json"; @Before public void setUp() { @@ -47,7 +46,7 @@ public void setUp() { .addResourceJson(executorResourceFileName); } - @Test(timeout = TIMEOUT) + @Test(timeout = ExampleTestArgs.TIMEOUT) public void testPi() throws Exception { final String numParallelism = "3"; @@ -59,13 +58,13 @@ public void testPi() throws Exception { .build()); } - @Test(timeout = TIMEOUT) + @Test(timeout = ExampleTestArgs.TIMEOUT) public void testWordCount() throws Exception { - final String inputFileName = "test_input_wordcount_spark"; - final String outputFileName = "test_output_wordcount_spark"; - final String expectedOutputFilename = "expected_output_wordcount_spark"; - final String inputFilePath = fileBasePath + inputFileName; - final String outputFilePath = fileBasePath + outputFileName; + final String inputFileName = "inputs/test_input_wordcount_spark"; + final String outputFileName = "inputs/test_output_wordcount_spark"; + final String expectedOutputFilename = "/outputs/expected_output_wordcount_spark"; + final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName; + final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName; JobLauncher.main(builder .addJobId(SparkWordCount.class.getSimpleName() + "_test") @@ -75,22 +74,22 @@ public void testWordCount() throws Exception { .build()); try { - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFilename); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFilename); } finally { - ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName); + ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName); } } - @Test(timeout = TIMEOUT) + @Test(timeout = ExampleTestArgs.TIMEOUT) public void testCachingWordCount() throws Exception { - final String inputFileName = "test_input_wordcount_spark"; + final String inputFileName = "inputs/test_input_wordcount_spark"; final String outputFileName1 = "test_output_wordcount_spark"; final String outputFileName2 = "test_output_reversed_wordcount_spark"; - final String expectedOutputFilename1 = "expected_output_wordcount_spark"; - final String expectedOutputFilename2 = "expected_output_reversed_wordcount_spark"; - final String inputFilePath = fileBasePath + inputFileName; - final String outputFilePath1 = fileBasePath + outputFileName1; - final String outputFilePath2 = fileBasePath + outputFileName2; + final String expectedOutputFilename1 = "outputs/expected_output_wordcount_spark"; + final String expectedOutputFilename2 = "outputs/expected_output_reversed_wordcount_spark"; + final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName; + final String outputFilePath1 = ExampleTestArgs.getFileBasePath() + outputFileName1; + final String outputFilePath2 = ExampleTestArgs.getFileBasePath() + outputFileName2; JobLauncher.main(builder .addJobId(SparkCachingWordCount.class.getSimpleName() + "_test") @@ -100,15 +99,15 @@ public void testCachingWordCount() throws Exception { .build()); try { - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName1, expectedOutputFilename1); - ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName2, expectedOutputFilename2); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName1, expectedOutputFilename1); + ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName2, expectedOutputFilename2); } finally { - ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName1); - ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName2); + ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName1); + ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName2); } } - @Test(timeout = TIMEOUT) + @Test(timeout = ExampleTestArgs.TIMEOUT) public void testALS() throws Exception { JobLauncher.main(builder .addJobId(SparkALS.class.getSimpleName() + "_test") @@ -118,7 +117,7 @@ public void testALS() throws Exception { .build()); } - @Test(timeout = TIMEOUT) + @Test(timeout = ExampleTestArgs.TIMEOUT) public void testALSEmptyUserArgs() throws Exception { JobLauncher.main(builder .addJobId(SparkALS.class.getSimpleName() + "_test") ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
